<commit_before>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"html/template"
	"io"
	"log"
	"net"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/gregjones/httpcache"

	"sourcegraph.com/sourcegraph/go-sourcegraph/sourcegraph"
)

var (
	addr          = flag.String("http", ":5400", "HTTP bind address")
	dev           = flag.Bool("dev", false, "development mode")
	sgURLStr      = flag.String("sg", "https://sourcegraph.com", "base Sourcegraph URL")
	sgAssetURLStr = flag.String("sg-asset", "https://sourcegraph.com/static/", "base Sourcegraph asset URL")

	clientTimeout = flag.Duration("client-timeout", time.Second*5, "timeout for HTTP requests")
	queryTimeout  = flag.Duration("query-timeout", time.Second*7, "timeout for query API call")
)

var (
	sgURL      *url.URL
	sgAssetURL *url.URL
)

var (
	httpClient = &http.Client{}
	sgc        = sourcegraph.NewClient(httpClient)
)

func main() {
	log.SetFlags(0)
	flag.Parse()

	httpClient.Timeout = *clientTimeout

	if !*dev {
		httpClient.Transport = newCancelableHTTPMemoryCacheTransport()
	}

	var err error
	sgURL, err = url.Parse(*sgURLStr)
	if err != nil {
		log.Fatal(err)
	}
	*sgURLStr = sgURL.String()
	sgc.BaseURL = sgURL.ResolveReference(&url.URL{Path: "/api/"})
	sgc.UserAgent = "xconf/0.0.1"

	sgAssetURL, err = url.Parse(*sgAssetURLStr)
	if err != nil {
		log.Fatal(err)
	}
	*sgAssetURLStr = sgAssetURL.String()

	if err := parseTemplates(); err != nil {
		log.Fatal(err)
	}

	http.HandleFunc("/", homeHandler)
	http.HandleFunc("/robots.txt", func(w http.ResponseWriter, req *http.Request) {
		io.WriteString(w, `User-agent: *
Allow: /
`)
	})

	log.Println("Listening on", *addr)
	log.Fatal(http.ListenAndServe(*addr, nil))
}

var (
	tmpl      *template.Template
	tmplMu    sync.Mutex
	tmplFuncs = template.FuncMap{
		"popularQueries": func() []string { return popularQueries },
		"queryURL": func(q string) string {
			return "/?" + url.Values{"q": []string{q}}.Encode()
		},
		"sgAssetURL": func(path string) string {
			return sgAssetURL.ResolveReference(&url.URL{Path: path}).String()
		},
		"assetInfix": func() string {
			if *dev {
				return "."
			}
			return ".min."
		},
		"voteLink": func(voteFor, label, class string) template.HTML {
			return template.HTML(fmt.Sprintf(`<a class="%s" target="_blank" href="https://twitter.com/intent/tweet?text=%s&via=srcgraph&url=%s">%s</a>`, class, url.QueryEscape(fmt.Sprintf("I wish #xconf let me search & see examples of %s config files", voteFor)), url.QueryEscape("https://xconf.io"), label))
		},
	}
)

func parseTemplates() error {
	tmplMu.Lock()
	defer tmplMu.Unlock()
	var err error
	tmpl, err = template.New("").Funcs(tmplFuncs).ParseGlob("tmpl/*")
	return err
}

var (
	popularQueries = []string{
		"nodejs",
		"docpad",
		"mysql",
		"postgres",
		"wordpress",
		"ubuntu",
		"apt-get install",
		"add-apt-repository",
		"go get",
		"",
		"RUN",
		"ONBUILD",
		"ENV",
	}
)

func homeHandler(w http.ResponseWriter, r *http.Request) {
	if *dev {
		if err := parseTemplates(); err != nil {
			log.Println(err.Error())
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	}

	if err := r.ParseForm(); err != nil {
		log.Println("ParseForm:", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	var data struct {
		Query   string
		Results []*sourcegraph.Sourcebox

		TimeoutError bool
		OtherError   bool
	}
	data.Query = strings.TrimSpace(r.FormValue("q"))

	if data.Query != "" {
		deadline := time.Now().Add(*queryTimeout)
		var err error
		data.Results, err = query(data.Query, deadline)
		if err != nil {
			if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
				data.TimeoutError = true
			}
			if err == errQueryTimeout {
				data.TimeoutError = true
			}
			if !data.TimeoutError {
				data.OtherError = true
			}
			log.Printf("Query %s error: %s", data.Query, err)
		}
	}

	var tmplFile string
	if r.Header.Get("x-pjax") != "" {
		tmplFile = "results.inc.html"
	} else {
		tmplFile = "home.html"
	}

	if err := tmpl.ExecuteTemplate(w, tmplFile, &data); err != nil {
		log.Println("ExecuteTemplate:", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
}

var isDockerfileInstruction = map[string]bool{
	"from": true, "maintainer": true, "run": true, "cmd": true,
	"expose": true, "env": true, "add": true, "copy": true,
	"entrypoint": true, "volume": true, "user": true, "workdir": true,
	"onbuild": true,
}

func query(query string, deadline time.Time) ([]*sourcegraph.Sourcebox, error) {
	if isDockerfileInstruction[strings.ToLower(query)] {
		// HACK: it searches the JSON encoded text, which means the text looks like "\nADD".
		query = "n" + query
	}
	opt := &sourcegraph.UnitListOptions{
		Query:       query,
		UnitType:    "Dockerfile",
		ListOptions: sourcegraph.ListOptions{PerPage: 4},
	}
	units, _, err := sgc.Units.List(opt)
	if err != nil {
		return nil, err
	}
	var sourceboxURLs []string
	for _, u := range units {
		su, err := u.SourceUnit()
		if err != nil {
			return nil, err
		}
		sourceboxURL := sgc.BaseURL.ResolveReference(&url.URL{
			Path: fmt.Sprintf("/%s@%s/.tree/%s/.sourcebox.json", u.Repo, u.CommitID, su.Files[0]),
		})
		sourceboxURLs = append(sourceboxURLs, sourceboxURL.String())
	}
	return getSourceboxes(sourceboxURLs, deadline)
}

func getSourceboxes(urls []string, deadline time.Time) ([]*sourcegraph.Sourcebox, error) {
	getSourcebox := func(url string) (*sourcegraph.Sourcebox, error) {
		resp, err := httpClient.Get(url)
		if err != nil {
			return nil, err
		}
		defer resp.Body.Close()
		if resp.StatusCode != http.StatusOK {
			return nil, fmt.Errorf("http response status %d from %s", resp.StatusCode, url)
		}
		var sb *sourcegraph.Sourcebox
		return sb, json.NewDecoder(resp.Body).Decode(&sb)
	}

	sbs := make([]*sourcegraph.Sourcebox, len(urls))
	errc := make(chan error)
	for i, url := range urls {
		go func(i int, url string) {
			sb, err := getSourcebox(url)
			sbs[i] = sb
			errc <- err
		}(i, url)
	}

	var firstErr error
	timedOut := time.After(deadline.Sub(time.Now()))
	okCount := 0
	for range urls {
		select {
		case err := <-errc:
			if err == nil {
				okCount++
			} else if err != nil && firstErr == nil {
				firstErr = err
			}
		case <-timedOut:
			if firstErr == nil {
				firstErr = errQueryTimeout
			}
			break
		}
	}

	// non-nil sbs
	okSBs := make([]*sourcegraph.Sourcebox, 0, okCount)
	for _, sb := range sbs {
		if sb != nil {
			okSBs = append(okSBs, sb)
		}
	}
	return okSBs, firstErr
}

var errQueryTimeout = errors.New("results timeout")

func newCancelableHTTPMemoryCacheTransport() http.RoundTripper {
	// httpcache doesn't support CancelRequest; wrap it. TODO(sqs):
	// submit a patch to httpcache to fix this.
	t := httpcache.NewMemoryCacheTransport()
	t.Transport = http.DefaultTransport
	return &cancelableHTTPCacheTransport{t}
}

type cancelableHTTPCacheTransport struct{ *httpcache.Transport }

func (t *cancelableHTTPCacheTransport) CancelRequest(req *http.Request) {
	t.Transport.Transport.(*http.Transport).CancelRequest(req)
}
<commit_msg>http for now :(<commit_after>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"html/template"
	"io"
	"log"
	"net"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/gregjones/httpcache"

	"sourcegraph.com/sourcegraph/go-sourcegraph/sourcegraph"
)

var (
	addr          = flag.String("http", ":5400", "HTTP bind address")
	dev           = flag.Bool("dev", false, "development mode")
	sgURLStr      = flag.String("sg", "https://sourcegraph.com", "base Sourcegraph URL")
	sgAssetURLStr = flag.String("sg-asset", "https://sourcegraph.com/static/", "base Sourcegraph asset URL")

	clientTimeout = flag.Duration("client-timeout", time.Second*5, "timeout for HTTP requests")
	queryTimeout  = flag.Duration("query-timeout", time.Second*7, "timeout for query API call")
)

var (
	sgURL      *url.URL
	sgAssetURL *url.URL
)

var (
	httpClient = &http.Client{}
	sgc        = sourcegraph.NewClient(httpClient)
)

func main() {
	log.SetFlags(0)
	flag.Parse()

	httpClient.Timeout = *clientTimeout

	if !*dev {
		httpClient.Transport = newCancelableHTTPMemoryCacheTransport()
	}

	var err error
	sgURL, err = url.Parse(*sgURLStr)
	if err != nil {
		log.Fatal(err)
	}
	*sgURLStr = sgURL.String()
	sgc.BaseURL = sgURL.ResolveReference(&url.URL{Path: "/api/"})
	sgc.UserAgent = "xconf/0.0.1"

	sgAssetURL, err = url.Parse(*sgAssetURLStr)
	if err != nil {
		log.Fatal(err)
	}
	*sgAssetURLStr = sgAssetURL.String()

	if err := parseTemplates(); err != nil {
		log.Fatal(err)
	}

	http.HandleFunc("/", homeHandler)
	http.HandleFunc("/robots.txt", func(w http.ResponseWriter, req *http.Request) {
		io.WriteString(w, `User-agent: *
Allow: /
`)
	})

	log.Println("Listening on", *addr)
	log.Fatal(http.ListenAndServe(*addr, nil))
}

var (
	tmpl      *template.Template
	tmplMu    sync.Mutex
	tmplFuncs = template.FuncMap{
		"popularQueries": func() []string { return popularQueries },
		"queryURL": func(q string) string {
			return "/?" + url.Values{"q": []string{q}}.Encode()
		},
		"sgAssetURL": func(path string) string {
			return sgAssetURL.ResolveReference(&url.URL{Path: path}).String()
		},
		"assetInfix": func() string {
			if *dev {
				return "."
			}
			return ".min."
		},
		"voteLink": func(voteFor, label, class string) template.HTML {
			return template.HTML(fmt.Sprintf(`<a class="%s" target="_blank" href="https://twitter.com/intent/tweet?text=%s&via=srcgraph&url=%s">%s</a>`, class, url.QueryEscape(fmt.Sprintf("I wish #xconf let me search & see examples of %s config files", voteFor)), url.QueryEscape("http://xconf.io"), label))
		},
	}
)

func parseTemplates() error {
	tmplMu.Lock()
	defer tmplMu.Unlock()
	var err error
	tmpl, err = template.New("").Funcs(tmplFuncs).ParseGlob("tmpl/*")
	return err
}

var (
	popularQueries = []string{
		"nodejs",
		"docpad",
		"mysql",
		"postgres",
		"wordpress",
		"ubuntu",
		"apt-get install",
		"add-apt-repository",
		"go get",
		"",
		"RUN",
		"ONBUILD",
		"ENV",
	}
)

func homeHandler(w http.ResponseWriter, r *http.Request) {
	if *dev {
		if err := parseTemplates(); err != nil {
			log.Println(err.Error())
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	}

	if err := r.ParseForm(); err != nil {
		log.Println("ParseForm:", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	var data struct {
		Query   string
		Results []*sourcegraph.Sourcebox

		TimeoutError bool
		OtherError   bool
	}
	data.Query = strings.TrimSpace(r.FormValue("q"))

	if data.Query != "" {
		deadline := time.Now().Add(*queryTimeout)
		var err error
		data.Results, err = query(data.Query, deadline)
		if err != nil {
			if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
				data.TimeoutError = true
			}
			if err == errQueryTimeout {
				data.TimeoutError = true
			}
			if !data.TimeoutError {
				data.OtherError = true
			}
			log.Printf("Query %s error: %s", data.Query, err)
		}
	}

	var tmplFile string
	if r.Header.Get("x-pjax") != "" {
		tmplFile = "results.inc.html"
	} else {
		tmplFile = "home.html"
	}

	if err := tmpl.ExecuteTemplate(w, tmplFile, &data); err != nil {
		log.Println("ExecuteTemplate:", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
}

var isDockerfileInstruction = map[string]bool{
	"from": true, "maintainer": true, "run": true, "cmd": true,
	"expose": true, "env": true, "add": true, "copy": true,
	"entrypoint": true, "volume": true, "user": true, "workdir": true,
	"onbuild": true,
}

func query(query string, deadline time.Time) ([]*sourcegraph.Sourcebox, error) {
	if isDockerfileInstruction[strings.ToLower(query)] {
		// HACK: it searches the JSON encoded text, which means the text looks like "\nADD".
		query = "n" + query
	}
	opt := &sourcegraph.UnitListOptions{
		Query:       query,
		UnitType:    "Dockerfile",
		ListOptions: sourcegraph.ListOptions{PerPage: 4},
	}
	units, _, err := sgc.Units.List(opt)
	if err != nil {
		return nil, err
	}
	var sourceboxURLs []string
	for _, u := range units {
		su, err := u.SourceUnit()
		if err != nil {
			return nil, err
		}
		sourceboxURL := sgc.BaseURL.ResolveReference(&url.URL{
			Path: fmt.Sprintf("/%s@%s/.tree/%s/.sourcebox.json", u.Repo, u.CommitID, su.Files[0]),
		})
		sourceboxURLs = append(sourceboxURLs, sourceboxURL.String())
	}
	return getSourceboxes(sourceboxURLs, deadline)
}

func getSourceboxes(urls []string, deadline time.Time) ([]*sourcegraph.Sourcebox, error) {
	getSourcebox := func(url string) (*sourcegraph.Sourcebox, error) {
		resp, err := httpClient.Get(url)
		if err != nil {
			return nil, err
		}
		defer resp.Body.Close()
		if resp.StatusCode != http.StatusOK {
			return nil, fmt.Errorf("http response status %d from %s", resp.StatusCode, url)
		}
		var sb *sourcegraph.Sourcebox
		return sb, json.NewDecoder(resp.Body).Decode(&sb)
	}

	sbs := make([]*sourcegraph.Sourcebox, len(urls))
	errc := make(chan error)
	for i, url := range urls {
		go func(i int, url string) {
			sb, err := getSourcebox(url)
			sbs[i] = sb
			errc <- err
		}(i, url)
	}

	var firstErr error
	timedOut := time.After(deadline.Sub(time.Now()))
	okCount := 0
	for range urls {
		select {
		case err := <-errc:
			if err == nil {
				okCount++
			} else if err != nil && firstErr == nil {
				firstErr = err
			}
		case <-timedOut:
			if firstErr == nil {
				firstErr = errQueryTimeout
			}
			break
		}
	}

	// non-nil sbs
	okSBs := make([]*sourcegraph.Sourcebox, 0, okCount)
	for _, sb := range sbs {
		if sb != nil {
			okSBs = append(okSBs, sb)
		}
	}
	return okSBs, firstErr
}

var errQueryTimeout = errors.New("results timeout")

func newCancelableHTTPMemoryCacheTransport() http.RoundTripper {
	// httpcache doesn't support CancelRequest; wrap it. TODO(sqs):
	// submit a patch to httpcache to fix this.
	t := httpcache.NewMemoryCacheTransport()
	t.Transport = http.DefaultTransport
	return &cancelableHTTPCacheTransport{t}
}

type cancelableHTTPCacheTransport struct{ *httpcache.Transport }

func (t *cancelableHTTPCacheTransport) CancelRequest(req *http.Request) {
	t.Transport.Transport.(*http.Transport).CancelRequest(req)
}
<|endoftext|>


<commit_before>// Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

// Package xurls extracts urls from plain text using regular expressions.
package xurls

import (
	"regexp"
	"strings"
	"unicode/utf8"
)

//go:generate go run ./generate/tldsgen
//go:generate go run ./generate/schemesgen
//go:generate go run ./generate/unicodegen

const (
	letter    = `\p{L}`
	mark      = `\p{M}`
	number    = `\p{N}`
	iriChar   = letter + mark + number
	currency  = `\p{Sc}`
	otherSymb = `\p{So}`
	endChar   = iriChar + `/\-_+&~%=#` + currency + otherSymb
	midChar   = endChar + "_*" + otherPuncMinusDoubleQuote
	wellParen = `\([` + midChar + `]*(\([` + midChar + `]*\)[` + midChar + `]*)*\)`
	wellBrack = `\[[` + midChar + `]*(\[[` + midChar + `]*\][` + midChar + `]*)*\]`
	wellBrace = `\{[` + midChar + `]*(\{[` + midChar + `]*\}[` + midChar + `]*)*\}`
	wellAll   = wellParen + `|` + wellBrack + `|` + wellBrace
	pathCont  = `([` + midChar + `]*(` + wellAll + `|[` + endChar + `])+)+`

	iri      = `[` + iriChar + `]([` + iriChar + `\-]*[` + iriChar + `])?`
	domain   = `(` + iri + `\.)+`
	octet    = `(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9][0-9]|[0-9])`
	ipv4Addr = `\b` + octet + `\.` + octet + `\.` + octet + `\.` + octet + `\b`
	ipv6Addr = `([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:[0-9a-fA-F]{0,4}|:[0-9a-fA-F]{1,4})?|(:[0-9a-fA-F]{1,4}){0,2})|(:[0-9a-fA-F]{1,4}){0,3})|(:[0-9a-fA-F]{1,4}){0,4})|:(:[0-9a-fA-F]{1,4}){0,5})((:[0-9a-fA-F]{1,4}){2}|:(25[0-5]|(2[0-4]|1[0-9]|[1-9])?[0-9])(\.(25[0-5]|(2[0-4]|1[0-9]|[1-9])?[0-9])){3})|(([0-9a-fA-F]{1,4}:){1,6}|:):[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){7}:`
	ipAddr   = `(` + ipv4Addr + `|` + ipv6Addr + `)`
	port     = `(:[0-9]*)?`
)

// AnyScheme can be passed to StrictMatchingScheme to match any possibly valid
// scheme, and not just the known ones.
var AnyScheme = `([a-zA-Z][a-zA-Z.\-+]*://|` + anyOf(SchemesNoAuthority...) + `:)`

// SchemesNoAuthority is a sorted list of some well-known url schemes that are
// followed by ":" instead of "://". The list includes both officially
// registered and unofficial schemes.
var SchemesNoAuthority = []string{
	`bitcoin`, // Bitcoin
	`cid`,     // Content-ID
	`file`,    // Files
	`magnet`,  // Torrent magnets
	`mailto`,  // Mail
	`mid`,     // Message-ID
	`sms`,     // SMS
	`tel`,     // Telephone
	`xmpp`,    // XMPP
}

// SchemesUnofficial is a sorted list of some well-known url schemes which
// aren't officially registered just yet. They tend to correspond to software.
//
// Mostly collected from https://en.wikipedia.org/wiki/List_of_URI_schemes#Unofficial_but_common_URI_schemes.
var SchemesUnofficial = []string{
	`jdbc`,       // Java database Connectivity
	`postgres`,   // PostgreSQL (short form)
	`postgresql`, // PostgreSQL
	`slack`,      // Slack
	`zoommtg`,    // Zoom (desktop)
	`zoomus`,     // Zoom (mobile)
}

func anyOf(strs ...string) string {
	var b strings.Builder
	b.WriteByte('(')
	for i, s := range strs {
		if i != 0 {
			b.WriteByte('|')
		}
		b.WriteString(regexp.QuoteMeta(s))
	}
	b.WriteByte(')')
	return b.String()
}

func strictExp() string {
	schemes := `((` + anyOf(Schemes...) + `|` + anyOf(SchemesUnofficial...) + `)://|` + anyOf(SchemesNoAuthority...) + `:)`
	return `(?i)` + schemes + `(?-i)` + pathCont
}

func relaxedExp() string {
	var asciiTLDs, unicodeTLDs []string
	for i, tld := range TLDs {
		if tld[0] >= utf8.RuneSelf {
			asciiTLDs = TLDs[:i:i]
			unicodeTLDs = TLDs[i+1:]
			break
		}
	}
	punycode := `xn--[a-z0-9-]+`

	// Use \b to make sure ASCII TLDs are immediately followed by a word break.
	// We can't do that with unicode TLDs, as they don't see following
	// whitespace as a word break.
	tlds := `(?i)(` + punycode + `|` + anyOf(append(asciiTLDs, PseudoTLDs...)...) + `\b|` + anyOf(unicodeTLDs...) + `)(?-i)`
	site := domain + tlds

	hostName := `(` + site + `|` + ipAddr + `)`
	webURL := hostName + port + `(/|/` + pathCont + `)?`
	email := `[a-zA-Z0-9._%\-+]+@` + site
	return strictExp() + `|` + webURL + `|` + email
}

// Strict produces a regexp that matches any URL with a scheme in either the
// Schemes or SchemesNoAuthority lists.
func Strict() *regexp.Regexp {
	re := regexp.MustCompile(strictExp())
	re.Longest()
	return re
}

// Relaxed produces a regexp that matches any URL matched by Strict, plus any
// URL with no scheme or email address.
func Relaxed() *regexp.Regexp {
	re := regexp.MustCompile(relaxedExp())
	re.Longest()
	return re
}

// StrictMatchingScheme produces a regexp similar to Strict, but requiring that
// the scheme match the given regular expression. See AnyScheme too.
func StrictMatchingScheme(exp string) (*regexp.Regexp, error) {
	strictMatching := `(?i)(` + exp + `)(?-i)` + pathCont
	re, err := regexp.Compile(strictMatching)
	if err != nil {
		return nil, err
	}
	re.Longest()
	return re, nil
}
<commit_msg>use indexes correctly when splitting the list of TLDs<commit_after>// Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

// Package xurls extracts urls from plain text using regular expressions.
package xurls

import (
	"regexp"
	"strings"
	"unicode/utf8"
)

//go:generate go run ./generate/tldsgen
//go:generate go run ./generate/schemesgen
//go:generate go run ./generate/unicodegen

const (
	letter    = `\p{L}`
	mark      = `\p{M}`
	number    = `\p{N}`
	iriChar   = letter + mark + number
	currency  = `\p{Sc}`
	otherSymb = `\p{So}`
	endChar   = iriChar + `/\-_+&~%=#` + currency + otherSymb
	midChar   = endChar + "_*" + otherPuncMinusDoubleQuote
	wellParen = `\([` + midChar + `]*(\([` + midChar + `]*\)[` + midChar + `]*)*\)`
	wellBrack = `\[[` + midChar + `]*(\[[` + midChar + `]*\][` + midChar + `]*)*\]`
	wellBrace = `\{[` + midChar + `]*(\{[` + midChar + `]*\}[` + midChar + `]*)*\}`
	wellAll   = wellParen + `|` + wellBrack + `|` + wellBrace
	pathCont  = `([` + midChar + `]*(` + wellAll + `|[` + endChar + `])+)+`

	iri      = `[` + iriChar + `]([` + iriChar + `\-]*[` + iriChar + `])?`
	domain   = `(` + iri + `\.)+`
	octet    = `(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9][0-9]|[0-9])`
	ipv4Addr = `\b` + octet + `\.` + octet + `\.` + octet + `\.` + octet + `\b`
	ipv6Addr = `([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:[0-9a-fA-F]{0,4}|:[0-9a-fA-F]{1,4})?|(:[0-9a-fA-F]{1,4}){0,2})|(:[0-9a-fA-F]{1,4}){0,3})|(:[0-9a-fA-F]{1,4}){0,4})|:(:[0-9a-fA-F]{1,4}){0,5})((:[0-9a-fA-F]{1,4}){2}|:(25[0-5]|(2[0-4]|1[0-9]|[1-9])?[0-9])(\.(25[0-5]|(2[0-4]|1[0-9]|[1-9])?[0-9])){3})|(([0-9a-fA-F]{1,4}:){1,6}|:):[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){7}:`
	ipAddr   = `(` + ipv4Addr + `|` + ipv6Addr + `)`
	port     = `(:[0-9]*)?`
)

// AnyScheme can be passed to StrictMatchingScheme to match any possibly valid
// scheme, and not just the known ones.
var AnyScheme = `([a-zA-Z][a-zA-Z.\-+]*://|` + anyOf(SchemesNoAuthority...) + `:)`

// SchemesNoAuthority is a sorted list of some well-known url schemes that are
// followed by ":" instead of "://". The list includes both officially
// registered and unofficial schemes.
var SchemesNoAuthority = []string{
	`bitcoin`, // Bitcoin
	`cid`,     // Content-ID
	`file`,    // Files
	`magnet`,  // Torrent magnets
	`mailto`,  // Mail
	`mid`,     // Message-ID
	`sms`,     // SMS
	`tel`,     // Telephone
	`xmpp`,    // XMPP
}

// SchemesUnofficial is a sorted list of some well-known url schemes which
// aren't officially registered just yet. They tend to correspond to software.
//
// Mostly collected from https://en.wikipedia.org/wiki/List_of_URI_schemes#Unofficial_but_common_URI_schemes.
var SchemesUnofficial = []string{
	`jdbc`,       // Java database Connectivity
	`postgres`,   // PostgreSQL (short form)
	`postgresql`, // PostgreSQL
	`slack`,      // Slack
	`zoommtg`,    // Zoom (desktop)
	`zoomus`,     // Zoom (mobile)
}

func anyOf(strs ...string) string {
	var b strings.Builder
	b.WriteByte('(')
	for i, s := range strs {
		if i != 0 {
			b.WriteByte('|')
		}
		b.WriteString(regexp.QuoteMeta(s))
	}
	b.WriteByte(')')
	return b.String()
}

func strictExp() string {
	schemes := `((` + anyOf(Schemes...) + `|` + anyOf(SchemesUnofficial...) + `)://|` + anyOf(SchemesNoAuthority...) + `:)`
	return `(?i)` + schemes + `(?-i)` + pathCont
}

func relaxedExp() string {
	var asciiTLDs, unicodeTLDs []string
	for i, tld := range TLDs {
		if tld[0] >= utf8.RuneSelf {
			asciiTLDs = TLDs[:i:i]
			unicodeTLDs = TLDs[i:]
			break
		}
	}
	punycode := `xn--[a-z0-9-]+`

	// Use \b to make sure ASCII TLDs are immediately followed by a word break.
	// We can't do that with unicode TLDs, as they don't see following
	// whitespace as a word break.
	tlds := `(?i)(` + punycode + `|` + anyOf(append(asciiTLDs, PseudoTLDs...)...) + `\b|` + anyOf(unicodeTLDs...) + `)(?-i)`
	site := domain + tlds

	hostName := `(` + site + `|` + ipAddr + `)`
	webURL := hostName + port + `(/|/` + pathCont + `)?`
	email := `[a-zA-Z0-9._%\-+]+@` + site
	return strictExp() + `|` + webURL + `|` + email
}

// Strict produces a regexp that matches any URL with a scheme in either the
// Schemes or SchemesNoAuthority lists.
func Strict() *regexp.Regexp {
	re := regexp.MustCompile(strictExp())
	re.Longest()
	return re
}

// Relaxed produces a regexp that matches any URL matched by Strict, plus any
// URL with no scheme or email address.
func Relaxed() *regexp.Regexp {
	re := regexp.MustCompile(relaxedExp())
	re.Longest()
	return re
}

// StrictMatchingScheme produces a regexp similar to Strict, but requiring that
// the scheme match the given regular expression. See AnyScheme too.
func StrictMatchingScheme(exp string) (*regexp.Regexp, error) {
	strictMatching := `(?i)(` + exp + `)(?-i)` + pathCont
	re, err := regexp.Compile(strictMatching)
	if err != nil {
		return nil, err
	}
	re.Longest()
	return re, nil
}
<|endoftext|>


<commit_before>// call somatic variants with multiple tumor samples associated with 1 normal. This uses
// the genotype likelihood approach from bcbio and speedseq but allows for multiple tumor
// samples. Anything that meets the criteria will have a "PASS" or "." FILTER and a list
// of tumor samples with the somatic variant in INFO["SOMATIC"]
// Copied from bcbio and speedseq.
package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/brentp/vcfgo"
	"github.com/brentp/xopen"
)

func getTumorLOD(gls []float64) float64 {
	lod := -1.0
	if len(gls) < 3 {
		return lod
	}
	for _, gl := range gls[1:] {
		t := gl - gls[0]
		if t > lod {
			lod = t
		}
	}
	return lod
}

func getNormalLOD(gls []float64, thresh float64) float64 {
	lod := 1e7
	if len(gls) < 3 {
		return thresh
	}
	for _, gl := range gls[1:] {
		t := gls[0] - gl
		if t < lod {
			lod = t
		}
	}
	return lod
}

func somaticLOD(normalGLs []float64, tumorGLs []float64, thresh float64) bool {
	normalLOD := getNormalLOD(normalGLs, thresh)
	tumorLOD := getTumorLOD(tumorGLs)
	return tumorLOD >= thresh && normalLOD >= thresh
}

func getFreq(refCount int, altCounts []int) float64 {
	ac := 0
	for _, a := range altCounts {
		ac += a
	}
	return float64(ac) / float64(refCount+ac)
}

func somaticFreqs(normal *vcfgo.SampleGenotype, tumor *vcfgo.SampleGenotype, freqRatio float64) bool {
	var nFreq, tFreq float64
	nrd, err := normal.RefDepth()
	if err != nil {
		nFreq = 0.0
	} else {
		nads, err := normal.AltDepths()
		if err != nil {
			nFreq = 0.0
		} else {
			nFreq = getFreq(nrd, nads)
		}
	}
	trd, err := tumor.RefDepth()
	if err != nil {
		tFreq = 0.0
	} else {
		tads, err := tumor.AltDepths()
		if err != nil {
			tFreq = 0.0
		} else {
			tFreq = getFreq(trd, tads)
		}
	}
	return nFreq <= 0.001 || nFreq <= tFreq/freqRatio

}

func Somatics(v *vcfgo.Variant, normalIdx int, thresh float64, freqRatio float64, skipMissing bool) []string {

	normal := v.Samples[normalIdx]
	somatics := make([]string, 0)
	// if we don't have GLs for normal, dont call a somatic.
	if skipMissing && len(normal.GL) == 0 {
		return somatics
	}
	for i, tumor := range v.Samples {
		if i == normalIdx {
			continue
		}
		if !somaticFreqs(normal, tumor, freqRatio) {
			continue
		}
		if somaticLOD(normal.GL, tumor.GL, thresh) {
			somatics = append(somatics, v.Header.SampleNames[i])
		}
	}
	return somatics
}

func check(e error) {
	if e != nil {
		log.Fatal(e)
	}
}

func sampleIndex(samples []string, sample string) int {
	for p, s := range samples {
		if s == sample {
			return p
		}
	}
	return -1
}

func main() {
	normalStr := flag.String("normal", "", "sample name of the normal sample. Default is the first sample")
	threshold := flag.Float64("threshold", 3.5, "threshold for difference in GLs to call a somatic variant")
	freqRatio := flag.Float64("freq-ratio", 2.7, "frequency in the tumor must be at least this many times the frequency in the normal")
	onlySomatic := flag.Bool("only-somatic", false, "print only the PASSing somatic variants (default is to set a flag and print all variants")
	skipMissing := flag.Bool("skip-missing-normals", false, "skip variants with missing normal. default is flag these as somatic.")
	flag.Parse()
	vcfs := flag.Args()
	if len(vcfs) != 1 {
		fmt.Printf("-----------------------------------------------------------------------------------\n")
		fmt.Printf("---------------- call somatic variants present in any tumor sample ----------------\n")
		fmt.Printf("----- uses the method from bcbio and speedseq but for multiple tumor samples ------\n")
		fmt.Printf("-----------------------------------------------------------------------------------\n")
		fmt.Printf("%s normal-and-tumors.vcf.gz\n", os.Args[0])
		flag.PrintDefaults()
		os.Exit(1)
	}

	/*
		f, err := os.Create("cpu.pprof")
		if err != nil {
			panic(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	*/

	fhr, err := xopen.Ropen(vcfs[0])
	check(err)

	rdr, err := vcfgo.NewReader(fhr, false)
	check(err)

	normal := 0
	if *normalStr != "" {
		normal = sampleIndex(rdr.Header.SampleNames, *normalStr)
		if normal == -1 {
			log.Fatalf("sample %s: not found in vcf (%v)\n", *normalStr, rdr.Header.SampleNames)
		}
	}

	fhw, err := xopen.Wopen("-")
	check(err)

	hdr := rdr.Header
	hdr.Filters["NOT_SOMATIC"] = "not somatic between normal and any tumor"

	hdr.Infos["SOMATIC"] = &vcfgo.Info{Id: "SOMATIC", Number: "1", Type: "String", Description: "Tumor samples with somatic event"}

	wtr, err := vcfgo.NewWriter(fhw, hdr)
	check(err)

	log.Printf("using %s as the normal sample\n", hdr.SampleNames[normal])

	j := 0
	for v := rdr.Read(); v != nil; v = rdr.Read() {
		j++
		if *onlySomatic && !(v.Filter == "." || v.Filter == "PASS") {
			continue
		}
		somatics := Somatics(v, normal, *threshold, *freqRatio, *skipMissing)
		if len(somatics) > 0 {
			v.Info.Add("SOMATIC", strings.Join(somatics, "|"))
		} else {
			if *onlySomatic {
				continue
			}
			if v.Filter == "." || v.Filter == "PASS" {
				v.Filter = ""
			}
			if v.Filter != "" {
				v.Filter += ";"
			}
			v.Filter += "NOT_SOMATIC"
		}
		wtr.WriteVariant(v)
	}
	log.Println("VCF warnings:", rdr.Error())
	log.Printf("evaluated %d variants\n", j)
	fhw.Close()
}
<commit_msg>add pairwise<commit_after>// call somatic variants with multiple tumor samples associated with 1 normal. This uses
// the genotype likelihood approach from bcbio and speedseq but allows for multiple tumor
// samples. Anything that meets the criteria will have a "PASS" or "." FILTER and a list
// of tumor samples with the somatic variant in INFO["SOMATIC"]
// Copied from bcbio and speedseq.
package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/brentp/vcfgo"
	"github.com/brentp/xopen"
)

func getTumorLOD(gls []float64) float64 {
	lod := -1.0
	if len(gls) < 3 {
		return lod
	}
	for _, gl := range gls[1:] {
		t := gl - gls[0]
		if t > lod {
			lod = t
		}
	}
	return lod
}

func getNormalLOD(gls []float64, thresh float64) float64 {
	lod := 1e7
	if len(gls) < 3 {
		return thresh
	}
	for _, gl := range gls[1:] {
		t := gls[0] - gl
		if t < lod {
			lod = t
		}
	}
	return lod
}

func somaticLOD(normalGLs []float64, tumorGLs []float64, thresh float64) bool {
	normalLOD := getNormalLOD(normalGLs, thresh)
	tumorLOD := getTumorLOD(tumorGLs)
	return tumorLOD >= thresh && normalLOD >= thresh
}

func getFreq(refCount int, altCounts []int) float64 {
	ac := 0
	for _, a := range altCounts {
		ac += a
	}
	return float64(ac) / float64(refCount+ac)
}

func somaticFreqs(normal *vcfgo.SampleGenotype, tumor *vcfgo.SampleGenotype, freqRatio float64) bool {
	var nFreq, tFreq float64
	nrd, err := normal.RefDepth()
	if err != nil {
		nFreq = 0.0
	} else {
		nads, err := normal.AltDepths()
		if err != nil {
			nFreq = 0.0
		} else {
			nFreq = getFreq(nrd, nads)
		}
	}
	trd, err := tumor.RefDepth()
	if err != nil {
		tFreq = 0.0
	} else {
		tads, err := tumor.AltDepths()
		if err != nil {
			tFreq = 0.0
		} else {
			tFreq = getFreq(trd, tads)
		}
	}
	return nFreq <= 0.001 || nFreq <= tFreq/freqRatio

}

func Somatics(v *vcfgo.Variant, normalIdx int, thresh float64, freqRatio float64, skipMissing bool) []string {

	normal := v.Samples[normalIdx]
	somatics := make([]string, 0)
	// if we don't have GLs for normal, dont call a somatic.
	if skipMissing && len(normal.GL) == 0 {
		return somatics
	}
	for i, tumor := range v.Samples {
		if i == normalIdx {
			continue
		}
		if !somaticFreqs(normal, tumor, freqRatio) {
			continue
		}
		if somaticLOD(normal.GL, tumor.GL, thresh) {
			somatics = append(somatics, v.Header.SampleNames[i])
		}
	}
	return somatics
}

func Contains(li []string, item string) bool {
	for _, i := range li {
		if item == i {
			return true
		}
	}
	return false
}

func PairWise(v *vcfgo.Variant, thresh float64, freqRatio float64, skipMissing bool) []string {
	pairs := make([]string, 0)
	for i := range v.Samples {
		samp := v.Header.SampleNames[i]
		soms := Somatics(v, i, thresh, freqRatio, skipMissing)
		for _, som := range soms {
			pair := samp + ".." + som
			// actually, shouldn't need this check because the first has to be reference and the other not.
			if !(Contains(pairs, som+".."+samp) || Contains(pairs, pair)) {
				pairs = append(pairs, pair)
			}
		}
	}
	return pairs
}

func check(e error) {
	if e != nil {
		log.Fatal(e)
	}
}

func sampleIndex(samples []string, sample string) int {
	for p, s := range samples {
		if s == sample {
			return p
		}
	}
	return -1
}

func main() {
	normalStr := flag.String("normal", "", "sample name of the normal sample. Default is the first sample")
	threshold := flag.Float64("threshold", 3.5, "threshold for difference in GLs to call a somatic variant")
	freqRatio := flag.Float64("freq-ratio", 2.7, "frequency in the tumor must be at least this many times the frequency in the normal")
	onlySomatic := flag.Bool("only-somatic", false, "print only the PASSing somatic variants (default is to set a flag and print all variants")
	skipMissing := flag.Bool("skip-missing-normals", false, "skip variants with missing normal. default is flag these as somatic.")
	pairwise := flag.Bool("pairwise", false, "compare each sample to all others")
	flag.Parse()
	vcfs := flag.Args()
	if len(vcfs) != 1 {
		fmt.Printf("-----------------------------------------------------------------------------------\n")
		fmt.Printf("---------------- call somatic variants present in any tumor sample ----------------\n")
		fmt.Printf("----- uses the method from bcbio and speedseq but for multiple tumor samples ------\n")
		fmt.Printf("-----------------------------------------------------------------------------------\n")
		fmt.Printf("%s normal-and-tumors.vcf.gz\n", os.Args[0])
		flag.PrintDefaults()
		os.Exit(1)
	}

	/*
		f, err := os.Create("cpu.pprof")
		if err != nil {
			panic(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	*/

	fhr, err := xopen.Ropen(vcfs[0])
	check(err)

	rdr, err := vcfgo.NewReader(fhr, false)
	check(err)

	normal := 0
	if *normalStr != "" {
		normal = sampleIndex(rdr.Header.SampleNames, *normalStr)
		if normal == -1 {
			log.Fatalf("sample %s: not found in vcf (%v)\n", *normalStr, rdr.Header.SampleNames)
		}
	}

	fhw, err := xopen.Wopen("-")
	check(err)

	hdr := rdr.Header
	hdr.Filters["NOT_SOMATIC"] = "not somatic between normal and any tumor"

	if *pairwise {
		hdr.Infos["SOMATICPAIRS"] = &vcfgo.Info{Id: "SOMATICPAIRS", Number: "1", Type: "String", Description: "pairwise indications of somatic events"}
	}
	hdr.Infos["SOMATIC"] = &vcfgo.Info{Id: "SOMATIC", Number: "1", Type: "String", Description: "Tumor samples with somatic event"}

	wtr, err := vcfgo.NewWriter(fhw, hdr)
	check(err)

	log.Printf("using %s as the normal sample\n", hdr.SampleNames[normal])

	j := 0
	for v := rdr.Read(); v != nil; v = rdr.Read() {
		j++
		if *onlySomatic && !(v.Filter == "." || v.Filter == "PASS") {
			continue
		}
		somatics := Somatics(v, normal, *threshold, *freqRatio, *skipMissing)
		if len(somatics) > 0 {
			v.Info.Add("SOMATIC", strings.Join(somatics, "|"))
		} else {
			if *onlySomatic {
				continue
			}
			if v.Filter == "." || v.Filter == "PASS" {
				v.Filter = ""
			}
			if v.Filter != "" {
				v.Filter += ";"
			}
			v.Filter += "NOT_SOMATIC"
		}
		if *pairwise {
			pairs := PairWise(v, *threshold, *freqRatio, *skipMissing)
			if len(pairs) > 0 {
				v.Info.Add("SOMATICPAIRS", strings.Join(pairs, "|"))
			}

		}

		wtr.WriteVariant(v)
	}
	log.Println("VCF warnings:", rdr.Error())
	log.Printf("evaluated %d variants\n", j)
	fhw.Close()
}
<|endoftext|>


<commit_before>package nsqd

import (
	"fmt"
	"net"
	"time"

	"github.com/bitly/go-nsq"
)

// lookupPeer is a low-level type for connecting/reading/writing to nsqlookupd
//
// A lookupPeer instance is designed to connect lazily to nsqlookupd and reconnect
// gracefully (i.e. it is all handled by the library).  Clients can simply use the
// Command interface to perform a round-trip.
type lookupPeer struct {
	l               logger
	addr            string
	conn            net.Conn
	state           int32
	connectCallback func(*lookupPeer)
	Info            peerInfo
}

// peerInfo contains metadata for a lookupPeer instance (and is JSON marshalable)
type peerInfo struct {
	TCPPort          int    `json:"tcp_port"`
	HTTPPort         int    `json:"http_port"`
	Version          string `json:"version"`
	BroadcastAddress string `json:"broadcast_address"`
}

// newLookupPeer creates a new lookupPeer instance connecting to the supplied address.
//
// The supplied connectCallback will be called *every* time the instance connects.
func newLookupPeer(addr string, l logger, connectCallback func(*lookupPeer)) *lookupPeer {
	return &lookupPeer{
		l:               l,
		addr:            addr,
		state:           stateDisconnected,
		connectCallback: connectCallback,
	}
}

// Connect will Dial the specified address, with timeouts
func (lp *lookupPeer) Connect() error {
	lp.l.Output(2, fmt.Sprintf("LOOKUP connecting to %s", lp.addr))
	conn, err := net.DialTimeout("tcp", lp.addr, time.Second)
	if err != nil {
		return err
	}
	lp.conn = conn
	return nil
}

// String returns the specified address
func (lp *lookupPeer) String() string {
	return lp.addr
}

// Read implements the io.Reader interface, adding deadlines
func (lp *lookupPeer) Read(data []byte) (int, error) {
	lp.conn.SetReadDeadline(time.Now().Add(time.Second))
	return lp.conn.Read(data)
}

// Write implements the io.Writer interface, adding deadlines
func (lp *lookupPeer) Write(data []byte) (int, error) {
	lp.conn.SetWriteDeadline(time.Now().Add(time.Second))
	return lp.conn.Write(data)
}

// Close implements the io.Closer interface
func (lp *lookupPeer) Close() error {
	lp.state = stateDisconnected
	return lp.conn.Close()
}

// Command performs a round-trip for the specified Command.
//
// It will lazily connect to nsqlookupd and gracefully handle
// reconnecting in the event of a failure.
//
// It returns the response from nsqlookupd as []byte
func (lp *lookupPeer) Command(cmd *nsq.Command) ([]byte, error) {
	initialState := lp.state
	if lp.state != stateConnected {
		err := lp.Connect()
		if err != nil {
			return nil, err
		}
		lp.state = stateConnected
		lp.Write(nsq.MagicV1)
		if initialState == stateDisconnected {
			lp.connectCallback(lp)
		}
	}
	if cmd == nil {
		return nil, nil
	}
	_, err := cmd.WriteTo(lp)
	if err != nil {
		lp.Close()
		return nil, err
	}
	resp, err := nsq.ReadResponse(lp)
	if err != nil {
		lp.Close()
		return nil, err
	}
	return resp, nil
}
<commit_msg>nsqd: only close lookupPeer connection when non-nil<commit_after>package nsqd

import (
	"fmt"
	"net"
	"time"

	"github.com/bitly/go-nsq"
)

// lookupPeer is a low-level type for connecting/reading/writing to nsqlookupd
//
// A lookupPeer instance is designed to connect lazily to nsqlookupd and reconnect
// gracefully (i.e. it is all handled by the library).  Clients can simply use the
// Command interface to perform a round-trip.
type lookupPeer struct {
	l               logger
	addr            string
	conn            net.Conn
	state           int32
	connectCallback func(*lookupPeer)
	Info            peerInfo
}

// peerInfo contains metadata for a lookupPeer instance (and is JSON marshalable)
type peerInfo struct {
	TCPPort          int    `json:"tcp_port"`
	HTTPPort         int    `json:"http_port"`
	Version          string `json:"version"`
	BroadcastAddress string `json:"broadcast_address"`
}

// newLookupPeer creates a new lookupPeer instance connecting to the supplied address.
//
// The supplied connectCallback will be called *every* time the instance connects.
func newLookupPeer(addr string, l logger, connectCallback func(*lookupPeer)) *lookupPeer {
	return &lookupPeer{
		l:               l,
		addr:            addr,
		state:           stateDisconnected,
		connectCallback: connectCallback,
	}
}

// Connect will Dial the specified address, with timeouts
func (lp *lookupPeer) Connect() error {
	lp.l.Output(2, fmt.Sprintf("LOOKUP connecting to %s", lp.addr))
	conn, err := net.DialTimeout("tcp", lp.addr, time.Second)
	if err != nil {
		return err
	}
	lp.conn = conn
	return nil
}

// String returns the specified address
func (lp *lookupPeer) String() string {
	return lp.addr
}

// Read implements the io.Reader interface, adding deadlines
func (lp *lookupPeer) Read(data []byte) (int, error) {
	lp.conn.SetReadDeadline(time.Now().Add(time.Second))
	return lp.conn.Read(data)
}

// Write implements the io.Writer interface, adding deadlines
func (lp *lookupPeer) Write(data []byte) (int, error) {
	lp.conn.SetWriteDeadline(time.Now().Add(time.Second))
	return lp.conn.Write(data)
}

// Close implements the io.Closer interface
func (lp *lookupPeer) Close() error {
	lp.state = stateDisconnected
	if lp.conn != nil {
		return lp.conn.Close()
	}
	return nil
}

// Command performs a round-trip for the specified Command.
//
// It will lazily connect to nsqlookupd and gracefully handle
// reconnecting in the event of a failure.
//
// It returns the response from nsqlookupd as []byte
func (lp *lookupPeer) Command(cmd *nsq.Command) ([]byte, error) {
	initialState := lp.state
	if lp.state != stateConnected {
		err := lp.Connect()
		if err != nil {
			return nil, err
		}
		lp.state = stateConnected
		lp.Write(nsq.MagicV1)
		if initialState == stateDisconnected {
			lp.connectCallback(lp)
		}
	}
	if cmd == nil {
		return nil, nil
	}
	_, err := cmd.WriteTo(lp)
	if err != nil {
		lp.Close()
		return nil, err
	}
	resp, err := nsq.ReadResponse(lp)
	if err != nil {
		lp.Close()
		return nil, err
	}
	return resp, nil
}
<|endoftext|>


<commit_before>package bridge

import (
	"github.com/42wim/matterbridge/bridge/api"
	"github.com/42wim/matterbridge/bridge/config"
	"github.com/42wim/matterbridge/bridge/discord"
	"github.com/42wim/matterbridge/bridge/gitter"
	"github.com/42wim/matterbridge/bridge/irc"
	"github.com/42wim/matterbridge/bridge/matrix"
	"github.com/42wim/matterbridge/bridge/mattermost"
	"github.com/42wim/matterbridge/bridge/rocketchat"
	"github.com/42wim/matterbridge/bridge/slack"
	"github.com/42wim/matterbridge/bridge/telegram"
	"github.com/42wim/matterbridge/bridge/xmpp"
	log "github.com/Sirupsen/logrus"

	"strings"
)

type Bridger interface {
	Send(msg config.Message) error
	Connect() error
	JoinChannel(channel string) error
	Disconnect() error
}

type Bridge struct {
	Config config.Protocol
	Bridger
	Name     string
	Account  string
	Protocol string
	Channels map[string]config.ChannelInfo
	Joined   map[string]bool
}

func New(cfg *config.Config, bridge *config.Bridge, c chan config.Message) *Bridge {
	b := new(Bridge)
	b.Channels = make(map[string]config.ChannelInfo)
	accInfo := strings.Split(bridge.Account, ".")
	protocol := accInfo[0]
	name := accInfo[1]
	b.Name = name
	b.Protocol = protocol
	b.Account = bridge.Account
	b.Joined = make(map[string]bool)

	// override config from environment
	config.OverrideCfgFromEnv(cfg, protocol, name)
	switch protocol {
	case "mattermost":
		b.Config = cfg.Mattermost[name]
		b.Bridger = bmattermost.New(cfg.Mattermost[name], bridge.Account, c)
	case "irc":
		b.Config = cfg.IRC[name]
		b.Bridger = birc.New(cfg.IRC[name], bridge.Account, c)
	case "gitter":
		b.Config = cfg.Gitter[name]
		b.Bridger = bgitter.New(cfg.Gitter[name], bridge.Account, c)
	case "slack":
		b.Config = cfg.Slack[name]
		b.Bridger = bslack.New(cfg.Slack[name], bridge.Account, c)
	case "xmpp":
		b.Config = cfg.Xmpp[name]
		b.Bridger = bxmpp.New(cfg.Xmpp[name], bridge.Account, c)
	case "discord":
		b.Config = cfg.Discord[name]
		b.Bridger = bdiscord.New(cfg.Discord[name], bridge.Account, c)
	case "telegram":
		b.Config = cfg.Telegram[name]
		b.Bridger = btelegram.New(cfg.Telegram[name], bridge.Account, c)
	case "rocketchat":
		b.Config = cfg.Rocketchat[name]
		b.Bridger = brocketchat.New(cfg.Rocketchat[name], bridge.Account, c)
	case "matrix":
		b.Config = cfg.Matrix[name]
		b.Bridger = bmatrix.New(cfg.Matrix[name], bridge.Account, c)
	case "api":
		b.Config = cfg.Api[name]
		b.Bridger = api.New(cfg.Api[name], bridge.Account, c)
	}
	return b
}

func (b *Bridge) JoinChannels() error {
	err := b.joinChannels(b.Channels, b.Joined)
	if err != nil {
		return err
	}
	return nil
}

func (b *Bridge) joinChannels(channels map[string]config.ChannelInfo, exists map[string]bool) error {
	mychannel := ""
	for ID, channel := range channels {
		if !exists[ID] {
			mychannel = channel.Name
			log.Infof("%s: joining %s (%s)", b.Account, channel.Name, ID)
			if b.Protocol == "irc" && channel.Options.Key != "" {
				log.Debugf("using key %s for channel %s", channel.Options.Key, channel.Name)
				mychannel = mychannel + " " + channel.Options.Key
			}
			err := b.JoinChannel(channel.Name)
			if err != nil {
				return err
			}
			exists[ID] = true
		}
	}
	return nil
}
<commit_msg>Fix JoinChannel argument to use IRC channel key (#172)<commit_after>package bridge

import (
	"github.com/42wim/matterbridge/bridge/api"
	"github.com/42wim/matterbridge/bridge/config"
	"github.com/42wim/matterbridge/bridge/discord"
	"github.com/42wim/matterbridge/bridge/gitter"
	"github.com/42wim/matterbridge/bridge/irc"
	"github.com/42wim/matterbridge/bridge/matrix"
	"github.com/42wim/matterbridge/bridge/mattermost"
	"github.com/42wim/matterbridge/bridge/rocketchat"
	"github.com/42wim/matterbridge/bridge/slack"
	"github.com/42wim/matterbridge/bridge/telegram"
	"github.com/42wim/matterbridge/bridge/xmpp"
	log "github.com/Sirupsen/logrus"

	"strings"
)

type Bridger interface {
	Send(msg config.Message) error
	Connect() error
	JoinChannel(channel string) error
	Disconnect() error
}

type Bridge struct {
	Config config.Protocol
	Bridger
	Name     string
	Account  string
	Protocol string
	Channels map[string]config.ChannelInfo
	Joined   map[string]bool
}

func New(cfg *config.Config, bridge *config.Bridge, c chan config.Message) *Bridge {
	b := new(Bridge)
	b.Channels = make(map[string]config.ChannelInfo)
	accInfo := strings.Split(bridge.Account, ".")
	protocol := accInfo[0]
	name := accInfo[1]
	b.Name = name
	b.Protocol = protocol
	b.Account = bridge.Account
	b.Joined = make(map[string]bool)

	// override config from environment
	config.OverrideCfgFromEnv(cfg, protocol, name)
	switch protocol {
	case "mattermost":
		b.Config = cfg.Mattermost[name]
		b.Bridger = bmattermost.New(cfg.Mattermost[name], bridge.Account, c)
	case "irc":
		b.Config = cfg.IRC[name]
		b.Bridger = birc.New(cfg.IRC[name], bridge.Account, c)
	case "gitter":
		b.Config = cfg.Gitter[name]
		b.Bridger = bgitter.New(cfg.Gitter[name], bridge.Account, c)
	case "slack":
		b.Config = cfg.Slack[name]
		b.Bridger = bslack.New(cfg.Slack[name], bridge.Account, c)
	case "xmpp":
		b.Config = cfg.Xmpp[name]
		b.Bridger = bxmpp.New(cfg.Xmpp[name], bridge.Account, c)
	case "discord":
		b.Config = cfg.Discord[name]
		b.Bridger = bdiscord.New(cfg.Discord[name], bridge.Account, c)
	case "telegram":
		b.Config = cfg.Telegram[name]
		b.Bridger = btelegram.New(cfg.Telegram[name], bridge.Account, c)
	case "rocketchat":
		b.Config = cfg.Rocketchat[name]
		b.Bridger = brocketchat.New(cfg.Rocketchat[name], bridge.Account, c)
	case "matrix":
		b.Config = cfg.Matrix[name]
		b.Bridger = bmatrix.New(cfg.Matrix[name], bridge.Account, c)
	case "api":
		b.Config = cfg.Api[name]
		b.Bridger = api.New(cfg.Api[name], bridge.Account, c)
	}
	return b
}

func (b *Bridge) JoinChannels() error {
	err := b.joinChannels(b.Channels, b.Joined)
	if err != nil {
		return err
	}
	return nil
}

func (b *Bridge) joinChannels(channels map[string]config.ChannelInfo, exists map[string]bool) error {
	mychannel := ""
	for ID, channel := range channels {
		if !exists[ID] {
			mychannel = channel.Name
			log.Infof("%s: joining %s (%s)", b.Account, channel.Name, ID)
			if b.Protocol == "irc" && channel.Options.Key != "" {
				log.Debugf("using key %s for channel %s", channel.Options.Key, channel.Name)
				mychannel = mychannel + " " + channel.Options.Key
			}
			err := b.JoinChannel(mychannel)
			if err != nil {
				return err
			}
			exists[ID] = true
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package bridge

import (
	"errors"
	"log"
	"net"
	"net/url"
	"os"
	"path"
	"strconv"
	"strings"
	"sync"

	dockerapi "github.com/fsouza/go-dockerclient"
)

type Bridge struct {
	sync.Mutex
	registry       RegistryAdapter
	docker         *dockerapi.Client
	services       map[string][]*Service
	deadContainers map[string]*DeadContainer
	config         Config
}

func New(docker *dockerapi.Client, adapterUri string, config Config) (*Bridge, error) {
	uri, err := url.Parse(adapterUri)
	if err != nil {
		return nil, errors.New("bad adapter uri: " + adapterUri)
	}
	factory, found := AdapterFactories.Lookup(uri.Scheme)
	if !found {
		return nil, errors.New("unrecognized adapter: " + adapterUri)
	}

	log.Println("Using", uri.Scheme, "adapter:", uri)
	return &Bridge{
		docker:         docker,
		config:         config,
		registry:       factory.New(uri),
		services:       make(map[string][]*Service),
		deadContainers: make(map[string]*DeadContainer),
	}, nil
}

func (b *Bridge) Ping() error {
	return b.registry.Ping()
}

func (b *Bridge) Add(containerId string) {
	b.Lock()
	defer b.Unlock()
	b.add(containerId, false)
}

func (b *Bridge) Remove(containerId string) {
	b.remove(containerId, true)
}

func (b *Bridge) RemoveOnExit(containerId string) {
	b.remove(containerId, b.config.DeregisterCheck == "always" || b.didExitCleanly(containerId))
}

func (b *Bridge) Refresh() {
	b.Lock()
	defer b.Unlock()

	for containerId, deadContainer := range b.deadContainers {
		deadContainer.TTL -= b.config.RefreshInterval
		if deadContainer.TTL <= 0 {
			delete(b.deadContainers, containerId)
		}
	}

	for containerId, services := range b.services {
		for _, service := range services {
			err := b.registry.Refresh(service)
			if err != nil {
				log.Println("refresh failed:", service.ID, err)
				continue
			}
			log.Println("refreshed:", containerId[:12], service.ID)
		}
	}
}

func (b *Bridge) Sync(quiet bool) {
	b.Lock()
	defer b.Unlock()

	containers, err := b.docker.ListContainers(dockerapi.ListContainersOptions{})
	if err != nil && quiet {
		log.Println("error listing containers, skipping sync")
		return
	} else if err != nil && !quiet {
		log.Fatal(err)
	}

	log.Printf("Syncing services on %d containers", len(containers))

	// NOTE: This assumes reregistering will do the right thing, i.e. nothing.
	// NOTE: This will NOT remove services.
	for _, listing := range containers {
		services := b.services[listing.ID]
		if services == nil {
			b.add(listing.ID, quiet)
		} else {
			for _, service := range services {
				err := b.registry.Register(service)
				if err != nil {
					log.Println("sync register failed:", service, err)
				}
			}
		}
	}
}

func (b *Bridge) add(containerId string, quiet bool) {
	if d := b.deadContainers[containerId]; d != nil {
		b.services[containerId] = d.Services
		delete(b.deadContainers, containerId)
	}

	if b.services[containerId] != nil {
		log.Println("container, ", containerId[:12], ", already exists, ignoring")
		// Alternatively, remove and readd or resubmit.
		return
	}

	container, err := b.docker.InspectContainer(containerId)
	if err != nil {
		log.Println("unable to inspect container:", containerId[:12], err)
		return
	}

	ports := make(map[string]ServicePort)

	// Extract configured host port mappings, relevant when using --net=host
	for port, published := range container.HostConfig.PortBindings {
		ports[string(port)] = servicePort(container, port, published)
	}

	// Extract runtime port mappings, relevant when using --net=bridge
	for port, published := range container.NetworkSettings.Ports {
		ports[string(port)] = servicePort(container, port, published)
	}

	metadata := serviceMetaData(container.Config, "")
	for k, v := range metadata {
		if strings.Contains(k, ":ipv6") {
			port := strings.Split(k, ":")[0]
			porttype := v
			ports[k] = ServicePort{HostPort: port,
				HostIP:            container.NetworkSettings.GlobalIPv6Address,
				ExposedPort:       port,
				ExposedIP:         container.NetworkSettings.GlobalIPv6Address,
				PortType:          porttype,
				ContainerID:       container.ID,
				ContainerHostname: container.Config.Hostname,
				container:         container}
		}
	}

	if len(ports) == 0 && !quiet {
		log.Println("ignored:", container.ID[:12], "no published ports")
		return
	}

	for _, port := range ports {
		if b.config.Internal != true && port.HostPort == "" {
			if !quiet {
				log.Println("ignored:", container.ID[:12], "port", port.ExposedPort, "not published on host")
			}
			continue
		}
		service := b.newService(port, len(ports) > 1)
		if service == nil {
			if !quiet {
				log.Println("ignored:", container.ID[:12], "service on port", port.ExposedPort)
			}
			continue
		}
		err := b.registry.Register(service)
		if err != nil {
			log.Println("register failed:", service, err)
			continue
		}
		b.services[container.ID] = append(b.services[container.ID], service)
		log.Println("added:", container.ID[:12], service.ID)
	}
}

func (b *Bridge) newService(port ServicePort, isgroup bool) *Service {
	ipv6 := false
	if strings.Contains(port.ExposedIP, ":") {
		ipv6 = true
	}
	container := port.container
	defaultName := strings.Split(path.Base(container.Config.Image), ":")[0]

	// not sure about this logic. kind of want to remove it.
	hostname, err := os.Hostname()
	if err != nil {
		hostname = port.HostIP
	} else {
		if port.HostIP == "0.0.0.0" {
			ip, err := net.ResolveIPAddr("ip", hostname)
			if err == nil {
				port.HostIP = ip.String()
			}
		}
	}

	if b.config.HostIp != "" {
		port.HostIP = b.config.HostIp
	}

	metadata := serviceMetaData(container.Config, port.ExposedPort)

	ignore := mapDefault(metadata, "ignore", "")
	if ignore != "" {
		return nil
	}

	service := new(Service)
	service.Origin = port
	service.ID = hostname + ":" + container.Name[1:] + ":" + port.ExposedPort
	if ipv6 {
		service.ID = service.ID + ":ipv6"
	}
	service.Name = mapDefault(metadata, "name", defaultName)
	if isgroup {
		service.Name += "-" + port.ExposedPort
	}
	var p int
	if b.config.Internal == true {
		service.IP = port.ExposedIP
		p, _ = strconv.Atoi(port.ExposedPort)
	} else {
		service.IP = port.HostIP
		p, _ = strconv.Atoi(port.HostPort)
	}
	service.Port = p

	if port.PortType == "udp" {
		service.Tags = combineTags(
			mapDefault(metadata, "tags", ""), b.config.ForceTags, "udp")
		service.ID = service.ID + ":udp"
	} else {
		service.Tags = combineTags(
			mapDefault(metadata, "tags", ""), b.config.ForceTags)
	}

	if ipv6 {
		// NAME_IPV6 overrides
		if mapDefault(metadata, "name_ipv6", defaultName) != defaultName {
			service.Name = mapDefault(metadata, "name_ipv6", defaultName)
		}
		delete(metadata, string(port.ExposedPort)+":ipv6")
	} else {
		// NAME_IPV4 overrides
		if mapDefault(metadata, "name_ipv4", defaultName) != defaultName {
			service.Name = mapDefault(metadata, "name_ipv4", defaultName)
		}
	}

	id := mapDefault(metadata, "id", "")
	if id != "" {
		if ipv6 {
			service.ID = id + ":ipv6"
		} else {
			service.ID = id
		}
	}

	delete(metadata, "id")
	delete(metadata, "tags")
	delete(metadata, "name")
	delete(metadata, "name_ipv6")
	delete(metadata, "name_ipv4")
	service.Attrs = metadata
	service.TTL = b.config.RefreshTtl

	return service
}

func (b *Bridge) remove(containerId string, deregister bool) {
	b.Lock()
	defer b.Unlock()

	if deregister {
		deregisterAll := func(services []*Service) {
			for _, service := range services {
				err := b.registry.Deregister(service)
				if err != nil {
					log.Println("deregister failed:", service.ID, err)
					continue
				}
				log.Println("removed:", containerId[:12], service.ID)
			}
		}
		deregisterAll(b.services[containerId])
		if d := b.deadContainers[containerId]; d != nil {
			deregisterAll(d.Services)
			delete(b.deadContainers, containerId)
		}
	} else if b.config.RefreshTtl != 0 && b.services[containerId] != nil {
		// need to stop the refreshing, but can't delete it yet
		b.deadContainers[containerId] = &DeadContainer{b.config.RefreshTtl, b.services[containerId]}
	}
	delete(b.services, containerId)
}

func (b *Bridge) didExitCleanly(containerId string) bool {
	container, err := b.docker.InspectContainer(containerId)
	if _, ok := err.(*dockerapi.NoSuchContainer); ok {
		// the container has already been removed from Docker
		// e.g. probabably run with "--rm" to remove immediately
		// so its exit code is not accessible
		log.Printf("registrator: container %v was removed, could not fetch exit code", containerId[:12])
		return true
	} else if err != nil {
		log.Printf("registrator: error fetching status for container %v on \"die\" event: %v\n", containerId[:12], err)
		return false
	}
	return !container.State.Running && container.State.ExitCode == 0
}
<commit_msg>Remove exposed port from servicename - revert to old behaviour<commit_after>package bridge

import (
	"errors"
	"log"
	"net"
	"net/url"
	"os"
	"path"
	"strconv"
	"strings"
	"sync"

	dockerapi "github.com/fsouza/go-dockerclient"
)

type Bridge struct {
	sync.Mutex
	registry       RegistryAdapter
	docker         *dockerapi.Client
	services       map[string][]*Service
	deadContainers map[string]*DeadContainer
	config         Config
}

func New(docker *dockerapi.Client, adapterUri string, config Config) (*Bridge, error) {
	uri, err := url.Parse(adapterUri)
	if err != nil {
		return nil, errors.New("bad adapter uri: " + adapterUri)
	}
	factory, found := AdapterFactories.Lookup(uri.Scheme)
	if !found {
		return nil, errors.New("unrecognized adapter: " + adapterUri)
	}

	log.Println("Using", uri.Scheme, "adapter:", uri)
	return &Bridge{
		docker:         docker,
		config:         config,
		registry:       factory.New(uri),
		services:       make(map[string][]*Service),
		deadContainers: make(map[string]*DeadContainer),
	}, nil
}

func (b *Bridge) Ping() error {
	return b.registry.Ping()
}

func (b *Bridge) Add(containerId string) {
	b.Lock()
	defer b.Unlock()
	b.add(containerId, false)
}

func (b *Bridge) Remove(containerId string) {
	b.remove(containerId, true)
}

func (b *Bridge) RemoveOnExit(containerId string) {
	b.remove(containerId, b.config.DeregisterCheck == "always" || b.didExitCleanly(containerId))
}

func (b *Bridge) Refresh() {
	b.Lock()
	defer b.Unlock()

	for containerId, deadContainer := range b.deadContainers {
		deadContainer.TTL -= b.config.RefreshInterval
		if deadContainer.TTL <= 0 {
			delete(b.deadContainers, containerId)
		}
	}

	for containerId, services := range b.services {
		for _, service := range services {
			err := b.registry.Refresh(service)
			if err != nil {
				log.Println("refresh failed:", service.ID, err)
				continue
			}
			log.Println("refreshed:", containerId[:12], service.ID)
		}
	}
}

func (b *Bridge) Sync(quiet bool) {
	b.Lock()
	defer b.Unlock()

	containers, err := b.docker.ListContainers(dockerapi.ListContainersOptions{})
	if err != nil && quiet {
		log.Println("error listing containers, skipping sync")
		return
	} else if err != nil && !quiet {
		log.Fatal(err)
	}

	log.Printf("Syncing services on %d containers", len(containers))

	// NOTE: This assumes reregistering will do the right thing, i.e. nothing.
	// NOTE: This will NOT remove services.
	for _, listing := range containers {
		services := b.services[listing.ID]
		if services == nil {
			b.add(listing.ID, quiet)
		} else {
			for _, service := range services {
				err := b.registry.Register(service)
				if err != nil {
					log.Println("sync register failed:", service, err)
				}
			}
		}
	}
}

func (b *Bridge) add(containerId string, quiet bool) {
	if d := b.deadContainers[containerId]; d != nil {
		b.services[containerId] = d.Services
		delete(b.deadContainers, containerId)
	}

	if b.services[containerId] != nil {
		log.Println("container, ", containerId[:12], ", already exists, ignoring")
		// Alternatively, remove and readd or resubmit.
		return
	}

	container, err := b.docker.InspectContainer(containerId)
	if err != nil {
		log.Println("unable to inspect container:", containerId[:12], err)
		return
	}

	ports := make(map[string]ServicePort)

	// Extract configured host port mappings, relevant when using --net=host
	for port, published := range container.HostConfig.PortBindings {
		ports[string(port)] = servicePort(container, port, published)
	}

	// Extract runtime port mappings, relevant when using --net=bridge
	for port, published := range container.NetworkSettings.Ports {
		ports[string(port)] = servicePort(container, port, published)
	}

	metadata := serviceMetaData(container.Config, "")
	for k, v := range metadata {
		if strings.Contains(k, ":ipv6") {
			port := strings.Split(k, ":")[0]
			porttype := v
			ports[k] = ServicePort{HostPort: port,
				HostIP:            container.NetworkSettings.GlobalIPv6Address,
				ExposedPort:       port,
				ExposedIP:         container.NetworkSettings.GlobalIPv6Address,
				PortType:          porttype,
				ContainerID:       container.ID,
				ContainerHostname: container.Config.Hostname,
				container:         container}
		}
	}

	if len(ports) == 0 && !quiet {
		log.Println("ignored:", container.ID[:12], "no published ports")
		return
	}

	for _, port := range ports {
		if b.config.Internal != true && port.HostPort == "" {
			if !quiet {
				log.Println("ignored:", container.ID[:12], "port", port.ExposedPort, "not published on host")
			}
			continue
		}
		service := b.newService(port, len(ports) > 1)
		if service == nil {
			if !quiet {
				log.Println("ignored:", container.ID[:12], "service on port", port.ExposedPort)
			}
			continue
		}
		err := b.registry.Register(service)
		if err != nil {
			log.Println("register failed:", service, err)
			continue
		}
		b.services[container.ID] = append(b.services[container.ID], service)
		log.Println("added:", container.ID[:12], service.ID)
	}
}

func (b *Bridge) newService(port ServicePort, isgroup bool) *Service {
	ipv6 := false
	if strings.Contains(port.ExposedIP, ":") {
		ipv6 = true
	}
	container := port.container
	defaultName := strings.Split(path.Base(container.Config.Image), ":")[0]

	// not sure about this logic. kind of want to remove it.
	hostname, err := os.Hostname()
	if err != nil {
		hostname = port.HostIP
	} else {
		if port.HostIP == "0.0.0.0" {
			ip, err := net.ResolveIPAddr("ip", hostname)
			if err == nil {
				port.HostIP = ip.String()
			}
		}
	}

	if b.config.HostIp != "" {
		port.HostIP = b.config.HostIp
	}

	metadata := serviceMetaData(container.Config, port.ExposedPort)

	ignore := mapDefault(metadata, "ignore", "")
	if ignore != "" {
		return nil
	}

	service := new(Service)
	service.Origin = port
	service.ID = hostname + ":" + container.Name[1:] + ":" + port.ExposedPort
	if ipv6 {
		service.ID = service.ID + ":ipv6"
	}
	service.Name = mapDefault(metadata, "name", defaultName)
	/*
		if isgroup {
			service.Name += "-" + port.ExposedPort
		}
	*/
	var p int
	if b.config.Internal == true {
		service.IP = port.ExposedIP
		p, _ = strconv.Atoi(port.ExposedPort)
	} else {
		service.IP = port.HostIP
		p, _ = strconv.Atoi(port.HostPort)
	}
	service.Port = p

	if port.PortType == "udp" {
		service.Tags = combineTags(
			mapDefault(metadata, "tags", ""), b.config.ForceTags, "udp")
		service.ID = service.ID + ":udp"
	} else {
		service.Tags = combineTags(
			mapDefault(metadata, "tags", ""), b.config.ForceTags)
	}

	if ipv6 {
		// NAME_IPV6 overrides
		if mapDefault(metadata, "name_ipv6", defaultName) != defaultName {
			service.Name = mapDefault(metadata, "name_ipv6", defaultName)
		}
		delete(metadata, string(port.ExposedPort)+":ipv6")
	} else {
		// NAME_IPV4 overrides
		if mapDefault(metadata, "name_ipv4", defaultName) != defaultName {
			service.Name = mapDefault(metadata, "name_ipv4", defaultName)
		}
	}

	id := mapDefault(metadata, "id", "")
	if id != "" {
		if ipv6 {
			service.ID = id + ":ipv6"
		} else {
			service.ID = id
		}
	}

	delete(metadata, "id")
	delete(metadata, "tags")
	delete(metadata, "name")
	delete(metadata, "name_ipv6")
	delete(metadata, "name_ipv4")
	service.Attrs = metadata
	service.TTL = b.config.RefreshTtl

	return service
}

func (b *Bridge) remove(containerId string, deregister bool) {
	b.Lock()
	defer b.Unlock()

	if deregister {
		deregisterAll := func(services []*Service) {
			for _, service := range services {
				err := b.registry.Deregister(service)
				if err != nil {
					log.Println("deregister failed:", service.ID, err)
					continue
				}
				log.Println("removed:", containerId[:12], service.ID)
			}
		}
		deregisterAll(b.services[containerId])
		if d := b.deadContainers[containerId]; d != nil {
			deregisterAll(d.Services)
			delete(b.deadContainers, containerId)
		}
	} else if b.config.RefreshTtl != 0 && b.services[containerId] != nil {
		// need to stop the refreshing, but can't delete it yet
		b.deadContainers[containerId] = &DeadContainer{b.config.RefreshTtl, b.services[containerId]}
	}
	delete(b.services, containerId)
}

func (b *Bridge) didExitCleanly(containerId string) bool {
	container, err := b.docker.InspectContainer(containerId)
	if _, ok := err.(*dockerapi.NoSuchContainer); ok {
		// the container has already been removed from Docker
		// e.g. probabably run with "--rm" to remove immediately
		// so its exit code is not accessible
		log.Printf("registrator: container %v was removed, could not fetch exit code", containerId[:12])
		return true
	} else if err != nil {
		log.Printf("registrator: error fetching status for container %v on \"die\" event: %v\n", containerId[:12], err)
		return false
	}
	return !container.State.Running && container.State.ExitCode == 0
}
<|endoftext|>


<commit_before>package libkbfs

import (
	"fmt"

	libkb "github.com/keybase/client/go/libkb"
)

type MDOpsStandard struct {
	config Config
}

func (md *MDOpsStandard) processMetadata(
	handle *DirHandle, rmds *RootMetadataSigned) error {
	crypto := md.config.Crypto()
	codec := md.config.Codec()
	// verify signature and deserialize root data, if the sig is not blank.
	// a blank sig means this is a brand new MD object, nothing to check
	if rmds.IsInitialized() {
		// decrypt the root data for non-public directories
		if !handle.IsPublic() {
			path := Path{rmds.MD.Id, []PathNode{}}
			k, err := md.config.KeyManager().GetSecretKey(path, &rmds.MD)
			if err != nil {
				return err
			}
			databuf, err := crypto.Decrypt(rmds.MD.SerializedPrivateMetadata, k)
			if err != nil {
				return err
			}
			if err := codec.Decode(databuf, &rmds.MD.data); err != nil {
				return err
			}
		} else if err := codec.Decode(
			rmds.MD.SerializedPrivateMetadata, &rmds.MD.data); err != nil {
			return err
		}

		// Make sure the last writer is really a valid writer
		writer := rmds.MD.data.LastWriter
		if !handle.IsWriter(writer) {
			return &MDMismatchError{
				handle.ToString(md.config),
				fmt.Sprintf("MD (id=%s) was written by a non-writer %s",
					rmds.MD.Id, writer)}
		}

		// TODO:
		// Both of these have to happen after decryption so
		// we can see who the last writer was.
		kops := md.config.KeyOps()
		kbpki := md.config.KBPKI()
		me, err := kbpki.GetLoggedInUser()
		if err != nil {
			return err
		}

		// re-marshal the metadata
		// TODO: can we somehow avoid the re-marshaling by saving the
		// marshalled metadata somewhere?
		var buf []byte
		if buf, err = codec.Encode(rmds.MD); err != nil {
			return err
		}

		if handle.IsPrivateShare() {
			// For private shares:
			//   * Get HMAC public key of last writer
			//   * Get shared secret with our private HMAC key
			//   * Verify using HMAC
			if hmac, ok := rmds.Macs[me]; !ok {
				return &MDMismatchError{
					handle.ToString(md.config),
					fmt.Sprintf("MD (id=%s) is a private share but doesn't "+
						"contain a key for my logged in user (%s)",
						rmds.MD.Id, me)}
				// TODO: figure out the right kid for the writer, should
				// be in the hmac somewhere
			} else if pubKey, err := kops.GetPublicMacKey(
				writer, nil); err != nil {
				return err
				// TODO: again, figure out the right kid here
			} else if privKey, err := kops.GetMyPrivateMacKey(nil); err != nil {
				return err
			} else if secret, err := crypto.SharedSecret(
				privKey, pubKey); err != nil {
				return err
			} else if err := crypto.VerifyHMAC(secret, buf, hmac); err != nil {
				return err
			}
		} else {
			// For any home or public directory:
			//   * Verify normally using the user's public key
			// TODO: what do we do if the signature is from a revoked
			// key?
			if user, err := kbpki.GetUser(writer); err != nil {
				return err
			} else if key, err := kbpki.GetPublicSigningKey(user); err != nil {
				return err
			} else if crypto.Verify(rmds.Sig, buf, key); err != nil {
				return err
			}
		}

		// Since we don't do conflict resolution yet, we don't care
		// about block changes when we're reading in the MD from the
		// server.  TODO: remove this once KBFSOps starts properly
		// doing copy-on-write for metadata.
		rmds.MD.ClearBlockChanges()
	}
	return nil
}

func (md *MDOpsStandard) GetAtHandle(handle *DirHandle) (*RootMetadata, error) {
	mdserv := md.config.MDServer()
	if rmds, err := mdserv.GetAtHandle(handle); err != nil {
		return nil, err
	} else if err := md.processMetadata(handle, rmds); err != nil {
		return nil, err
	} else {
		if rmds.IsInitialized() {
			// Make the the signed-over UIDs in the latest Keys match the handle
			handleString := handle.ToString(md.config)
			fetchedHandleString := rmds.MD.GetDirHandle().ToString(md.config)
			if fetchedHandleString != handleString {
				return nil, &MDMismatchError{
					handleString,
					fmt.Sprintf("MD (id=%s) contained unexpected handle %s",
						rmds.MD.Id, fetchedHandleString)}
			}
		}
		return &rmds.MD, nil
	}
}

func (md *MDOpsStandard) Get(id DirId) (*RootMetadata, error) {
	mdserv := md.config.MDServer()
	if rmds, err := mdserv.Get(id); err != nil {
		return nil, err
	} else {
		// Make sure the signed-over ID matches
		if id != rmds.MD.Id {
			return nil, &MDMismatchError{
				id.String(),
				fmt.Sprintf("MD contained unexpected id %s",
					rmds.MD.Id.String())}
		}
		return &rmds.MD, md.processMetadata(rmds.MD.GetDirHandle(), rmds)
	}
}

func (md *MDOpsStandard) GetAtId(id DirId, mdId MDId) (
	*RootMetadata, error) {
	// TODO: implement a cache for non-current MD
	if rmds, err := md.config.MDServer().GetAtId(id, mdId); err == nil {
		// TODO: validate and process MD
		return &rmds.MD, err
	} else {
		return nil, err
	}
}

func (md *MDOpsStandard) Put(id DirId, rmd *RootMetadata) error {
	me, err := md.config.KBPKI().GetLoggedInUser()
	if err != nil {
		return err
	}
	rmd.data.LastWriter = me

	// First encode (and maybe encrypt) the root data
	codec := md.config.Codec()
	databuf, err := codec.Encode(rmd.data)
	if err != nil {
		return err
	}
	crypto := md.config.Crypto()
	if !id.IsPublic() {
		// TODO: do we need a server-side key half for the encrypted
		// metadata?
		path := Path{rmd.Id, []PathNode{}}
		rk, err := md.config.KeyManager().GetSecretKey(path, rmd)
		if err != nil {
			return err
		}
		rmd.SerializedPrivateMetadata, err = crypto.Encrypt(databuf, rk)
		if err != nil {
			return err
		}
	} else {
		rmd.SerializedPrivateMetadata = databuf
	}

	// encode the metadata and sign it
	buf, err := codec.Encode(rmd)
	if err != nil {
		return err
	}

	handle := rmd.GetDirHandle()
	rmds := &RootMetadataSigned{}
	rmds.MD = *rmd
	if handle.IsPrivateShare() {
		// For private shares:
		//   * For each reader/writer:
		//     - Get HMAC public key
		//     - Get shared secret with our private HMAC key
		//     - Sign using HMAC
		kops := md.config.KeyOps()
		privKey, err := kops.GetMyPrivateMacKey(nil)
		if err != nil {
			return err
		}

		rmds.Macs = make(map[libkb.UID][]byte)
		hmacFunc := func(user libkb.UID) error {
			// use the latest mac keys
			if pubKey, err := kops.GetPublicMacKey(user, nil); err != nil {
				return err
			} else if secret, err := crypto.SharedSecret(
				privKey, pubKey); err != nil {
				return err
			} else if hmac, err := crypto.HMAC(secret, buf); err != nil {
				return err
			} else {
				rmds.Macs[user] = hmac
			}
			return nil
		}

		for _, w := range handle.Writers {
			if err := hmacFunc(w); err != nil {
				return err
			}
		}
		for _, r := range handle.Readers {
			if err := hmacFunc(r); err != nil {
				return err
			}
		}
	} else {
		// For our home and public directory:
		//   * Sign normally using the local device private key
		sig, err := crypto.Sign(buf)
		if err != nil {
			return err
		}
		rmds.Sig = sig
	}

	mdId, err := rmd.MetadataId(md.config)
	if err != nil {
		return err
	}
	return md.config.MDServer().Put(id, mdId, rmds)
}

func (md *MDOpsStandard) GetFavorites() ([]DirId, error) {
	mdserv := md.config.MDServer()
	return mdserv.GetFavorites()
}
<commit_msg>Fix bug where we drop an error in md_ops.go<commit_after>package libkbfs

import (
	"fmt"

	libkb "github.com/keybase/client/go/libkb"
)

type MDOpsStandard struct {
	config Config
}

func (md *MDOpsStandard) processMetadata(
	handle *DirHandle, rmds *RootMetadataSigned) error {
	crypto := md.config.Crypto()
	codec := md.config.Codec()
	// verify signature and deserialize root data, if the sig is not blank.
	// a blank sig means this is a brand new MD object, nothing to check
	if rmds.IsInitialized() {
		// decrypt the root data for non-public directories
		if !handle.IsPublic() {
			path := Path{rmds.MD.Id, []PathNode{}}
			k, err := md.config.KeyManager().GetSecretKey(path, &rmds.MD)
			if err != nil {
				return err
			}
			databuf, err := crypto.Decrypt(rmds.MD.SerializedPrivateMetadata, k)
			if err != nil {
				return err
			}
			if err := codec.Decode(databuf, &rmds.MD.data); err != nil {
				return err
			}
		} else if err := codec.Decode(
			rmds.MD.SerializedPrivateMetadata, &rmds.MD.data); err != nil {
			return err
		}

		// Make sure the last writer is really a valid writer
		writer := rmds.MD.data.LastWriter
		if !handle.IsWriter(writer) {
			return &MDMismatchError{
				handle.ToString(md.config),
				fmt.Sprintf("MD (id=%s) was written by a non-writer %s",
					rmds.MD.Id, writer)}
		}

		// TODO:
		// Both of these have to happen after decryption so
		// we can see who the last writer was.
		kops := md.config.KeyOps()
		kbpki := md.config.KBPKI()
		me, err := kbpki.GetLoggedInUser()
		if err != nil {
			return err
		}

		// re-marshal the metadata
		// TODO: can we somehow avoid the re-marshaling by saving the
		// marshalled metadata somewhere?
		var buf []byte
		if buf, err = codec.Encode(rmds.MD); err != nil {
			return err
		}

		if handle.IsPrivateShare() {
			// For private shares:
			//   * Get HMAC public key of last writer
			//   * Get shared secret with our private HMAC key
			//   * Verify using HMAC
			if hmac, ok := rmds.Macs[me]; !ok {
				return &MDMismatchError{
					handle.ToString(md.config),
					fmt.Sprintf("MD (id=%s) is a private share but doesn't "+
						"contain a key for my logged in user (%s)",
						rmds.MD.Id, me)}
				// TODO: figure out the right kid for the writer, should
				// be in the hmac somewhere
			} else if pubKey, err := kops.GetPublicMacKey(
				writer, nil); err != nil {
				return err
				// TODO: again, figure out the right kid here
			} else if privKey, err := kops.GetMyPrivateMacKey(nil); err != nil {
				return err
			} else if secret, err := crypto.SharedSecret(
				privKey, pubKey); err != nil {
				return err
			} else if err := crypto.VerifyHMAC(secret, buf, hmac); err != nil {
				return err
			}
		} else {
			// For any home or public directory:
			//   * Verify normally using the user's public key
			// TODO: what do we do if the signature is from a revoked
			// key?
			if user, err := kbpki.GetUser(writer); err != nil {
				return err
			} else if key, err := kbpki.GetPublicSigningKey(user); err != nil {
				return err
			} else if err := crypto.Verify(rmds.Sig, buf, key); err != nil {
				return err
			}
		}

		// Since we don't do conflict resolution yet, we don't care
		// about block changes when we're reading in the MD from the
		// server.  TODO: remove this once KBFSOps starts properly
		// doing copy-on-write for metadata.
		rmds.MD.ClearBlockChanges()
	}
	return nil
}

func (md *MDOpsStandard) GetAtHandle(handle *DirHandle) (*RootMetadata, error) {
	mdserv := md.config.MDServer()
	if rmds, err := mdserv.GetAtHandle(handle); err != nil {
		return nil, err
	} else if err := md.processMetadata(handle, rmds); err != nil {
		return nil, err
	} else {
		if rmds.IsInitialized() {
			// Make the the signed-over UIDs in the latest Keys match the handle
			handleString := handle.ToString(md.config)
			fetchedHandleString := rmds.MD.GetDirHandle().ToString(md.config)
			if fetchedHandleString != handleString {
				return nil, &MDMismatchError{
					handleString,
					fmt.Sprintf("MD (id=%s) contained unexpected handle %s",
						rmds.MD.Id, fetchedHandleString)}
			}
		}
		return &rmds.MD, nil
	}
}

func (md *MDOpsStandard) Get(id DirId) (*RootMetadata, error) {
	mdserv := md.config.MDServer()
	if rmds, err := mdserv.Get(id); err != nil {
		return nil, err
	} else {
		// Make sure the signed-over ID matches
		if id != rmds.MD.Id {
			return nil, &MDMismatchError{
				id.String(),
				fmt.Sprintf("MD contained unexpected id %s",
					rmds.MD.Id.String())}
		}
		return &rmds.MD, md.processMetadata(rmds.MD.GetDirHandle(), rmds)
	}
}

func (md *MDOpsStandard) GetAtId(id DirId, mdId MDId) (
	*RootMetadata, error) {
	// TODO: implement a cache for non-current MD
	if rmds, err := md.config.MDServer().GetAtId(id, mdId); err == nil {
		// TODO: validate and process MD
		return &rmds.MD, err
	} else {
		return nil, err
	}
}

func (md *MDOpsStandard) Put(id DirId, rmd *RootMetadata) error {
	me, err := md.config.KBPKI().GetLoggedInUser()
	if err != nil {
		return err
	}
	rmd.data.LastWriter = me

	// First encode (and maybe encrypt) the root data
	codec := md.config.Codec()
	databuf, err := codec.Encode(rmd.data)
	if err != nil {
		return err
	}
	crypto := md.config.Crypto()
	if !id.IsPublic() {
		// TODO: do we need a server-side key half for the encrypted
		// metadata?
		path := Path{rmd.Id, []PathNode{}}
		rk, err := md.config.KeyManager().GetSecretKey(path, rmd)
		if err != nil {
			return err
		}
		rmd.SerializedPrivateMetadata, err = crypto.Encrypt(databuf, rk)
		if err != nil {
			return err
		}
	} else {
		rmd.SerializedPrivateMetadata = databuf
	}

	// encode the metadata and sign it
	buf, err := codec.Encode(rmd)
	if err != nil {
		return err
	}

	handle := rmd.GetDirHandle()
	rmds := &RootMetadataSigned{}
	rmds.MD = *rmd
	if handle.IsPrivateShare() {
		// For private shares:
		//   * For each reader/writer:
		//     - Get HMAC public key
		//     - Get shared secret with our private HMAC key
		//     - Sign using HMAC
		kops := md.config.KeyOps()
		privKey, err := kops.GetMyPrivateMacKey(nil)
		if err != nil {
			return err
		}

		rmds.Macs = make(map[libkb.UID][]byte)
		hmacFunc := func(user libkb.UID) error {
			// use the latest mac keys
			if pubKey, err := kops.GetPublicMacKey(user, nil); err != nil {
				return err
			} else if secret, err := crypto.SharedSecret(
				privKey, pubKey); err != nil {
				return err
			} else if hmac, err := crypto.HMAC(secret, buf); err != nil {
				return err
			} else {
				rmds.Macs[user] = hmac
			}
			return nil
		}

		for _, w := range handle.Writers {
			if err := hmacFunc(w); err != nil {
				return err
			}
		}
		for _, r := range handle.Readers {
			if err := hmacFunc(r); err != nil {
				return err
			}
		}
	} else {
		// For our home and public directory:
		//   * Sign normally using the local device private key
		sig, err := crypto.Sign(buf)
		if err != nil {
			return err
		}
		rmds.Sig = sig
	}

	mdId, err := rmd.MetadataId(md.config)
	if err != nil {
		return err
	}
	return md.config.MDServer().Put(id, mdId, rmds)
}

func (md *MDOpsStandard) GetFavorites() ([]DirId, error) {
	mdserv := md.config.MDServer()
	return mdserv.GetFavorites()
}
<|endoftext|>


<commit_before>package speedtest

import (
	"context"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"golang.org/x/sync/errgroup"
)

type downloadWarmUpFunc func(context.Context, *http.Client, string) error
type downloadFunc func(context.Context, *http.Client, string, int) error
type uploadWarmUpFunc func(context.Context, *http.Client, string) error
type uploadFunc func(context.Context, *http.Client, string, int) error

var dlSizes = [...]int{350, 500, 750, 1000, 1500, 2000, 2500, 3000, 3500, 4000}
var ulSizes = [...]int{100, 300, 500, 800, 1000, 1500, 2500, 3000, 3500, 4000} //kB

// DownloadTest executes the test to measure download speed
func (s *Server) DownloadTest(savingMode bool) error {
	return s.downloadTestContext(context.Background(), savingMode, dlWarmUp, downloadRequest)
}

// DownloadTestContext executes the test to measure download speed, observing the given context.
func (s *Server) DownloadTestContext(ctx context.Context, savingMode bool) error {
	return s.downloadTestContext(ctx, savingMode, dlWarmUp, downloadRequest)
}

func (s *Server) downloadTestContext(
	ctx context.Context,
	savingMode bool,
	dlWarmUp downloadWarmUpFunc,
	downloadRequest downloadFunc,
) error {
	dlURL := strings.Split(s.URL, "/upload.php")[0]
	eg := errgroup.Group{}

	// Warming up
	sTime := time.Now()
	for i := 0; i < 2; i++ {
		eg.Go(func() error {
			return dlWarmUp(ctx, s.doer, dlURL)
		})
	}
	if err := eg.Wait(); err != nil {
		return err
	}
	fTime := time.Now()
	// 1.125MB for each request (750 * 750 * 2)
	wuSpeed := 1.125 * 8 * 2 / fTime.Sub(sTime.Add(s.Latency)).Seconds()

	// Decide workload by warm up speed
	workload := 0
	weight := 0
	skip := false
	if savingMode {
		workload = 6
		weight = 3
	} else if 50.0 < wuSpeed {
		workload = 32
		weight = 6
	} else if 10.0 < wuSpeed {
		workload = 16
		weight = 4
	} else if 4.0 < wuSpeed {
		workload = 8
		weight = 4
	} else if 2.5 < wuSpeed {
		workload = 4
		weight = 4
	} else {
		skip = true
	}

	// Main speedtest
	dlSpeed := wuSpeed
	if !skip {
		sTime = time.Now()
		for i := 0; i < workload; i++ {
			eg.Go(func() error {
				return downloadRequest(ctx, s.doer, dlURL, weight)
			})
		}
		if err := eg.Wait(); err != nil {
			return err
		}
		fTime = time.Now()

		reqMB := dlSizes[weight] * dlSizes[weight] * 2 / 1000 / 1000
		dlSpeed = float64(reqMB) * 8 * float64(workload) / fTime.Sub(sTime).Seconds()
	}

	s.DLSpeed = dlSpeed
	return nil
}

// UploadTest executes the test to measure upload speed
func (s *Server) UploadTest(savingMode bool) error {
	return s.uploadTestContext(context.Background(), savingMode, ulWarmUp, uploadRequest)
}

// UploadTestContext executes the test to measure upload speed, observing the given context.
func (s *Server) UploadTestContext(ctx context.Context, savingMode bool) error {
	return s.uploadTestContext(ctx, savingMode, ulWarmUp, uploadRequest)
}
func (s *Server) uploadTestContext(
	ctx context.Context,
	savingMode bool,
	ulWarmUp uploadWarmUpFunc,
	uploadRequest uploadFunc,
) error {
	// Warm up
	sTime := time.Now()
	eg := errgroup.Group{}
	for i := 0; i < 2; i++ {
		eg.Go(func() error {
			return ulWarmUp(ctx, s.doer, s.URL)
		})
	}
	if err := eg.Wait(); err != nil {
		return err
	}
	fTime := time.Now()
	// 1.0 MB for each request
	wuSpeed := 1.0 * 8 * 2 / fTime.Sub(sTime.Add(s.Latency)).Seconds()

	// Decide workload by warm up speed
	workload := 0
	weight := 0
	skip := false
	if savingMode {
		workload = 1
		weight = 7
	} else if 50.0 < wuSpeed {
		workload = 40
		weight = 9
	} else if 10.0 < wuSpeed {
		workload = 16
		weight = 9
	} else if 4.0 < wuSpeed {
		workload = 8
		weight = 9
	} else if 2.5 < wuSpeed {
		workload = 4
		weight = 5
	} else {
		skip = true
	}

	// Main speedtest
	ulSpeed := wuSpeed
	if !skip {
		sTime = time.Now()
		for i := 0; i < workload; i++ {
			eg.Go(func() error {
				return uploadRequest(ctx, s.doer, s.URL, weight)
			})
		}
		if err := eg.Wait(); err != nil {
			return err
		}
		fTime = time.Now()

		reqMB := float64(ulSizes[weight]) / 1000
		ulSpeed = reqMB * 8 * float64(workload) / fTime.Sub(sTime).Seconds()
	}

	s.ULSpeed = ulSpeed

	return nil
}

func dlWarmUp(ctx context.Context, doer *http.Client, dlURL string) error {
	size := dlSizes[2]
	xdlURL := dlURL + "/random" + strconv.Itoa(size) + "x" + strconv.Itoa(size) + ".jpg"

	req, err := http.NewRequestWithContext(ctx, http.MethodGet, xdlURL, nil)
	if err != nil {
		return err
	}

	resp, err := doer.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	_, err = io.Copy(ioutil.Discard, resp.Body)
	return err
}

func ulWarmUp(ctx context.Context, doer *http.Client, ulURL string) error {
	size := ulSizes[4]
	v := url.Values{}
	v.Add("content", strings.Repeat("0123456789", size*100-51))

	req, err := http.NewRequestWithContext(ctx, http.MethodPost, ulURL, strings.NewReader(v.Encode()))
	if err != nil {
		return err
	}

	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	resp, err := doer.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	_, err = io.Copy(ioutil.Discard, resp.Body)
	return err
}

func downloadRequest(ctx context.Context, doer *http.Client, dlURL string, w int) error {
	size := dlSizes[w]
	xdlURL := dlURL + "/random" + strconv.Itoa(size) + "x" + strconv.Itoa(size) + ".jpg"

	req, err := http.NewRequestWithContext(ctx, http.MethodGet, xdlURL, nil)
	if err != nil {
		return err
	}

	resp, err := doer.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	_, err = io.Copy(ioutil.Discard, resp.Body)
	return err
}

func uploadRequest(ctx context.Context, doer *http.Client, ulURL string, w int) error {
	size := ulSizes[w]
	v := url.Values{}
	v.Add("content", strings.Repeat("0123456789", size*100-51))

	req, err := http.NewRequestWithContext(ctx, http.MethodPost, ulURL, strings.NewReader(v.Encode()))
	if err != nil {
		return err
	}

	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	resp, err := doer.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	_, err = io.Copy(ioutil.Discard, resp.Body)
	return err
}

// PingTest executes test to measure latency
func (s *Server) PingTest() error {
	return s.PingTestContext(context.Background())
}

// PingTestContext executes test to measure latency, observing the given context.
func (s *Server) PingTestContext(ctx context.Context) error {
	pingURL := strings.Split(s.URL, "/upload.php")[0] + "/latency.txt"

	l := time.Second * 10
	for i := 0; i < 3; i++ {
		sTime := time.Now()

		req, err := http.NewRequestWithContext(ctx, http.MethodGet, pingURL, nil)
		if err != nil {
			return err
		}

		resp, err := s.doer.Do(req)
		if err != nil {
			return err
		}

		fTime := time.Now()
		if fTime.Sub(sTime) < l {
			l = fTime.Sub(sTime)
		}

		resp.Body.Close()
	}

	s.Latency = time.Duration(int64(l.Nanoseconds() / 2))

	return nil
}
<commit_msg>fix: speed value can be minus if the bandwidth is large<commit_after>package speedtest

import (
	"context"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"golang.org/x/sync/errgroup"
)

type downloadWarmUpFunc func(context.Context, *http.Client, string) error
type downloadFunc func(context.Context, *http.Client, string, int) error
type uploadWarmUpFunc func(context.Context, *http.Client, string) error
type uploadFunc func(context.Context, *http.Client, string, int) error

var dlSizes = [...]int{350, 500, 750, 1000, 1500, 2000, 2500, 3000, 3500, 4000}
var ulSizes = [...]int{100, 300, 500, 800, 1000, 1500, 2500, 3000, 3500, 4000} //kB

// DownloadTest executes the test to measure download speed
func (s *Server) DownloadTest(savingMode bool) error {
	return s.downloadTestContext(context.Background(), savingMode, dlWarmUp, downloadRequest)
}

// DownloadTestContext executes the test to measure download speed, observing the given context.
func (s *Server) DownloadTestContext(ctx context.Context, savingMode bool) error {
	return s.downloadTestContext(ctx, savingMode, dlWarmUp, downloadRequest)
}

func (s *Server) downloadTestContext(
	ctx context.Context,
	savingMode bool,
	dlWarmUp downloadWarmUpFunc,
	downloadRequest downloadFunc,
) error {
	dlURL := strings.Split(s.URL, "/upload.php")[0]
	eg := errgroup.Group{}

	// Warming up
	sTime := time.Now()
	for i := 0; i < 2; i++ {
		eg.Go(func() error {
			return dlWarmUp(ctx, s.doer, dlURL)
		})
	}
	if err := eg.Wait(); err != nil {
		return err
	}
	fTime := time.Now()

	// If the bandwidth is too large, the download sometimes finish earlier than the latency.
	// In this case, we ignore the the latency that is included server information.
	// This is not affected to the final result since this is a warm up test.
	timeToSpend := fTime.Sub(sTime.Add(s.Latency)).Seconds()
	if timeToSpend < 0 {
		timeToSpend = fTime.Sub(sTime).Seconds()
	}

	// 1.125MB for each request (750 * 750 * 2)
	wuSpeed := 1.125 * 8 * 2 / timeToSpend

	// Decide workload by warm up speed
	workload := 0
	weight := 0
	skip := false
	if savingMode {
		workload = 6
		weight = 3
	} else if 50.0 < wuSpeed {
		workload = 32
		weight = 6
	} else if 10.0 < wuSpeed {
		workload = 16
		weight = 4
	} else if 4.0 < wuSpeed {
		workload = 8
		weight = 4
	} else if 2.5 < wuSpeed {
		workload = 4
		weight = 4
	} else {
		skip = true
	}

	// Main speedtest
	dlSpeed := wuSpeed
	if !skip {
		sTime = time.Now()
		for i := 0; i < workload; i++ {
			eg.Go(func() error {
				return downloadRequest(ctx, s.doer, dlURL, weight)
			})
		}
		if err := eg.Wait(); err != nil {
			return err
		}
		fTime = time.Now()

		reqMB := dlSizes[weight] * dlSizes[weight] * 2 / 1000 / 1000
		dlSpeed = float64(reqMB) * 8 * float64(workload) / fTime.Sub(sTime).Seconds()
	}

	s.DLSpeed = dlSpeed
	return nil
}

// UploadTest executes the test to measure upload speed
func (s *Server) UploadTest(savingMode bool) error {
	return s.uploadTestContext(context.Background(), savingMode, ulWarmUp, uploadRequest)
}

// UploadTestContext executes the test to measure upload speed, observing the given context.
func (s *Server) UploadTestContext(ctx context.Context, savingMode bool) error {
	return s.uploadTestContext(ctx, savingMode, ulWarmUp, uploadRequest)
}
func (s *Server) uploadTestContext(
	ctx context.Context,
	savingMode bool,
	ulWarmUp uploadWarmUpFunc,
	uploadRequest uploadFunc,
) error {
	// Warm up
	sTime := time.Now()
	eg := errgroup.Group{}
	for i := 0; i < 2; i++ {
		eg.Go(func() error {
			return ulWarmUp(ctx, s.doer, s.URL)
		})
	}
	if err := eg.Wait(); err != nil {
		return err
	}
	fTime := time.Now()
	// 1.0 MB for each request
	wuSpeed := 1.0 * 8 * 2 / fTime.Sub(sTime.Add(s.Latency)).Seconds()

	// Decide workload by warm up speed
	workload := 0
	weight := 0
	skip := false
	if savingMode {
		workload = 1
		weight = 7
	} else if 50.0 < wuSpeed {
		workload = 40
		weight = 9
	} else if 10.0 < wuSpeed {
		workload = 16
		weight = 9
	} else if 4.0 < wuSpeed {
		workload = 8
		weight = 9
	} else if 2.5 < wuSpeed {
		workload = 4
		weight = 5
	} else {
		skip = true
	}

	// Main speedtest
	ulSpeed := wuSpeed
	if !skip {
		sTime = time.Now()
		for i := 0; i < workload; i++ {
			eg.Go(func() error {
				return uploadRequest(ctx, s.doer, s.URL, weight)
			})
		}
		if err := eg.Wait(); err != nil {
			return err
		}
		fTime = time.Now()

		reqMB := float64(ulSizes[weight]) / 1000
		ulSpeed = reqMB * 8 * float64(workload) / fTime.Sub(sTime).Seconds()
	}

	s.ULSpeed = ulSpeed

	return nil
}

func dlWarmUp(ctx context.Context, doer *http.Client, dlURL string) error {
	size := dlSizes[2]
	xdlURL := dlURL + "/random" + strconv.Itoa(size) + "x" + strconv.Itoa(size) + ".jpg"

	req, err := http.NewRequestWithContext(ctx, http.MethodGet, xdlURL, nil)
	if err != nil {
		return err
	}

	resp, err := doer.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	_, err = io.Copy(ioutil.Discard, resp.Body)
	return err
}

func ulWarmUp(ctx context.Context, doer *http.Client, ulURL string) error {
	size := ulSizes[4]
	v := url.Values{}
	v.Add("content", strings.Repeat("0123456789", size*100-51))

	req, err := http.NewRequestWithContext(ctx, http.MethodPost, ulURL, strings.NewReader(v.Encode()))
	if err != nil {
		return err
	}

	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	resp, err := doer.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	_, err = io.Copy(ioutil.Discard, resp.Body)
	return err
}

func downloadRequest(ctx context.Context, doer *http.Client, dlURL string, w int) error {
	size := dlSizes[w]
	xdlURL := dlURL + "/random" + strconv.Itoa(size) + "x" + strconv.Itoa(size) + ".jpg"

	req, err := http.NewRequestWithContext(ctx, http.MethodGet, xdlURL, nil)
	if err != nil {
		return err
	}

	resp, err := doer.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	_, err = io.Copy(ioutil.Discard, resp.Body)
	return err
}

func uploadRequest(ctx context.Context, doer *http.Client, ulURL string, w int) error {
	size := ulSizes[w]
	v := url.Values{}
	v.Add("content", strings.Repeat("0123456789", size*100-51))

	req, err := http.NewRequestWithContext(ctx, http.MethodPost, ulURL, strings.NewReader(v.Encode()))
	if err != nil {
		return err
	}

	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	resp, err := doer.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	_, err = io.Copy(ioutil.Discard, resp.Body)
	return err
}

// PingTest executes test to measure latency
func (s *Server) PingTest() error {
	return s.PingTestContext(context.Background())
}

// PingTestContext executes test to measure latency, observing the given context.
func (s *Server) PingTestContext(ctx context.Context) error {
	pingURL := strings.Split(s.URL, "/upload.php")[0] + "/latency.txt"

	l := time.Second * 10
	for i := 0; i < 3; i++ {
		sTime := time.Now()

		req, err := http.NewRequestWithContext(ctx, http.MethodGet, pingURL, nil)
		if err != nil {
			return err
		}

		resp, err := s.doer.Do(req)
		if err != nil {
			return err
		}

		fTime := time.Now()
		if fTime.Sub(sTime) < l {
			l = fTime.Sub(sTime)
		}

		resp.Body.Close()
	}

	s.Latency = time.Duration(int64(l.Nanoseconds() / 2))

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"os"
	"strconv"
	"strings"
)

func main() {
	file, _ := os.Open(os.Args[1])

	scanner := bufio.NewScanner(file)
	triangle := make([][]int64, 0)

	for scanner.Scan() {

		line := make([]int64, 0)

		for _, elem := range strings.Split(scanner.Text(), " ") {
			num, _ := strconv.ParseInt(elem, 10, 64)
			line = append(line, num)
		}

		triangle = append(triangle, line)
	}

	fmt.Printf("%v", triangle)
}
<commit_msg>Implemented Solution<commit_after>package main

import (
	"bufio"
	"fmt"
	"math"
	"os"
	"strconv"
	"strings"
)

func main() {
	file, _ := os.Open(os.Args[1])

	scanner := bufio.NewScanner(file)
	triangle := make([][]float64, 0)

	for scanner.Scan() {

		line := make([]float64, 0)

		for _, elem := range strings.Split(scanner.Text(), " ") {
			num, _ := strconv.ParseFloat(elem, 64)
			line = append(line, num)
		}

		triangle = append(triangle, line)
	}

	for row := len(triangle) - 2; row >= 0; row-- {
		for sub := 0; sub < len(triangle[row]); sub++ {
			triangle[row][sub] = math.Max(
				triangle[row][sub]+triangle[row+1][sub],
				triangle[row][sub]+triangle[row+1][sub+1])
		}
	}

	fmt.Printf("%v", triangle)
}
<|endoftext|>