<commit_before>// osop
// Copyright (C) 2014 Karol 'Kenji Takahashi' Woźniak
//
// Permission is hereby granted, free of charge, to any person obtaining
// a copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
// TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
// OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"strconv"
)

const URL = "http://api.openweathermap.org/data/2.5/weather"

type Owm struct {
	url string
}

type owmResponse struct {
	City     string
	Country  string
	Sunrise  uint64
	Sunset   uint64
	Temp     float64
	TempMin  float64
	TempMax  float64
	Pressure int
	Humidity int

	Wind struct {
		Speed float64
		Deg   int
	}

	Coord struct {
		Lon float64
		Lat float64
	}
}

func (o *Owm) Get() (interface{}, error) {
	resp, err := http.Get(o.url)
	if err != nil {
		return nil, fmt.Errorf("Cannot get response: `%s`", err)
	}
	defer resp.Body.Close()

	var decoded struct {
		Coord struct {
			Lon float64
			Lat float64
		}
		Sys struct {
			Country string
			Sunrise uint64
			Sunset  uint64
		}
		Main struct {
			Temp     float64
			Pressure int
			Humidity int
			Temp_min float64
			Temp_max float64
		}
		Wind struct {
			Speed float64
			Deg   int
		}
		Name string
	}
	json.NewDecoder(resp.Body).Decode(&decoded)

	return owmResponse{
		City:     decoded.Name,
		Country:  decoded.Sys.Country,
		Sunrise:  decoded.Sys.Sunrise,
		Sunset:   decoded.Sys.Sunset,
		Temp:     decoded.Main.Temp,
		TempMin:  decoded.Main.Temp_min,
		TempMax:  decoded.Main.Temp_max,
		Pressure: decoded.Main.Pressure,
		Humidity: decoded.Main.Humidity,
		Wind:     decoded.Wind,
		Coord:    decoded.Coord,
	}, nil
}

func (o *Owm) Init(config config) error {
	if config["location"] == nil {
		return fmt.Errorf("Location parameter is required for Owm receiver")
	}
	if config["apiKey"] == nil {
		return fmt.Errorf("`apiKey` parameter is required")
	}

	_url, err := url.Parse(URL)
	if err != nil {
		return fmt.Errorf("Cannot parse Owm URL: `%s`", err)
	}
	location := config["location"].(string)
	_, err = strconv.Atoi(location)
	urlQuery := url.Values{}
	if err != nil {
		urlQuery.Add("q", location)
	} else {
		urlQuery.Add("id", location)
	}
	urlQuery.Add("APPID", config["apiKey"].(string))

	units := "metric"
	if config["units"] != nil {
		_units := config["units"].(string)
		if _units != "metric" && _units != "imperial" {
			log.Printf("Unknown units (%s), using `metric`\n", _units)
		} else {
			units = _units
		}
	}
	urlQuery.Add("units", units)

	_url.RawQuery = urlQuery.Encode()

	o.url = _url.String()
	return nil
}

func init() {
	registry.AddReceiver("Owm", &Owm{}, owmResponse{})
}
<commit_msg>owm: adjust error messages<commit_after>// osop
// Copyright (C) 2014,2016 Karol 'Kenji Takahashi' Woźniak
//
// Permission is hereby granted, free of charge, to any person obtaining
// a copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
// TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
// OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"strconv"
)

const URL = "http://api.openweathermap.org/data/2.5/weather"

type Owm struct {
	url string
}

type owmResponse struct {
	City     string
	Country  string
	Sunrise  uint64
	Sunset   uint64
	Temp     float64
	TempMin  float64
	TempMax  float64
	Pressure int
	Humidity int

	Wind struct {
		Speed float64
		Deg   int
	}

	Coord struct {
		Lon float64
		Lat float64
	}
}

func (o *Owm) Get() (interface{}, error) {
	resp, err := http.Get(o.url)
	if err != nil {
		return nil, fmt.Errorf("Cannot get response: `%s`", err)
	}
	defer resp.Body.Close()

	var decoded struct {
		Coord struct {
			Lon float64
			Lat float64
		}
		Sys struct {
			Country string
			Sunrise uint64
			Sunset  uint64
		}
		Main struct {
			Temp     float64
			Pressure int
			Humidity int
			Temp_min float64
			Temp_max float64
		}
		Wind struct {
			Speed float64
			Deg   int
		}
		Name string
	}
	json.NewDecoder(resp.Body).Decode(&decoded)

	return owmResponse{
		City:     decoded.Name,
		Country:  decoded.Sys.Country,
		Sunrise:  decoded.Sys.Sunrise,
		Sunset:   decoded.Sys.Sunset,
		Temp:     decoded.Main.Temp,
		TempMin:  decoded.Main.Temp_min,
		TempMax:  decoded.Main.Temp_max,
		Pressure: decoded.Main.Pressure,
		Humidity: decoded.Main.Humidity,
		Wind:     decoded.Wind,
		Coord:    decoded.Coord,
	}, nil
}

func (o *Owm) Init(config config) error {
	if config["location"] == nil {
		return fmt.Errorf("`location` parameter is required")
	}
	if config["apiKey"] == nil {
		return fmt.Errorf("`apiKey` parameter is required")
	}

	_url, err := url.Parse(URL)
	if err != nil {
		return fmt.Errorf("Cannot parse URL: `%s`", err)
	}
	location := config["location"].(string)
	_, err = strconv.Atoi(location)
	urlQuery := url.Values{}
	if err != nil {
		urlQuery.Add("q", location)
	} else {
		urlQuery.Add("id", location)
	}
	urlQuery.Add("APPID", config["apiKey"].(string))

	units := "metric"
	if config["units"] != nil {
		_units := config["units"].(string)
		if _units != "metric" && _units != "imperial" {
			log.Printf("Unknown units `%s`, using `metric`\n", _units)
		} else {
			units = _units
		}
	}
	urlQuery.Add("units", units)

	_url.RawQuery = urlQuery.Encode()

	o.url = _url.String()
	return nil
}

func init() {
	registry.AddReceiver("Owm", &Owm{}, owmResponse{})
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
	"time"
)

// how often to check the feeds (in minutes)
const checkEvery = 3

// ignore all posts that are older than X minutes
const freshness = 90

// if there’s an error reading a feed, retry after X minutes
const retryAfter = 9

// how many items to show if there have been many updates in an interval
const maxItems = 3

var bootTimestamp = time.Now()

var rssHttpClient = http.Client{Timeout: 10 * time.Second}

func Rss() {
	// this feels wrong, the missing alignment making it hard to read.
	// Does anybody have a suggestion how to make this nice in go?
	// go pollFeed("#i3", "i3faq", timeFormat1, "https://faq.i3wm.org/feeds/rss/")

	go pollFeed("#chaos-hd", "nn-web", "https://www.noname-ev.de/gitcommits.atom")
	go pollFeed("#chaos-hd", "nn-wiki", "https://www.noname-ev.de/wiki/index.php?title=Special:RecentChanges&feed=atom")
	go pollFeed("#chaos-hd", "nn-planet", "http://blogs.noname-ev.de/atom.xml")
	go pollFeed("#chaos-hd", "frank", "https://github.com/breunigs/frank/commits/robust.atom")
}

type Feed struct {
	// XMLName Name      `xml:"http://www.w3.org/2005/Atom feed"`
	TitleRaw string    `xml:"title"`
	Id       string    `xml:"id"`
	Link     string    `xml:"link"`
	Updated  time.Time `xml:"updated,attr"`
	Author   string    `xml:"author"`
	Entry    []Entry   `xml:"entry"`
}

func (f Feed) postableForIrc() []string {
	oneLiners := []string{}

	for _, entry := range f.Entry {
		if !entry.RecentlyPublished() {
			if *verbose {
				// log.Printf("RSS: skipping non-recent entry. published @ %s :: %s %s", entry.Updated, f.Title(), entry.Title())
			}
			continue
		}

		if isRecentUrl(entry.Href()) {
			if *verbose {
				log.Printf("RSS: skipping already already posted :: %s %s", f.Title(), entry.Title())
			}
			continue
		}
		addRecentUrl(entry.Href())

		oneLiners = appendIfMiss(oneLiners, entry.OneLiner())
	}

	return oneLiners
}

func (f Feed) Title() string {
	return strings.TrimSpace(f.TitleRaw)
}

type Entry struct {
	TitleRaw string    `xml:"title"`
	Id       string    `xml:"id"`
	Link     []Link    `xml:"link"`
	Updated  time.Time `xml:"updated"`
	Author   string    `xml:"author>name"`
}

func (e Entry) Title() string {
	return strings.TrimSpace(e.TitleRaw)
}

func (e Entry) RecentlyPublished() bool {
	if bootTimestamp.After(e.Updated) {
		return false
	}

	return time.Since(e.Updated) < freshness*time.Minute
}

func (e Entry) Href() string {
	if len(e.Link) == 0 {
		return ""
	}

	return strings.TrimSpace(e.Link[0].Href)
}

func (e Entry) OneLiner() string {
	author := strings.TrimSpace(e.Author)
	if author != "" {
		author = " (by " + author + ")"
	}

	return e.Title() + author + " " + e.Href()
}

type Link struct {
	Rel  string `xml:"rel,attr,omitempty"`
	Href string `xml:"href,attr"`
}

func loadURL(url string) []byte {
	r, err := rssHttpClient.Get(url)

	if err != nil {
		log.Printf("RSS: could resolve URL %s: %s\n", url, err)
		return []byte{}
	}
	defer r.Body.Close()

	// read up to 1 MB
	limitedBody := io.LimitReader(r.Body, 1024*1024)
	body, err := ioutil.ReadAll(limitedBody)
	if err != nil {
		log.Printf("RSS: could read data from URL %s: %s\n", url, err)
		return []byte{}
	}

	return body
}

func parseAtomFeed(url string) Feed {
	f := Feed{}
	if err := xml.Unmarshal(loadURL(url), &f); err != nil {
		log.Printf("RSS: could not parse %s: %s\n", url, err)
	}

	return f
}

func pollFeed(channel string, feedName string, url string) {
	for {
		time.Sleep(checkEvery * time.Minute)
		if *verbose {
			log.Printf("RSS %s: checking", feedName)
		}
		pollFeedRunner(channel, feedName, url)
	}
}

func pollFeedRunner(channel string, feedName string, url string) {
	defer func() {
		if r := recover(); r != nil {
			log.Printf("MEGA-WTF:pkg:RSS: %v\n", r)
			time.Sleep(retryAfter * time.Minute)
			return
		}
	}()

	postitems := parseAtomFeed(url).postableForIrc()
	cnt := len(postitems)
	log.Printf("RSS %s: found %d new items: %v", feedName, cnt, postitems)

	// hide updates if they exceed the maxItems counter. If there’s only
	// one more item in the list than specified in maxItems, all of the
	// items will be printed – otherwise that item would be replaced by
	// a useless message that it has been hidden.
	if cnt > maxItems+1 {
		msg := fmt.Sprintf("::%s:: had %d updates, showing the latest %d", feedName, cnt, maxItems)
		Privmsg(channel, msg)
		postitems = postitems[cnt-maxItems : cnt]
		log.Printf("RSS %s: posting %s\n", feedName, msg)
	}

	// newer items appear first in feeds, so reverse them here to keep
	// the order in line with how IRC wprks
	for i := len(postitems) - 1; i >= 0; i -= 1 {
		Privmsg(channel, "::"+feedName+":: "+postitems[i])
		log.Printf("RSS %s: posting %s\n", feedName, postitems[i])
	}
}

// append string to slice only if it’s not already present.
func appendIfMiss(slice []string, s string) []string {
	for _, elm := range slice {
		if elm == s {
			return slice
		}
	}
	return append(slice, s)
}

// LIFO that stores the recent posted URLs. Used to avoid posting entries multiple times.
var recent []string = make([]string, 50)
var recentIndex = 0

func addRecentUrl(url string) {
	recent[recentIndex] = url
	recentIndex += 1
	if len(recent) == recentIndex {
		recentIndex = 0
	}
}

func isRecentUrl(url string) bool {
	for _, a := range recent {
		if url == a {
			return true
		}
	}
	return false
}
<commit_msg>re-enable skipped entry printing<commit_after>package main

import (
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
	"time"
)

// how often to check the feeds (in minutes)
const checkEvery = 3

// ignore all posts that are older than X minutes
const freshness = 90

// if there’s an error reading a feed, retry after X minutes
const retryAfter = 9

// how many items to show if there have been many updates in an interval
const maxItems = 3

var bootTimestamp = time.Now()

var rssHttpClient = http.Client{Timeout: 10 * time.Second}

func Rss() {
	// this feels wrong, the missing alignment making it hard to read.
	// Does anybody have a suggestion how to make this nice in go?
	// go pollFeed("#i3", "i3faq", timeFormat1, "https://faq.i3wm.org/feeds/rss/")

	go pollFeed("#chaos-hd", "nn-web", "https://www.noname-ev.de/gitcommits.atom")
	go pollFeed("#chaos-hd", "nn-wiki", "https://www.noname-ev.de/wiki/index.php?title=Special:RecentChanges&feed=atom")
	go pollFeed("#chaos-hd", "nn-planet", "http://blogs.noname-ev.de/atom.xml")
	go pollFeed("#chaos-hd", "frank", "https://github.com/breunigs/frank/commits/robust.atom")
}

type Feed struct {
	// XMLName Name      `xml:"http://www.w3.org/2005/Atom feed"`
	TitleRaw string    `xml:"title"`
	Id       string    `xml:"id"`
	Link     string    `xml:"link"`
	Updated  time.Time `xml:"updated,attr"`
	Author   string    `xml:"author"`
	Entry    []Entry   `xml:"entry"`
}

func (f Feed) postableForIrc() []string {
	oneLiners := []string{}

	for _, entry := range f.Entry {
		if !entry.RecentlyPublished() {
			if *verbose {
				log.Printf("RSS: skipping non-recent entry. published @ %s :: %s %s", entry.Updated, f.Title(), entry.Title())
			}
			continue
		}

		if isRecentUrl(entry.Href()) {
			if *verbose {
				log.Printf("RSS: skipping already already posted :: %s %s", f.Title(), entry.Title())
			}
			continue
		}
		addRecentUrl(entry.Href())

		oneLiners = appendIfMiss(oneLiners, entry.OneLiner())
	}

	return oneLiners
}

func (f Feed) Title() string {
	return strings.TrimSpace(f.TitleRaw)
}

type Entry struct {
	TitleRaw string    `xml:"title"`
	Id       string    `xml:"id"`
	Link     []Link    `xml:"link"`
	Updated  time.Time `xml:"updated"`
	Author   string    `xml:"author>name"`
}

func (e Entry) Title() string {
	return strings.TrimSpace(e.TitleRaw)
}

func (e Entry) RecentlyPublished() bool {
	if bootTimestamp.After(e.Updated) {
		return false
	}

	return time.Since(e.Updated) < freshness*time.Minute
}

func (e Entry) Href() string {
	if len(e.Link) == 0 {
		return ""
	}

	return strings.TrimSpace(e.Link[0].Href)
}

func (e Entry) OneLiner() string {
	author := strings.TrimSpace(e.Author)
	if author != "" {
		author = " (by " + author + ")"
	}

	return e.Title() + author + " " + e.Href()
}

type Link struct {
	Rel  string `xml:"rel,attr,omitempty"`
	Href string `xml:"href,attr"`
}

func loadURL(url string) []byte {
	r, err := rssHttpClient.Get(url)

	if err != nil {
		log.Printf("RSS: could resolve URL %s: %s\n", url, err)
		return []byte{}
	}
	defer r.Body.Close()

	// read up to 1 MB
	limitedBody := io.LimitReader(r.Body, 1024*1024)
	body, err := ioutil.ReadAll(limitedBody)
	if err != nil {
		log.Printf("RSS: could read data from URL %s: %s\n", url, err)
		return []byte{}
	}

	return body
}

func parseAtomFeed(url string) Feed {
	f := Feed{}
	if err := xml.Unmarshal(loadURL(url), &f); err != nil {
		log.Printf("RSS: could not parse %s: %s\n", url, err)
	}

	return f
}

func pollFeed(channel string, feedName string, url string) {
	for {
		time.Sleep(checkEvery * time.Minute)
		if *verbose {
			log.Printf("RSS %s: checking", feedName)
		}
		pollFeedRunner(channel, feedName, url)
	}
}

func pollFeedRunner(channel string, feedName string, url string) {
	defer func() {
		if r := recover(); r != nil {
			log.Printf("MEGA-WTF:pkg:RSS: %v\n", r)
			time.Sleep(retryAfter * time.Minute)
			return
		}
	}()

	postitems := parseAtomFeed(url).postableForIrc()
	cnt := len(postitems)
	log.Printf("RSS %s: found %d new items: %v", feedName, cnt, postitems)

	// hide updates if they exceed the maxItems counter. If there’s only
	// one more item in the list than specified in maxItems, all of the
	// items will be printed – otherwise that item would be replaced by
	// a useless message that it has been hidden.
	if cnt > maxItems+1 {
		msg := fmt.Sprintf("::%s:: had %d updates, showing the latest %d", feedName, cnt, maxItems)
		Privmsg(channel, msg)
		postitems = postitems[cnt-maxItems : cnt]
		log.Printf("RSS %s: posting %s\n", feedName, msg)
	}

	// newer items appear first in feeds, so reverse them here to keep
	// the order in line with how IRC wprks
	for i := len(postitems) - 1; i >= 0; i -= 1 {
		Privmsg(channel, "::"+feedName+":: "+postitems[i])
		log.Printf("RSS %s: posting %s\n", feedName, postitems[i])
	}
}

// append string to slice only if it’s not already present.
func appendIfMiss(slice []string, s string) []string {
	for _, elm := range slice {
		if elm == s {
			return slice
		}
	}
	return append(slice, s)
}

// LIFO that stores the recent posted URLs. Used to avoid posting entries multiple times.
var recent []string = make([]string, 50)
var recentIndex = 0

func addRecentUrl(url string) {
	recent[recentIndex] = url
	recentIndex += 1
	if len(recent) == recentIndex {
		recentIndex = 0
	}
}

func isRecentUrl(url string) bool {
	for _, a := range recent {
		if url == a {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>// Tests for recurring jobs firing appropriately.
package job

import (
	"fmt"
	"runtime"
	"testing"
	"time"

	"github.com/mixer/clock"
	"github.com/stretchr/testify/assert"
)

var recurTableTests = []struct {
	Name        string
	Location    string
	Start       string
	Interval    string
	Checkpoints []string
}{
	{
		Name:     "Daily",
		Location: "America/Los_Angeles",
		Start:    "2020-Jan-13 14:09",
		Interval: "P1D",
		Checkpoints: []string{
			"2020-Jan-14 14:09",
			"2020-Jan-15 14:09",
			"2020-Jan-16 14:09",
		},
	},
	{
		Name:     "Daily across DST boundary",
		Location: "America/Los_Angeles",
		Start:    "2020-Mar-05 14:09",
		Interval: "P1D",
		Checkpoints: []string{
			"2020-Mar-06 14:09",
			"2020-Mar-07 14:09",
			"2020-Mar-08 14:09",
			"2020-Mar-09 14:09",
		},
	},
	{
		Name:     "24 Hourly across DST boundary",
		Location: "America/Los_Angeles",
		Start:    "2020-Mar-05 14:09",
		Interval: "PT24H",
		Checkpoints: []string{
			"2020-Mar-06 14:09",
			"2020-Mar-07 14:09",
			"2020-Mar-08 15:09",
			"2020-Mar-09 15:09",
		},
	},
	{
		Name:     "Weekly",
		Location: "America/Los_Angeles",
		Start:    "2020-Jan-13 14:09",
		Interval: "P1W",
		Checkpoints: []string{
			"2020-Jan-20 14:09",
			"2020-Jan-27 14:09",
			"2020-Feb-03 14:09",
		},
	},
	{
		Name:     "Monthly",
		Location: "America/Los_Angeles",
		Start:    "2020-Jan-20 14:09",
		Interval: "P1M",
		Checkpoints: []string{
			"2020-Feb-20 14:09",
			"2020-Mar-20 14:09",
			"2020-Apr-20 14:09",
			"2020-May-20 14:09",
			"2020-Jun-20 14:09",
			"2020-Jul-20 14:09",
			"2020-Aug-20 14:09",
			"2020-Sep-20 14:09",
			"2020-Oct-20 14:09",
			"2020-Nov-20 14:09",
			"2020-Dec-20 14:09",
			"2021-Jan-20 14:09",
		},
	},
	{
		Name:     "Monthly with Normalization",
		Location: "America/Los_Angeles",
		Start:    "2020-Jul-31 14:09",
		Interval: "P1M",
		Checkpoints: []string{
			"2020-Aug-31 14:09",
			"2020-Oct-01 14:09",
			"2020-Nov-01 14:09",
		},
	},
	{
		Name:     "Yearly across Leap Year boundary",
		Location: "America/Los_Angeles",
		Start:    "2020-Jan-20 14:09",
		Interval: "P1Y",
		Checkpoints: []string{
			"2021-Jan-20 14:09",
			"2022-Jan-20 14:09",
			"2023-Jan-20 14:09",
			"2024-Jan-20 14:09",
			"2025-Jan-20 14:09",
		},
	},
}

// This test works by using a series of checkpoints, spaced <interval> apart.
// A job is scheduled 5 seconds after the first checkpoint.
// By moving the clock to each checkpoint, and then 6 seconds later,
// you can verify that the job hasn't run between the two checkpoints,
// and only runs at the scheduled point.
//
// This is useful for ensuring that durations behave correctly on a grand scale.
func TestRecur(t *testing.T) {

	for _, testStruct := range recurTableTests {

		func() {

			now := parseTimeInLocation(t, testStruct.Start, testStruct.Location)

			clk := clock.NewMockClock(now)

			start := now.Add(time.Second * 5)
			j := GetMockRecurringJobWithSchedule(start, testStruct.Interval)
			j.clk.SetClock(clk)
			j.ResumeAtNextScheduledTime = true // This is important to have on so that there's no drift.

			cache := NewMockCache()
			j.Init(cache)
			j.ranChan = make(chan struct{})

			checkpoints := append([]string{testStruct.Start}, testStruct.Checkpoints...)

			for i, chk := range checkpoints {

				clk.SetTime(parseTimeInLocation(t, chk, testStruct.Location))

				select {
				case <-j.ranChan:
					t.Fatalf("Expected job not run on checkpoint %d of test %s.", i, testStruct.Name)
				case <-time.After(time.Second):
				}

				j.lock.RLock()
				assert.Equal(t, i, int(j.Metadata.SuccessCount), fmt.Sprintf("1st Test of %s index %d", testStruct.Name, i))
				j.lock.RUnlock()

				clk.AddTime(time.Second * 6)

				select {
				case <-j.ranChan:
				case <-time.After(time.Second):
					t.Fatalf("Expected job to have run on checkpoint %d of test %s.", i, testStruct.Name)
				}

				j.lock.RLock()
				assert.Equal(t, i+1, int(j.Metadata.SuccessCount), fmt.Sprintf("2nd Test of %s index %d", testStruct.Name, i))
				j.lock.RUnlock()

				runtime.Gosched()
			}

		}()

	}

}
<commit_msg>Another attempt to handle the failing test.<commit_after>// Tests for recurring jobs firing appropriately.
package job

import (
	"fmt"
	"runtime"
	"testing"
	"time"

	"github.com/mixer/clock"
	"github.com/stretchr/testify/assert"
)

var recurTableTests = []struct {
	Name        string
	Location    string
	Start       string
	Interval    string
	Checkpoints []string
}{
	{
		Name:     "Daily",
		Location: "America/Los_Angeles",
		Start:    "2020-Jan-13 14:09",
		Interval: "P1D",
		Checkpoints: []string{
			"2020-Jan-14 14:09",
			"2020-Jan-15 14:09",
			"2020-Jan-16 14:09",
		},
	},
	{
		Name:     "Daily across DST boundary",
		Location: "America/Los_Angeles",
		Start:    "2020-Mar-05 14:09",
		Interval: "P1D",
		Checkpoints: []string{
			"2020-Mar-06 14:09",
			"2020-Mar-07 14:09",
			"2020-Mar-08 14:09",
			"2020-Mar-09 14:09",
		},
	},
	{
		Name:     "24 Hourly across DST boundary",
		Location: "America/Los_Angeles",
		Start:    "2020-Mar-05 14:09",
		Interval: "PT24H",
		Checkpoints: []string{
			"2020-Mar-06 14:09",
			"2020-Mar-07 14:09",
			"2020-Mar-08 15:09",
			"2020-Mar-09 15:09",
		},
	},
	{
		Name:     "Weekly",
		Location: "America/Los_Angeles",
		Start:    "2020-Jan-13 14:09",
		Interval: "P1W",
		Checkpoints: []string{
			"2020-Jan-20 14:09",
			"2020-Jan-27 14:09",
			"2020-Feb-03 14:09",
		},
	},
	{
		Name:     "Monthly",
		Location: "America/Los_Angeles",
		Start:    "2020-Jan-20 14:09",
		Interval: "P1M",
		Checkpoints: []string{
			"2020-Feb-20 14:09",
			"2020-Mar-20 14:09",
			"2020-Apr-20 14:09",
			"2020-May-20 14:09",
			"2020-Jun-20 14:09",
			"2020-Jul-20 14:09",
			"2020-Aug-20 14:09",
			"2020-Sep-20 14:09",
			"2020-Oct-20 14:09",
			"2020-Nov-20 14:09",
			"2020-Dec-20 14:09",
			"2021-Jan-20 14:09",
		},
	},
	{
		Name:     "Monthly with Normalization",
		Location: "America/Los_Angeles",
		Start:    "2020-Jul-31 14:09",
		Interval: "P1M",
		Checkpoints: []string{
			"2020-Aug-31 14:09",
			"2020-Oct-01 14:09",
			"2020-Nov-01 14:09",
		},
	},
	{
		Name:     "Yearly across Leap Year boundary",
		Location: "America/Los_Angeles",
		Start:    "2020-Jan-20 14:09",
		Interval: "P1Y",
		Checkpoints: []string{
			"2021-Jan-20 14:09",
			"2022-Jan-20 14:09",
			"2023-Jan-20 14:09",
			"2024-Jan-20 14:09",
			"2025-Jan-20 14:09",
		},
	},
}

// This test works by using a series of checkpoints, spaced <interval> apart.
// A job is scheduled 5 seconds after the first checkpoint.
// By moving the clock to each checkpoint, and then 6 seconds later,
// you can verify that the job hasn't run between the two checkpoints,
// and only runs at the scheduled point.
//
// This is useful for ensuring that durations behave correctly on a grand scale.
func TestRecur(t *testing.T) {

	for _, testStruct := range recurTableTests {

		func() {

			now := parseTimeInLocation(t, testStruct.Start, testStruct.Location)

			clk := clock.NewMockClock(now)

			start := now.Add(time.Second * 5)
			j := GetMockRecurringJobWithSchedule(start, testStruct.Interval)
			j.clk.SetClock(clk)
			j.ResumeAtNextScheduledTime = true // This is important to have on so that there's no drift.

			cache := NewMockCache()
			j.Init(cache)
			j.ranChan = make(chan struct{})

			checkpoints := append([]string{testStruct.Start}, testStruct.Checkpoints...)

			for i, chk := range checkpoints {

				clk.SetTime(parseTimeInLocation(t, chk, testStruct.Location))

				select {
				case <-j.ranChan:
					t.Fatalf("Expected job not run on checkpoint %d of test %s.", i, testStruct.Name)
				case <-time.After(time.Second * 2):
				}

				j.lock.RLock()
				assert.Equal(t, i, int(j.Metadata.SuccessCount), fmt.Sprintf("1st Test of %s index %d", testStruct.Name, i))
				j.lock.RUnlock()

				clk.AddTime(time.Second * 6)

				select {
				case <-j.ranChan:
				case <-time.After(time.Second * 2):
					t.Fatalf("Expected job to have run on checkpoint %d of test %s.", i, testStruct.Name)
				}

				j.lock.RLock()
				assert.Equal(t, i+1, int(j.Metadata.SuccessCount), fmt.Sprintf("2nd Test of %s index %d", testStruct.Name, i))
				j.lock.RUnlock()

				briefPause()
			}

		}()

	}

}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	r "github.com/dancannon/gorethink"
	"github.com/garyburd/redigo/redis"
	"log"
	"strings"
)

var (
	protected = []string{"list", "add", "api", "counter", "css", "img", "js"}
)

type Url struct {
	id     string
	Link   string
	Short  string
	Clicks int64
}

type SiteStats struct {
	Clicks       int
	Links        int
	ClicksPerUrl float64
}

func GetUrlById(id string, host string) (*Url, error) {
	DB := pool.Get()
	defer DB.Close()
	cr := UrlCache.Get(id)
	if cr != nil {
		log.Print("UrlCache: Cache HIT!")
		log.Print("Updating click count in goroutine")
		go UpdateClickCount(id)
		return cr.(*Url), nil
	}
	log.Print("UrlCache: Cache Miss, retrieving from DB")
	id = strings.Split(id, ":")[0]
	k, err := DB.Do("GET", "url:link:"+id)
	if err != nil {
		return nil, err
	}
	switch k.(type) {
	case nil:
		return nil, nil
	default:
		c, _ := UpdateClickCount(id)
		resp := &Url{}
		resp.id = id
		resp.Short = config.GetBaseUrl(host) + id
		resp.Link, _ = redis.String(k, err)
		resp.Clicks = int64(c)
		UrlCache.Set(id, resp)
		return resp, nil
	}
}

func GetNewUrl(link string, host string) (*Url, error) {
	DB := pool.Get()
	defer DB.Close()
	i, err := GetNewID()
	if err != nil {
		return nil, err
	}
	for _, k := range protected {
		for b62_Encode(uint64(i)) == k {
			i, err = GetNewID()
			if err != nil {
				return nil, err
			}
		}
	}
	pos := b62_Encode(uint64(i))
	_, err = DB.Do("SET", "url:link:"+pos, link)
	if err != nil {
		return nil, err
	}
	go func(pos string) {
		d := pool.Get()
		defer d.Close()
		_, err := d.Do("SET", "url:clicks:"+pos, 0)
		if err != nil {
			log.Printf("Error setting %s clicks to 0", pos)
		} else {
			log.Printf("%s clicks set to 0", pos)
		}
	}(pos)
	new := &Url{}
	new.id = pos
	new.Link = link
	new.Clicks = 0
	new.Short = config.GetBaseUrl(host) + new.id
	UrlCache.Set(new.id, new)
	log.Printf("Shortened %s to %s", new.Link, config.GetBaseUrl(host)+new.id)
	return new, nil
}

func GetNewID() (int64, error) {
	var target interface{}
	err := r.Table("meta").Get("counter").Update(map[string]interface{}{"value": r.Row.Field("value").Add(1)}).Exec(session)
	if err != nil {
		return 0, err
	}
	cursor, err := r.Table("meta").Get("counter").Field("value").Run(session)
	if err != nil {
		return 0, err
	}
	cursor.One(&target)
	if cursor.Err() != nil {
		return 0, cursor.Err()
	}
	final, ok := target.(float64)
	if !ok {
		return 0, errors.New("Cannot convert counter to float64")
	}
	return int64(final), nil
}

func GetSiteStats() SiteStats {
	cc := StatsCache.Get("Stats")
	if cc != nil {
		log.Print("Cache: Site Stats HIT")
		return cc.(SiteStats)
	} else {
		log.Print("Cache: Site Stats MISS")
	}
	k := SiteStats{}
	a, _ := GetTotalClicks()
	b, _ := GetTotalUrls()
	c, _ := GetClicksPerUrl()
	k.Clicks = a
	k.Links = b
	k.ClicksPerUrl = c
	StatsCache.Set("Stats", k)
	return k
}

func newPool() *redis.Pool {
	return redis.NewPool(func() (redis.Conn, error) {
		conn, err := redis.Dial("tcp", config.DBAddress)
		if err != nil {
			return nil, err
		}
		_, err = conn.Do("AUTH", config.DBPassword)
		if err != nil {
			log.Print(err.Error())
			return nil, err
		}
		return conn, nil

	}, 3)
}

func GetTotalUrls() (int, error) {
	db := pool.Get()
	defer db.Close()
	l, err := redis.Int(db.Do("GET", "meta:total:links"))
	if err != nil {
		return 0, err
	}
	return l, err
}

func GetTotalClicks() (int, error) {
	DB := pool.Get()
	defer DB.Close()
	j, err := redis.Int(DB.Do("GET", "meta:total:clicks"))
	if err != nil {
		return 0, nil
	}
	return j, nil
}

func UpdateClickCount(id string) (int, error) {
	DB := pool.Get()
	defer DB.Close()
	k, err := DB.Do("INCR", "url:clicks:"+id)
	i, err := redis.Int(k, err)
	return i, err
}
<commit_msg>convert GetTotalURLS to rethinkdb<commit_after>package main

import (
	"errors"
	r "github.com/dancannon/gorethink"
	"github.com/garyburd/redigo/redis"
	"log"
	"strings"
)

var (
	protected = []string{"list", "add", "api", "counter", "css", "img", "js"}
)

type Url struct {
	id     string
	Link   string
	Short  string
	Clicks int64
}

type SiteStats struct {
	Clicks       int
	Links        int
	ClicksPerUrl float64
}

func GetUrlById(id string, host string) (*Url, error) {
	DB := pool.Get()
	defer DB.Close()
	cr := UrlCache.Get(id)
	if cr != nil {
		log.Print("UrlCache: Cache HIT!")
		log.Print("Updating click count in goroutine")
		go UpdateClickCount(id)
		return cr.(*Url), nil
	}
	log.Print("UrlCache: Cache Miss, retrieving from DB")
	id = strings.Split(id, ":")[0]
	k, err := DB.Do("GET", "url:link:"+id)
	if err != nil {
		return nil, err
	}
	switch k.(type) {
	case nil:
		return nil, nil
	default:
		c, _ := UpdateClickCount(id)
		resp := &Url{}
		resp.id = id
		resp.Short = config.GetBaseUrl(host) + id
		resp.Link, _ = redis.String(k, err)
		resp.Clicks = int64(c)
		UrlCache.Set(id, resp)
		return resp, nil
	}
}

func GetNewUrl(link string, host string) (*Url, error) {
	DB := pool.Get()
	defer DB.Close()
	i, err := GetNewID()
	if err != nil {
		return nil, err
	}
	for _, k := range protected {
		for b62_Encode(uint64(i)) == k {
			i, err = GetNewID()
			if err != nil {
				return nil, err
			}
		}
	}
	pos := b62_Encode(uint64(i))
	_, err = DB.Do("SET", "url:link:"+pos, link)
	if err != nil {
		return nil, err
	}
	go func(pos string) {
		d := pool.Get()
		defer d.Close()
		_, err := d.Do("SET", "url:clicks:"+pos, 0)
		if err != nil {
			log.Printf("Error setting %s clicks to 0", pos)
		} else {
			log.Printf("%s clicks set to 0", pos)
		}
	}(pos)
	new := &Url{}
	new.id = pos
	new.Link = link
	new.Clicks = 0
	new.Short = config.GetBaseUrl(host) + new.id
	UrlCache.Set(new.id, new)
	log.Printf("Shortened %s to %s", new.Link, config.GetBaseUrl(host)+new.id)
	return new, nil
}

func GetNewID() (int64, error) {
	var target interface{}
	err := r.Table("meta").Get("counter").Update(map[string]interface{}{"value": r.Row.Field("value").Add(1)}).Exec(session)
	if err != nil {
		return 0, err
	}
	cursor, err := r.Table("meta").Get("counter").Field("value").Run(session)
	if err != nil {
		return 0, err
	}
	cursor.One(&target)
	if cursor.Err() != nil {
		return 0, cursor.Err()
	}
	final, ok := target.(float64)
	if !ok {
		return 0, errors.New("Cannot convert counter to float64")
	}
	return int64(final), nil
}

func GetSiteStats() SiteStats {
	cc := StatsCache.Get("Stats")
	if cc != nil {
		log.Print("Cache: Site Stats HIT")
		return cc.(SiteStats)
	} else {
		log.Print("Cache: Site Stats MISS")
	}
	k := SiteStats{}
	a, _ := GetTotalClicks()
	b, _ := GetTotalUrls()
	c, _ := GetClicksPerUrl()
	k.Clicks = a
	k.Links = b
	k.ClicksPerUrl = c
	StatsCache.Set("Stats", k)
	return k
}

func newPool() *redis.Pool {
	return redis.NewPool(func() (redis.Conn, error) {
		conn, err := redis.Dial("tcp", config.DBAddress)
		if err != nil {
			return nil, err
		}
		_, err = conn.Do("AUTH", config.DBPassword)
		if err != nil {
			log.Print(err.Error())
			return nil, err
		}
		return conn, nil

	}, 3)
}

func GetTotalUrls() (int, error) {
	var target interface{}
	cursor, err := r.Table("meta").Get("total_links").Field("value").Run(session)
	if err != nil {
		return 0, err
	}
	cursor.One(&target)
	if cursor.Err() != nil {
		return 0, cursor.Err()
	}
	result, ok := target.(float64)
	if !ok {
		return 0, errors.New("meta.total_links is not a float64")
	}
	return int(result), nil
}

func GetTotalClicks() (int, error) {
	DB := pool.Get()
	defer DB.Close()
	j, err := redis.Int(DB.Do("GET", "meta:total:clicks"))
	if err != nil {
		return 0, nil
	}
	return j, nil
}

func UpdateClickCount(id string) (int, error) {
	DB := pool.Get()
	defer DB.Close()
	k, err := DB.Do("INCR", "url:clicks:"+id)
	i, err := redis.Int(k, err)
	return i, err
}
<|endoftext|>


<commit_before>package multiaddr

import (
	"bytes"
	"encoding/base32"
	"encoding/base64"
	"encoding/binary"
	"fmt"
	"net"
	"strconv"
	"strings"

	"github.com/ipfs/go-cid"
	mh "github.com/multiformats/go-multihash"
)

type Transcoder interface {
	// Validates and encodes to bytes a multiaddr that's in the string representation.
	StringToBytes(string) ([]byte, error)
	// Validates and decodes to a string a multiaddr that's in the bytes representation.
	BytesToString([]byte) (string, error)
	// Validates bytes when parsing a multiaddr that's already in the bytes representation.
	ValidateBytes([]byte) error
}

func NewTranscoderFromFunctions(
	s2b func(string) ([]byte, error),
	b2s func([]byte) (string, error),
	val func([]byte) error,
) Transcoder {
	return twrp{s2b, b2s, val}
}

type twrp struct {
	strtobyte func(string) ([]byte, error)
	bytetostr func([]byte) (string, error)
	validbyte func([]byte) error
}

func (t twrp) StringToBytes(s string) ([]byte, error) {
	return t.strtobyte(s)
}
func (t twrp) BytesToString(b []byte) (string, error) {
	return t.bytetostr(b)
}

func (t twrp) ValidateBytes(b []byte) error {
	if t.validbyte == nil {
		return nil
	}
	return t.validbyte(b)
}

var TranscoderIP4 = NewTranscoderFromFunctions(ip4StB, ip4BtS, nil)
var TranscoderIP6 = NewTranscoderFromFunctions(ip6StB, ip6BtS, nil)
var TranscoderIP6Zone = NewTranscoderFromFunctions(ip6zoneStB, ip6zoneBtS, ip6zoneVal)

func ip4StB(s string) ([]byte, error) {
	i := net.ParseIP(s).To4()
	if i == nil {
		return nil, fmt.Errorf("failed to parse ip4 addr: %s", s)
	}
	return i, nil
}

func ip6zoneStB(s string) ([]byte, error) {
	if len(s) == 0 {
		return nil, fmt.Errorf("empty ip6zone")
	}
	if strings.Contains(s, "/") {
		return nil, fmt.Errorf("IPv6 zone ID contains '/': %s", s)
	}
	return []byte(s), nil
}

func ip6zoneBtS(b []byte) (string, error) {
	if len(b) == 0 {
		return "", fmt.Errorf("invalid length (should be > 0)")
	}
	return string(b), nil
}

func ip6zoneVal(b []byte) error {
	if len(b) == 0 {
		return fmt.Errorf("invalid length (should be > 0)")
	}
	// Not supported as this would break multiaddrs.
	if bytes.IndexByte(b, '/') >= 0 {
		return fmt.Errorf("IPv6 zone ID contains '/': %s", string(b))
	}
	return nil
}

func ip6StB(s string) ([]byte, error) {
	i := net.ParseIP(s).To16()
	if i == nil {
		return nil, fmt.Errorf("failed to parse ip6 addr: %s", s)
	}
	return i, nil
}

func ip6BtS(b []byte) (string, error) {
	ip := net.IP(b)
	if ip4 := ip.To4(); ip4 != nil {
		// Go fails to prepend the `::ffff:` part.
		return "::ffff:" + ip4.String(), nil
	}
	return ip.String(), nil
}

func ip4BtS(b []byte) (string, error) {
	return net.IP(b).String(), nil
}

var TranscoderPort = NewTranscoderFromFunctions(portStB, portBtS, nil)

func portStB(s string) ([]byte, error) {
	i, err := strconv.Atoi(s)
	if err != nil {
		return nil, fmt.Errorf("failed to parse port addr: %s", err)
	}
	if i >= 65536 {
		return nil, fmt.Errorf("failed to parse port addr: %s", "greater than 65536")
	}
	b := make([]byte, 2)
	binary.BigEndian.PutUint16(b, uint16(i))
	return b, nil
}

func portBtS(b []byte) (string, error) {
	i := binary.BigEndian.Uint16(b)
	return strconv.Itoa(int(i)), nil
}

var TranscoderOnion = NewTranscoderFromFunctions(onionStB, onionBtS, nil)

func onionStB(s string) ([]byte, error) {
	addr := strings.Split(s, ":")
	if len(addr) != 2 {
		return nil, fmt.Errorf("failed to parse onion addr: %s does not contain a port number.", s)
	}

	// onion address without the ".onion" substring
	if len(addr[0]) != 16 {
		return nil, fmt.Errorf("failed to parse onion addr: %s not a Tor onion address.", s)
	}
	onionHostBytes, err := base32.StdEncoding.DecodeString(strings.ToUpper(addr[0]))
	if err != nil {
		return nil, fmt.Errorf("failed to decode base32 onion addr: %s %s", s, err)
	}

	// onion port number
	i, err := strconv.Atoi(addr[1])
	if err != nil {
		return nil, fmt.Errorf("failed to parse onion addr: %s", err)
	}
	if i >= 65536 {
		return nil, fmt.Errorf("failed to parse onion addr: %s", "port greater than 65536")
	}
	if i < 1 {
		return nil, fmt.Errorf("failed to parse onion addr: %s", "port less than 1")
	}

	onionPortBytes := make([]byte, 2)
	binary.BigEndian.PutUint16(onionPortBytes, uint16(i))
	bytes := []byte{}
	bytes = append(bytes, onionHostBytes...)
	bytes = append(bytes, onionPortBytes...)
	return bytes, nil
}

func onionBtS(b []byte) (string, error) {
	addr := strings.ToLower(base32.StdEncoding.EncodeToString(b[0:10]))
	port := binary.BigEndian.Uint16(b[10:12])
	return addr + ":" + strconv.Itoa(int(port)), nil
}

var TranscoderOnion3 = NewTranscoderFromFunctions(onion3StB, onion3BtS, nil)

func onion3StB(s string) ([]byte, error) {
	addr := strings.Split(s, ":")
	if len(addr) != 2 {
		return nil, fmt.Errorf("failed to parse onion addr: %s does not contain a port number.", s)
	}

	// onion address without the ".onion" substring
	if len(addr[0]) != 56 {
		return nil, fmt.Errorf("failed to parse onion addr: %s not a Tor onionv3 address. len == %d", s, len(addr[0]))
	}
	onionHostBytes, err := base32.StdEncoding.DecodeString(strings.ToUpper(addr[0]))
	if err != nil {
		return nil, fmt.Errorf("failed to decode base32 onion addr: %s %s", s, err)
	}

	// onion port number
	i, err := strconv.Atoi(addr[1])
	if err != nil {
		return nil, fmt.Errorf("failed to parse onion addr: %s", err)
	}
	if i >= 65536 {
		return nil, fmt.Errorf("failed to parse onion addr: %s", "port greater than 65536")
	}
	if i < 1 {
		return nil, fmt.Errorf("failed to parse onion addr: %s", "port less than 1")
	}

	onionPortBytes := make([]byte, 2)
	binary.BigEndian.PutUint16(onionPortBytes, uint16(i))
	bytes := []byte{}
	bytes = append(bytes, onionHostBytes[0:35]...)
	bytes = append(bytes, onionPortBytes...)
	return bytes, nil
}

func onion3BtS(b []byte) (string, error) {
	addr := strings.ToLower(base32.StdEncoding.EncodeToString(b[0:35]))
	port := binary.BigEndian.Uint16(b[35:37])
	str := addr + ":" + strconv.Itoa(int(port))
	return str, nil
}

var TranscoderGarlic64 = NewTranscoderFromFunctions(garlic64StB, garlic64BtS, garlic64Validate)

// i2p uses an alternate character set for base64 addresses. This returns an appropriate encoder.
var garlicBase64Encoding = base64.NewEncoding("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-~")

func garlic64StB(s string) ([]byte, error) {
	// i2p base64 address will be between 516 and 616 characters long, depending on
	// certificate type
	if len(s) < 516 || len(s) > 616 {
		return nil, fmt.Errorf("failed to parse garlic addr: %s not an i2p base64 address. len: %d\n", s, len(s))
	}
	garlicHostBytes, err := garlicBase64Encoding.DecodeString(s)
	if err != nil {
		return nil, fmt.Errorf("failed to decode base64 i2p addr: %s %s", s, err)
	}

	return garlicHostBytes, nil
}

func garlic64BtS(b []byte) (string, error) {
	if err := garlic64Validate(b); err != nil {
		return "", err
	}
	addr := garlicBase64Encoding.EncodeToString(b)
	return addr, nil
}

func garlic64Validate(b []byte) error {
	// A garlic64 address will always be greater than 386 bytes long when encoded.
	if len(b) < 386 {
		return fmt.Errorf("failed to validate garlic addr: %s not an i2p base64 address. len: %d\n", b, len(b))
	}
	return nil
}

var TranscoderGarlic32 = NewTranscoderFromFunctions(garlic32StB, garlic32BtS, garlic32Validate)

var garlicBase32Encoding = base32.NewEncoding("abcdefghijklmnopqrstuvwxyz234567")

func garlic32StB(s string) ([]byte, error) {
	// an i2p base32 address with a length of greater than 55 characters is
	// using an Encrypted Leaseset v2. all other base32 addresses will always be
	// exactly 52 characters
	if len(s) < 55 && len(s) != 52 {
		return nil, fmt.Errorf("failed to parse garlic addr: %s not a i2p base32 address. len: %d", s, len(s))
	}
	for len(s)%8 != 0 {
		s += "="
	}
	garlicHostBytes, err := garlicBase32Encoding.DecodeString(s)
	if err != nil {
		return nil, fmt.Errorf("failed to decode base32 garlic addr: %s, err: %v len: %v", s, err, len(s))
	}
	return garlicHostBytes, nil
}

func garlic32BtS(b []byte) (string, error) {
	if err := garlic32Validate(b); err != nil {
		return "", err
	}
	return strings.TrimRight(garlicBase32Encoding.EncodeToString(b), "="), nil
}

func garlic32Validate(b []byte) error {
	// an i2p base64 for an Encrypted Leaseset v2 will be at least 35 bytes
	// long other than that, they will be exactly 32 bytes
	if len(b) < 35 && len(b) != 32 {
		return fmt.Errorf("failed to validate garlic addr: %s not an i2p base32 address. len: %d\n", b, len(b))
	}
	return nil
}

var TranscoderP2P = NewTranscoderFromFunctions(p2pStB, p2pBtS, p2pVal)

// The encoded peer ID can either be a CID of a key or a raw multihash (identity
// or sha256-256).
func p2pStB(s string) ([]byte, error) {
	// check if the address is a base58 encoded sha256 or identity multihash
	if strings.HasPrefix(s, "Qm") || strings.HasPrefix(s, "1") {
		m, err := mh.FromB58String(s)
		if err != nil {
			return nil, fmt.Errorf("failed to parse p2p addr: %s %s", s, err)
		}
		return m, nil
	}

	// check if the address is a CID
	c, err := cid.Decode(s)
	if err != nil {
		return nil, fmt.Errorf("failed to parse p2p addr: %s %s", s, err)
	}

	if ty := c.Type(); ty == cid.Libp2pKey {
		return c.Hash(), nil
	} else {
		return nil, fmt.Errorf("failed to parse p2p addr: %s has the invalid codec %d", s, ty)
	}
}

func p2pVal(b []byte) error {
	_, err := mh.Cast(b)
	return err
}

func p2pBtS(b []byte) (string, error) {
	m, err := mh.Cast(b)
	if err != nil {
		return "", err
	}
	return m.B58String(), nil
}

var TranscoderUnix = NewTranscoderFromFunctions(unixStB, unixBtS, nil)

func unixStB(s string) ([]byte, error) {
	return []byte(s), nil
}

func unixBtS(b []byte) (string, error) {
	return string(b), nil
}

var TranscoderDns = NewTranscoderFromFunctions(dnsStB, dnsBtS, dnsVal)

func dnsVal(b []byte) error {
	if bytes.IndexByte(b, '/') >= 0 {
		return fmt.Errorf("domain name %q contains a slash", string(b))
	}
	return nil
}

func dnsStB(s string) ([]byte, error) {
	return []byte(s), nil
}

func dnsBtS(b []byte) (string, error) {
	return string(b), nil
}
<commit_msg>fix error strings<commit_after>package multiaddr

import (
	"bytes"
	"encoding/base32"
	"encoding/base64"
	"encoding/binary"
	"fmt"
	"net"
	"strconv"
	"strings"

	"github.com/ipfs/go-cid"
	mh "github.com/multiformats/go-multihash"
)

type Transcoder interface {
	// Validates and encodes to bytes a multiaddr that's in the string representation.
	StringToBytes(string) ([]byte, error)
	// Validates and decodes to a string a multiaddr that's in the bytes representation.
	BytesToString([]byte) (string, error)
	// Validates bytes when parsing a multiaddr that's already in the bytes representation.
	ValidateBytes([]byte) error
}

func NewTranscoderFromFunctions(
	s2b func(string) ([]byte, error),
	b2s func([]byte) (string, error),
	val func([]byte) error,
) Transcoder {
	return twrp{s2b, b2s, val}
}

type twrp struct {
	strtobyte func(string) ([]byte, error)
	bytetostr func([]byte) (string, error)
	validbyte func([]byte) error
}

func (t twrp) StringToBytes(s string) ([]byte, error) {
	return t.strtobyte(s)
}
func (t twrp) BytesToString(b []byte) (string, error) {
	return t.bytetostr(b)
}

func (t twrp) ValidateBytes(b []byte) error {
	if t.validbyte == nil {
		return nil
	}
	return t.validbyte(b)
}

var TranscoderIP4 = NewTranscoderFromFunctions(ip4StB, ip4BtS, nil)
var TranscoderIP6 = NewTranscoderFromFunctions(ip6StB, ip6BtS, nil)
var TranscoderIP6Zone = NewTranscoderFromFunctions(ip6zoneStB, ip6zoneBtS, ip6zoneVal)

func ip4StB(s string) ([]byte, error) {
	i := net.ParseIP(s).To4()
	if i == nil {
		return nil, fmt.Errorf("failed to parse ip4 addr: %s", s)
	}
	return i, nil
}

func ip6zoneStB(s string) ([]byte, error) {
	if len(s) == 0 {
		return nil, fmt.Errorf("empty ip6zone")
	}
	if strings.Contains(s, "/") {
		return nil, fmt.Errorf("IPv6 zone ID contains '/': %s", s)
	}
	return []byte(s), nil
}

func ip6zoneBtS(b []byte) (string, error) {
	if len(b) == 0 {
		return "", fmt.Errorf("invalid length (should be > 0)")
	}
	return string(b), nil
}

func ip6zoneVal(b []byte) error {
	if len(b) == 0 {
		return fmt.Errorf("invalid length (should be > 0)")
	}
	// Not supported as this would break multiaddrs.
	if bytes.IndexByte(b, '/') >= 0 {
		return fmt.Errorf("IPv6 zone ID contains '/': %s", string(b))
	}
	return nil
}

func ip6StB(s string) ([]byte, error) {
	i := net.ParseIP(s).To16()
	if i == nil {
		return nil, fmt.Errorf("failed to parse ip6 addr: %s", s)
	}
	return i, nil
}

func ip6BtS(b []byte) (string, error) {
	ip := net.IP(b)
	if ip4 := ip.To4(); ip4 != nil {
		// Go fails to prepend the `::ffff:` part.
		return "::ffff:" + ip4.String(), nil
	}
	return ip.String(), nil
}

func ip4BtS(b []byte) (string, error) {
	return net.IP(b).String(), nil
}

var TranscoderPort = NewTranscoderFromFunctions(portStB, portBtS, nil)

func portStB(s string) ([]byte, error) {
	i, err := strconv.Atoi(s)
	if err != nil {
		return nil, fmt.Errorf("failed to parse port addr: %s", err)
	}
	if i >= 65536 {
		return nil, fmt.Errorf("failed to parse port addr: %s", "greater than 65536")
	}
	b := make([]byte, 2)
	binary.BigEndian.PutUint16(b, uint16(i))
	return b, nil
}

func portBtS(b []byte) (string, error) {
	i := binary.BigEndian.Uint16(b)
	return strconv.Itoa(int(i)), nil
}

var TranscoderOnion = NewTranscoderFromFunctions(onionStB, onionBtS, nil)

func onionStB(s string) ([]byte, error) {
	addr := strings.Split(s, ":")
	if len(addr) != 2 {
		return nil, fmt.Errorf("failed to parse onion addr: %s does not contain a port number", s)
	}

	// onion address without the ".onion" substring
	if len(addr[0]) != 16 {
		return nil, fmt.Errorf("failed to parse onion addr: %s not a Tor onion address", s)
	}
	onionHostBytes, err := base32.StdEncoding.DecodeString(strings.ToUpper(addr[0]))
	if err != nil {
		return nil, fmt.Errorf("failed to decode base32 onion addr: %s %s", s, err)
	}

	// onion port number
	i, err := strconv.Atoi(addr[1])
	if err != nil {
		return nil, fmt.Errorf("failed to parse onion addr: %s", err)
	}
	if i >= 65536 {
		return nil, fmt.Errorf("failed to parse onion addr: %s", "port greater than 65536")
	}
	if i < 1 {
		return nil, fmt.Errorf("failed to parse onion addr: %s", "port less than 1")
	}

	onionPortBytes := make([]byte, 2)
	binary.BigEndian.PutUint16(onionPortBytes, uint16(i))
	bytes := []byte{}
	bytes = append(bytes, onionHostBytes...)
	bytes = append(bytes, onionPortBytes...)
	return bytes, nil
}

func onionBtS(b []byte) (string, error) {
	addr := strings.ToLower(base32.StdEncoding.EncodeToString(b[0:10]))
	port := binary.BigEndian.Uint16(b[10:12])
	return addr + ":" + strconv.Itoa(int(port)), nil
}

var TranscoderOnion3 = NewTranscoderFromFunctions(onion3StB, onion3BtS, nil)

func onion3StB(s string) ([]byte, error) {
	addr := strings.Split(s, ":")
	if len(addr) != 2 {
		return nil, fmt.Errorf("failed to parse onion addr: %s does not contain a port number", s)
	}

	// onion address without the ".onion" substring
	if len(addr[0]) != 56 {
		return nil, fmt.Errorf("failed to parse onion addr: %s not a Tor onionv3 address. len == %d", s, len(addr[0]))
	}
	onionHostBytes, err := base32.StdEncoding.DecodeString(strings.ToUpper(addr[0]))
	if err != nil {
		return nil, fmt.Errorf("failed to decode base32 onion addr: %s %s", s, err)
	}

	// onion port number
	i, err := strconv.Atoi(addr[1])
	if err != nil {
		return nil, fmt.Errorf("failed to parse onion addr: %s", err)
	}
	if i >= 65536 {
		return nil, fmt.Errorf("failed to parse onion addr: %s", "port greater than 65536")
	}
	if i < 1 {
		return nil, fmt.Errorf("failed to parse onion addr: %s", "port less than 1")
	}

	onionPortBytes := make([]byte, 2)
	binary.BigEndian.PutUint16(onionPortBytes, uint16(i))
	bytes := []byte{}
	bytes = append(bytes, onionHostBytes[0:35]...)
	bytes = append(bytes, onionPortBytes...)
	return bytes, nil
}

func onion3BtS(b []byte) (string, error) {
	addr := strings.ToLower(base32.StdEncoding.EncodeToString(b[0:35]))
	port := binary.BigEndian.Uint16(b[35:37])
	str := addr + ":" + strconv.Itoa(int(port))
	return str, nil
}

var TranscoderGarlic64 = NewTranscoderFromFunctions(garlic64StB, garlic64BtS, garlic64Validate)

// i2p uses an alternate character set for base64 addresses. This returns an appropriate encoder.
var garlicBase64Encoding = base64.NewEncoding("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-~")

func garlic64StB(s string) ([]byte, error) {
	// i2p base64 address will be between 516 and 616 characters long, depending on
	// certificate type
	if len(s) < 516 || len(s) > 616 {
		return nil, fmt.Errorf("failed to parse garlic addr: %s not an i2p base64 address. len: %d", s, len(s))
	}
	garlicHostBytes, err := garlicBase64Encoding.DecodeString(s)
	if err != nil {
		return nil, fmt.Errorf("failed to decode base64 i2p addr: %s %s", s, err)
	}

	return garlicHostBytes, nil
}

func garlic64BtS(b []byte) (string, error) {
	if err := garlic64Validate(b); err != nil {
		return "", err
	}
	addr := garlicBase64Encoding.EncodeToString(b)
	return addr, nil
}

func garlic64Validate(b []byte) error {
	// A garlic64 address will always be greater than 386 bytes long when encoded.
	if len(b) < 386 {
		return fmt.Errorf("failed to validate garlic addr: %s not an i2p base64 address. len: %d", b, len(b))
	}
	return nil
}

var TranscoderGarlic32 = NewTranscoderFromFunctions(garlic32StB, garlic32BtS, garlic32Validate)

var garlicBase32Encoding = base32.NewEncoding("abcdefghijklmnopqrstuvwxyz234567")

func garlic32StB(s string) ([]byte, error) {
	// an i2p base32 address with a length of greater than 55 characters is
	// using an Encrypted Leaseset v2. all other base32 addresses will always be
	// exactly 52 characters
	if len(s) < 55 && len(s) != 52 {
		return nil, fmt.Errorf("failed to parse garlic addr: %s not a i2p base32 address. len: %d", s, len(s))
	}
	for len(s)%8 != 0 {
		s += "="
	}
	garlicHostBytes, err := garlicBase32Encoding.DecodeString(s)
	if err != nil {
		return nil, fmt.Errorf("failed to decode base32 garlic addr: %s, err: %v len: %v", s, err, len(s))
	}
	return garlicHostBytes, nil
}

func garlic32BtS(b []byte) (string, error) {
	if err := garlic32Validate(b); err != nil {
		return "", err
	}
	return strings.TrimRight(garlicBase32Encoding.EncodeToString(b), "="), nil
}

func garlic32Validate(b []byte) error {
	// an i2p base64 for an Encrypted Leaseset v2 will be at least 35 bytes
	// long other than that, they will be exactly 32 bytes
	if len(b) < 35 && len(b) != 32 {
		return fmt.Errorf("failed to validate garlic addr: %s not an i2p base32 address. len: %d", b, len(b))
	}
	return nil
}

var TranscoderP2P = NewTranscoderFromFunctions(p2pStB, p2pBtS, p2pVal)

// The encoded peer ID can either be a CID of a key or a raw multihash (identity
// or sha256-256).
func p2pStB(s string) ([]byte, error) {
	// check if the address is a base58 encoded sha256 or identity multihash
	if strings.HasPrefix(s, "Qm") || strings.HasPrefix(s, "1") {
		m, err := mh.FromB58String(s)
		if err != nil {
			return nil, fmt.Errorf("failed to parse p2p addr: %s %s", s, err)
		}
		return m, nil
	}

	// check if the address is a CID
	c, err := cid.Decode(s)
	if err != nil {
		return nil, fmt.Errorf("failed to parse p2p addr: %s %s", s, err)
	}

	if ty := c.Type(); ty == cid.Libp2pKey {
		return c.Hash(), nil
	} else {
		return nil, fmt.Errorf("failed to parse p2p addr: %s has the invalid codec %d", s, ty)
	}
}

func p2pVal(b []byte) error {
	_, err := mh.Cast(b)
	return err
}

func p2pBtS(b []byte) (string, error) {
	m, err := mh.Cast(b)
	if err != nil {
		return "", err
	}
	return m.B58String(), nil
}

var TranscoderUnix = NewTranscoderFromFunctions(unixStB, unixBtS, nil)

func unixStB(s string) ([]byte, error) {
	return []byte(s), nil
}

func unixBtS(b []byte) (string, error) {
	return string(b), nil
}

var TranscoderDns = NewTranscoderFromFunctions(dnsStB, dnsBtS, dnsVal)

func dnsVal(b []byte) error {
	if bytes.IndexByte(b, '/') >= 0 {
		return fmt.Errorf("domain name %q contains a slash", string(b))
	}
	return nil
}

func dnsStB(s string) ([]byte, error) {
	return []byte(s), nil
}

func dnsBtS(b []byte) (string, error) {
	return string(b), nil
}
<|endoftext|>


<commit_before>package socket

import (
    "time"
    "net/http"
    "github.com/gorilla/websocket"
    "app/hub"
    "app/message"
)

var upgrader = websocket.Upgrader{}

// Handler handles websocket connections at /ws
func Handler(w http.ResponseWriter, r *http.Request) {
    c, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
        panic(err)
	}
	defer c.Close()

    received := make(chan message.SocketMessage)

    // all messages pushed to the 'received' channel are written out to the socket
    go writeSocket(c, received)

    // handle all incoming messages from the socket
    for {
        message := message.SocketMessage{}
        message.CreatedAt = time.Now().UTC()

		err := c.ReadJSON(&message)
		if err != nil {
			panic(err)
		}

        if message.Action == "publish" {
            hub.Published <- message
        }

        if message.Action == "subscribe" {
            hub.Subscribed[message.Event] = append(hub.Subscribed[message.Event], received)
        }
	}

}
<commit_msg>Notes<commit_after>package socket

import (
    "time"
    "net/http"
    "github.com/gorilla/websocket"
    "app/hub"
    "app/message"
)

var upgrader = websocket.Upgrader{}

// Handler handles websocket connections at /ws
func Handler(w http.ResponseWriter, r *http.Request) {
    c, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
        panic(err)
	}
	defer c.Close()

    // each socket connection has a 'received' channel
    received := make(chan message.SocketMessage)

    // all messages pushed to the 'received' channel
    // are written out to the socket
    go writeSocket(c, received)

    // read incoming messages from the socket
    for {
        message := message.SocketMessage{}
        message.CreatedAt = time.Now().UTC()

		err := c.ReadJSON(&message)
		if err != nil {
			panic(err)
		}

        if message.Action == "publish" {
            hub.Published <- message
        }

        if message.Action == "subscribe" {
            hub.Subscribed[message.Event] = append(hub.Subscribed[message.Event], received)
        }
	}

}
<|endoftext|>


<commit_before>package provision

import (
	"bytes"
	"errors"
	"fmt"
	"log"
	"strings"
	"time"

	docker "github.com/fsouza/go-dockerclient"
	"github.com/nuveo/gofn/iaas"
)

var (
	// ErrImageNotFound is raised when image is not found
	ErrImageNotFound = errors.New("provision: image not found")

	// ErrContainerNotFound is raised when image is not found
	ErrContainerNotFound = errors.New("provision: container not found")
)

// VolumeOptions are options to mount a host directory as data volume
type VolumeOptions struct {
	Source      string
	Destination string
}

// BuildOptions are options used in the image build
type BuildOptions struct {
	ContextDir string
	Dockerfile string
	ImageName  string
	RemoteURI  string
	Iaas       iaas.Iaas
}

// FnClient instantiate a docker client
func FnClient(endPoint string) (client *docker.Client) {
	if endPoint == "" {
		endPoint = "unix:///var/run/docker.sock"
	}

	client, err := docker.NewClient(endPoint)
	if err != nil {
		panic(err)
	}
	return
}

// FnRemove remove container
func FnRemove(client *docker.Client, containerID string) (err error) {
	err = client.RemoveContainer(docker.RemoveContainerOptions{ID: containerID, Force: true})
	return
}

// FnContainer create container
func FnContainer(client *docker.Client, image, volume string) (container *docker.Container, err error) {
	t := time.Now()
	binds := []string{}
	if volume != "" {
		binds = append(binds, volume)
	}
	container, err = client.CreateContainer(docker.CreateContainerOptions{
		Name:       fmt.Sprintf("gofn-%s", t.Format("20060102150405")),
		HostConfig: &docker.HostConfig{Binds: binds},
		Config: &docker.Config{
			Image:     image,
			StdinOnce: true,
			OpenStdin: true,
		},
	})
	return
}

// FnImageBuild builds an image
func FnImageBuild(client *docker.Client, opts *BuildOptions) (Name string, Stdout *bytes.Buffer) {
	if opts.Dockerfile == "" {
		opts.Dockerfile = "Dockerfile"
	}
	stdout := new(bytes.Buffer)
	Name = "gofn/" + opts.ImageName
	err := client.BuildImage(docker.BuildImageOptions{
		Name:           Name,
		Dockerfile:     opts.Dockerfile,
		SuppressOutput: true,
		OutputStream:   stdout,
		ContextDir:     opts.ContextDir,
		Remote:         opts.RemoteURI,
	})
	if err != nil {
		panic(err)
	}
	Stdout = stdout
	return
}

// FnFindImage returns image data by name
func FnFindImage(client *docker.Client, imageName string) (image docker.APIImages, err error) {
	var imgs []docker.APIImages
	name := "gofn/" + imageName

	imgs, err = client.ListImages(docker.ListImagesOptions{Filter: name})
	if err != nil {
		return
	}

	if len(imgs) == 0 {
		err = ErrImageNotFound
		return
	}

	image = imgs[0]
	return
}

// FnFindContainer return container by image name
func FnFindContainer(client *docker.Client, imageName string) (container docker.APIContainers, err error) {
	var containers []docker.APIContainers
	containers, err = client.ListContainers(docker.ListContainersOptions{All: true})
	if err != nil {
		return
	}

	if !strings.HasPrefix(imageName, "gofn") {
		imageName = "gofn/" + imageName
	}

	for _, v := range containers {
		if v.Image == imageName {
			container = v
			return
		}
	}
	err = ErrContainerNotFound
	return
}

// FnKillContainer kill the container
func FnKillContainer(client *docker.Client, containerID string) (err error) {
	err = client.KillContainer(docker.KillContainerOptions{ID: containerID})
	return
}

// FnRun runs the container
func FnRun(client *docker.Client, containerID string) (Stdout *bytes.Buffer) {
	err := client.StartContainer(containerID, nil)
	if err != nil {
		log.Println(err)
	}
	client.WaitContainerWithContext(containerID, nil)
	stdout := new(bytes.Buffer)

	client.Logs(docker.LogsOptions{
		Container:    containerID,
		Stdout:       true,
		OutputStream: stdout,
	})
	Stdout = stdout
	return
}

// FnConfigVolume set volume options
func FnConfigVolume(opts *VolumeOptions) string {
	if opts.Source == "" && opts.Destination == "" {
		return ""
	}
	if opts.Destination == "" {
		opts.Destination = opts.Source
	}
	return opts.Source + ":" + opts.Destination
}
<commit_msg>randomize container name to avoid conflict<commit_after>package provision

import (
	"bytes"
	"errors"
	"fmt"
	"log"
	"strings"

	docker "github.com/fsouza/go-dockerclient"
	"github.com/nuveo/gofn/iaas"
	uuid "github.com/satori/go.uuid"
)

var (
	// ErrImageNotFound is raised when image is not found
	ErrImageNotFound = errors.New("provision: image not found")

	// ErrContainerNotFound is raised when image is not found
	ErrContainerNotFound = errors.New("provision: container not found")
)

// VolumeOptions are options to mount a host directory as data volume
type VolumeOptions struct {
	Source      string
	Destination string
}

// BuildOptions are options used in the image build
type BuildOptions struct {
	ContextDir string
	Dockerfile string
	ImageName  string
	RemoteURI  string
	Iaas       iaas.Iaas
}

// FnClient instantiate a docker client
func FnClient(endPoint string) (client *docker.Client) {
	if endPoint == "" {
		endPoint = "unix:///var/run/docker.sock"
	}

	client, err := docker.NewClient(endPoint)
	if err != nil {
		panic(err)
	}
	return
}

// FnRemove remove container
func FnRemove(client *docker.Client, containerID string) (err error) {
	err = client.RemoveContainer(docker.RemoveContainerOptions{ID: containerID, Force: true})
	return
}

// FnContainer create container
func FnContainer(client *docker.Client, image, volume string) (container *docker.Container, err error) {
	binds := []string{}
	if volume != "" {
		binds = append(binds, volume)
	}
	container, err = client.CreateContainer(docker.CreateContainerOptions{
		Name:       fmt.Sprintf("gofn-%s", uuid.NewV4().String()),
		HostConfig: &docker.HostConfig{Binds: binds},
		Config: &docker.Config{
			Image:     image,
			StdinOnce: true,
			OpenStdin: true,
		},
	})
	return
}

// FnImageBuild builds an image
func FnImageBuild(client *docker.Client, opts *BuildOptions) (Name string, Stdout *bytes.Buffer) {
	if opts.Dockerfile == "" {
		opts.Dockerfile = "Dockerfile"
	}
	stdout := new(bytes.Buffer)
	Name = "gofn/" + opts.ImageName
	err := client.BuildImage(docker.BuildImageOptions{
		Name:           Name,
		Dockerfile:     opts.Dockerfile,
		SuppressOutput: true,
		OutputStream:   stdout,
		ContextDir:     opts.ContextDir,
		Remote:         opts.RemoteURI,
	})
	if err != nil {
		panic(err)
	}
	Stdout = stdout
	return
}

// FnFindImage returns image data by name
func FnFindImage(client *docker.Client, imageName string) (image docker.APIImages, err error) {
	var imgs []docker.APIImages
	name := "gofn/" + imageName

	imgs, err = client.ListImages(docker.ListImagesOptions{Filter: name})
	if err != nil {
		return
	}

	if len(imgs) == 0 {
		err = ErrImageNotFound
		return
	}

	image = imgs[0]
	return
}

// FnFindContainer return container by image name
func FnFindContainer(client *docker.Client, imageName string) (container docker.APIContainers, err error) {
	var containers []docker.APIContainers
	containers, err = client.ListContainers(docker.ListContainersOptions{All: true})
	if err != nil {
		return
	}

	if !strings.HasPrefix(imageName, "gofn") {
		imageName = "gofn/" + imageName
	}

	for _, v := range containers {
		if v.Image == imageName {
			container = v
			return
		}
	}
	err = ErrContainerNotFound
	return
}

// FnKillContainer kill the container
func FnKillContainer(client *docker.Client, containerID string) (err error) {
	err = client.KillContainer(docker.KillContainerOptions{ID: containerID})
	return
}

// FnRun runs the container
func FnRun(client *docker.Client, containerID string) (Stdout *bytes.Buffer) {
	err := client.StartContainer(containerID, nil)
	if err != nil {
		log.Println(err)
	}
	client.WaitContainerWithContext(containerID, nil)
	stdout := new(bytes.Buffer)

	client.Logs(docker.LogsOptions{
		Container:    containerID,
		Stdout:       true,
		OutputStream: stdout,
	})
	Stdout = stdout
	return
}

// FnConfigVolume set volume options
func FnConfigVolume(opts *VolumeOptions) string {
	if opts.Source == "" && opts.Destination == "" {
		return ""
	}
	if opts.Destination == "" {
		opts.Destination = opts.Source
	}
	return opts.Source + ":" + opts.Destination
}
<|endoftext|>


<commit_before>//  Copyright (c) 2015 Rackspace
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
//  implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package proxyserver

import (
	"flag"
	"fmt"
	"net/http"
	"strings"

	"github.com/troubling/hummingbird/client"
	"github.com/troubling/hummingbird/common"
	"github.com/troubling/hummingbird/common/conf"
	"github.com/troubling/hummingbird/common/ring"
	"github.com/troubling/hummingbird/common/srv"
	"github.com/troubling/hummingbird/proxyserver/middleware"

	"github.com/justinas/alice"
	"go.uber.org/zap"
)

type ProxyServer struct {
	logger            srv.LowLevelLogger
	logLevel          zap.AtomicLevel
	mc                ring.MemcacheRing
	proxyDirectClient *client.ProxyDirectClient
}

func (server *ProxyServer) Finalize() {
}

func (server *ProxyServer) GetHandler(config conf.Config) http.Handler {
	router := srv.NewRouter()
	router.Get("/loglevel", server.logLevel)
	router.Put("/loglevel", server.logLevel)
	router.Get("/v1/:account/:container/*obj", http.HandlerFunc(server.ObjectGetHandler))
	router.Head("/v1/:account/:container/*obj", http.HandlerFunc(server.ObjectHeadHandler))
	router.Put("/v1/:account/:container/*obj", http.HandlerFunc(server.ObjectPutHandler))
	router.Delete("/v1/:account/:container/*obj", http.HandlerFunc(server.ObjectDeleteHandler))

	router.Get("/v1/:account/:container", http.HandlerFunc(server.ContainerGetHandler))
	router.Get("/v1/:account/:container/", http.HandlerFunc(server.ContainerGetHandler))
	router.Head("/v1/:account/:container", http.HandlerFunc(server.ContainerHeadHandler))
	router.Head("/v1/:account/:container/", http.HandlerFunc(server.ContainerHeadHandler))
	router.Put("/v1/:account/:container", http.HandlerFunc(server.ContainerPutHandler))
	router.Put("/v1/:account/:container/", http.HandlerFunc(server.ContainerPutHandler))
	router.Delete("/v1/:account/:container", http.HandlerFunc(server.ContainerDeleteHandler))
	router.Delete("/v1/:account/:container/", http.HandlerFunc(server.ContainerDeleteHandler))
	router.Post("/v1/:account/:container", http.HandlerFunc(server.ContainerPostHandler))
	router.Post("/v1/:account/:container/", http.HandlerFunc(server.ContainerPostHandler))

	router.Get("/v1/:account", http.HandlerFunc(server.AccountGetHandler))
	router.Get("/v1/:account/", http.HandlerFunc(server.AccountGetHandler))
	router.Head("/v1/:account", http.HandlerFunc(server.AccountHeadHandler))
	router.Head("/v1/:account/", http.HandlerFunc(server.AccountHeadHandler))
	router.Put("/v1/:account", http.HandlerFunc(server.AccountPutHandler))
	router.Put("/v1/:account/", http.HandlerFunc(server.AccountPutHandler))
	router.Delete("/v1/:account", http.HandlerFunc(server.AccountDeleteHandler))
	router.Delete("/v1/:account/", http.HandlerFunc(server.AccountDeleteHandler))
	router.Post("/v1/:account", http.HandlerFunc(server.AccountPostHandler))
	router.Post("/v1/:account/", http.HandlerFunc(server.AccountPostHandler))

	tempAuth := config.GetBool("proxy-server", "tempauth_enabled", true)
	var middlewares []struct {
		construct func(config conf.Section) (func(http.Handler) http.Handler, error)
		section   string
	}
	// TODO: make this all dynamical and stuff
	if tempAuth {
		middlewares = []struct {
			construct func(config conf.Section) (func(http.Handler) http.Handler, error)
			section   string
		}{
			{middleware.NewCatchError, "filter:catch_errors"},
			{middleware.NewHealthcheck, "filter:healthcheck"},
			{middleware.NewRequestLogger, "filter:proxy-logging"},
			{middleware.NewFormPost, "filter:formpost"},
			{middleware.NewTempURL, "filter:tempurl"},
			{middleware.NewTempAuth, "filter:tempauth"},
			{middleware.NewRatelimiter, "filter:ratelimit"},
			//{middleware.NewStaticWeb, "filter:staticweb"},
			{middleware.NewCopyMiddleware, "filter:copy"},
			{middleware.NewXlo, "fliter:slo"},
		}
	} else {
		middlewares = []struct {
			construct func(config conf.Section) (func(http.Handler) http.Handler, error)
			section   string
		}{
			{middleware.NewCatchError, "filter:catch_errors"},
			{middleware.NewHealthcheck, "filter:healthcheck"},
			{middleware.NewRequestLogger, "filter:proxy-logging"},
			{middleware.NewFormPost, "filter:formpost"},
			{middleware.NewTempURL, "filter:tempurl"},
			{middleware.NewAuthToken, "filter:authtoken"},
			{middleware.NewKeystoneAuth, "filter:keystoneauth"},
			{middleware.NewRatelimiter, "filter:ratelimit"},
			//{middleware.NewStaticWeb, "filter:staticweb"},
			{middleware.NewCopyMiddleware, "filter:copy"},
			{middleware.NewXlo, "filter:slo"},
		}
	}
	pipeline := alice.New(middleware.NewContext(server.mc, server.logger, server.proxyDirectClient))
	for _, m := range middlewares {
		mid, err := m.construct(config.GetSection(m.section))
		if err != nil {
			// TODO: propagate error upwards instead of panicking
			panic("Unable to construct middleware")
		}
		pipeline = pipeline.Append(mid)
	}
	return pipeline.Then(router)
}

func GetServer(serverconf conf.Config, flags *flag.FlagSet) (string, int, srv.Server, srv.LowLevelLogger, error) {
	var err error
	server := &ProxyServer{}
	server.mc, err = ring.NewMemcacheRingFromConfig(serverconf)
	if err != nil {
		return "", 0, nil, nil, err
	}

	bindIP := serverconf.GetDefault("DEFAULT", "bind_ip", "0.0.0.0")
	bindPort := serverconf.GetInt("DEFAULT", "bind_port", 8080)

	logLevelString := serverconf.GetDefault("proxy-server", "log_level", "INFO")
	server.logLevel = zap.NewAtomicLevel()
	server.logLevel.UnmarshalText([]byte(strings.ToLower(logLevelString)))

	if server.logger, err = srv.SetupLogger("proxy-server", &server.logLevel, flags); err != nil {
		return "", 0, nil, nil, fmt.Errorf("Error setting up logger: %v", err)
	}
	policies := conf.LoadPolicies()
	server.proxyDirectClient, err = client.NewProxyDirectClient(policies)
	if err != nil {
		return "", 0, nil, nil, fmt.Errorf("Error setting up proxyDirectClient: %v", err)
	}
	info := map[string]interface{}{
		"version":          common.Version,
		"strict_cors_mode": true,
		"policies":         policies.GetPolicyInfo(),
	}
	for k, v := range DEFAULT_CONSTRAINTS {
		info[k] = v
	}
	middleware.RegisterInfo("swift", info)
	return bindIP, int(bindPort), server, server.logger, nil
}
<commit_msg>fix typo<commit_after>//  Copyright (c) 2015 Rackspace
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
//  implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package proxyserver

import (
	"flag"
	"fmt"
	"net/http"
	"strings"

	"github.com/troubling/hummingbird/client"
	"github.com/troubling/hummingbird/common"
	"github.com/troubling/hummingbird/common/conf"
	"github.com/troubling/hummingbird/common/ring"
	"github.com/troubling/hummingbird/common/srv"
	"github.com/troubling/hummingbird/proxyserver/middleware"

	"github.com/justinas/alice"
	"go.uber.org/zap"
)

type ProxyServer struct {
	logger            srv.LowLevelLogger
	logLevel          zap.AtomicLevel
	mc                ring.MemcacheRing
	proxyDirectClient *client.ProxyDirectClient
}

func (server *ProxyServer) Finalize() {
}

func (server *ProxyServer) GetHandler(config conf.Config) http.Handler {
	router := srv.NewRouter()
	router.Get("/loglevel", server.logLevel)
	router.Put("/loglevel", server.logLevel)
	router.Get("/v1/:account/:container/*obj", http.HandlerFunc(server.ObjectGetHandler))
	router.Head("/v1/:account/:container/*obj", http.HandlerFunc(server.ObjectHeadHandler))
	router.Put("/v1/:account/:container/*obj", http.HandlerFunc(server.ObjectPutHandler))
	router.Delete("/v1/:account/:container/*obj", http.HandlerFunc(server.ObjectDeleteHandler))

	router.Get("/v1/:account/:container", http.HandlerFunc(server.ContainerGetHandler))
	router.Get("/v1/:account/:container/", http.HandlerFunc(server.ContainerGetHandler))
	router.Head("/v1/:account/:container", http.HandlerFunc(server.ContainerHeadHandler))
	router.Head("/v1/:account/:container/", http.HandlerFunc(server.ContainerHeadHandler))
	router.Put("/v1/:account/:container", http.HandlerFunc(server.ContainerPutHandler))
	router.Put("/v1/:account/:container/", http.HandlerFunc(server.ContainerPutHandler))
	router.Delete("/v1/:account/:container", http.HandlerFunc(server.ContainerDeleteHandler))
	router.Delete("/v1/:account/:container/", http.HandlerFunc(server.ContainerDeleteHandler))
	router.Post("/v1/:account/:container", http.HandlerFunc(server.ContainerPostHandler))
	router.Post("/v1/:account/:container/", http.HandlerFunc(server.ContainerPostHandler))

	router.Get("/v1/:account", http.HandlerFunc(server.AccountGetHandler))
	router.Get("/v1/:account/", http.HandlerFunc(server.AccountGetHandler))
	router.Head("/v1/:account", http.HandlerFunc(server.AccountHeadHandler))
	router.Head("/v1/:account/", http.HandlerFunc(server.AccountHeadHandler))
	router.Put("/v1/:account", http.HandlerFunc(server.AccountPutHandler))
	router.Put("/v1/:account/", http.HandlerFunc(server.AccountPutHandler))
	router.Delete("/v1/:account", http.HandlerFunc(server.AccountDeleteHandler))
	router.Delete("/v1/:account/", http.HandlerFunc(server.AccountDeleteHandler))
	router.Post("/v1/:account", http.HandlerFunc(server.AccountPostHandler))
	router.Post("/v1/:account/", http.HandlerFunc(server.AccountPostHandler))

	tempAuth := config.GetBool("proxy-server", "tempauth_enabled", true)
	var middlewares []struct {
		construct func(config conf.Section) (func(http.Handler) http.Handler, error)
		section   string
	}
	// TODO: make this all dynamical and stuff
	if tempAuth {
		middlewares = []struct {
			construct func(config conf.Section) (func(http.Handler) http.Handler, error)
			section   string
		}{
			{middleware.NewCatchError, "filter:catch_errors"},
			{middleware.NewHealthcheck, "filter:healthcheck"},
			{middleware.NewRequestLogger, "filter:proxy-logging"},
			{middleware.NewFormPost, "filter:formpost"},
			{middleware.NewTempURL, "filter:tempurl"},
			{middleware.NewTempAuth, "filter:tempauth"},
			{middleware.NewRatelimiter, "filter:ratelimit"},
			//{middleware.NewStaticWeb, "filter:staticweb"},
			{middleware.NewCopyMiddleware, "filter:copy"},
			{middleware.NewXlo, "filter:slo"},
		}
	} else {
		middlewares = []struct {
			construct func(config conf.Section) (func(http.Handler) http.Handler, error)
			section   string
		}{
			{middleware.NewCatchError, "filter:catch_errors"},
			{middleware.NewHealthcheck, "filter:healthcheck"},
			{middleware.NewRequestLogger, "filter:proxy-logging"},
			{middleware.NewFormPost, "filter:formpost"},
			{middleware.NewTempURL, "filter:tempurl"},
			{middleware.NewAuthToken, "filter:authtoken"},
			{middleware.NewKeystoneAuth, "filter:keystoneauth"},
			{middleware.NewRatelimiter, "filter:ratelimit"},
			//{middleware.NewStaticWeb, "filter:staticweb"},
			{middleware.NewCopyMiddleware, "filter:copy"},
			{middleware.NewXlo, "filter:slo"},
		}
	}
	pipeline := alice.New(middleware.NewContext(server.mc, server.logger, server.proxyDirectClient))
	for _, m := range middlewares {
		mid, err := m.construct(config.GetSection(m.section))
		if err != nil {
			// TODO: propagate error upwards instead of panicking
			panic("Unable to construct middleware")
		}
		pipeline = pipeline.Append(mid)
	}
	return pipeline.Then(router)
}

func GetServer(serverconf conf.Config, flags *flag.FlagSet) (string, int, srv.Server, srv.LowLevelLogger, error) {
	var err error
	server := &ProxyServer{}
	server.mc, err = ring.NewMemcacheRingFromConfig(serverconf)
	if err != nil {
		return "", 0, nil, nil, err
	}

	bindIP := serverconf.GetDefault("DEFAULT", "bind_ip", "0.0.0.0")
	bindPort := serverconf.GetInt("DEFAULT", "bind_port", 8080)

	logLevelString := serverconf.GetDefault("proxy-server", "log_level", "INFO")
	server.logLevel = zap.NewAtomicLevel()
	server.logLevel.UnmarshalText([]byte(strings.ToLower(logLevelString)))

	if server.logger, err = srv.SetupLogger("proxy-server", &server.logLevel, flags); err != nil {
		return "", 0, nil, nil, fmt.Errorf("Error setting up logger: %v", err)
	}
	policies := conf.LoadPolicies()
	server.proxyDirectClient, err = client.NewProxyDirectClient(policies)
	if err != nil {
		return "", 0, nil, nil, fmt.Errorf("Error setting up proxyDirectClient: %v", err)
	}
	info := map[string]interface{}{
		"version":          common.Version,
		"strict_cors_mode": true,
		"policies":         policies.GetPolicyInfo(),
	}
	for k, v := range DEFAULT_CONSTRAINTS {
		info[k] = v
	}
	middleware.RegisterInfo("swift", info)
	return bindIP, int(bindPort), server, server.logger, nil
}
<|endoftext|>


<commit_before>package dota

import (
	"errors"
	"net/http"
	"strings"
	"time"

	opendota "github.com/jasonodonnell/go-opendota"
)

type Dota struct {
	client *opendota.Client
	teams  map[int]string
}

type Match struct {
	Radiant string
	Dire    string
	League  string
}

func New(teams []string, httpClient *http.Client) (*Dota, error) {
	var dota Dota
	dota.teams = make(map[int]string)
	dota.client = opendota.NewClient(httpClient)
	if err := dota.getTeams(teams); err != nil {
		return nil, err
	}

	return &dota, nil
}

func (d Dota) GetMatches() ([]Match, error) {
	var results []Match
	if len(d.teams) == 0 {
		return nil, errors.New("No teams whitelisted, aborting")
	}

	games, _, err := d.client.LiveService.Live()
	if err != nil {
		return nil, err
	}

	for _, game := range games {
		now := time.Now().Unix()
		if (now - game.ActivateTime) > 60 {
			continue
		}

		if _, ok := d.teams[game.RadiantTeamID]; !ok {
			if _, ok := d.teams[game.DireTeamID]; !ok {
				continue
			}
		}

		var match Match
		leagues, _, err := d.client.LeagueService.Leagues()
		if err != nil {
			return nil, err
		}

		for _, league := range leagues {
			if league.LeagueID == game.LeagueID {
				match.League = league.Name
				if match.League == "" {
					match.League = "Unknown League"
				}
				break
			}
		}

		match.Radiant = game.RadiantTeamName
		match.Dire = game.DireTeamName
		results = append(results, match)
	}
	return results, nil
}

func (d Dota) getTeams(names []string) error {
	teams, _, err := d.client.TeamService.Teams()
	if err != nil {
		return err
	}
	for _, team := range teams {
		if lookupTeam(team.Name, names) {
			d.teams[team.TeamID] = team.Name
		}
	}
	return nil
}

func lookupTeam(name string, whitelist []string) bool {
	for _, whitelisted := range whitelist {
		if strings.ToLower(whitelisted) == strings.ToLower(name) {
			return true
		}
	}
	return false
}
<commit_msg>Add matchID<commit_after>package dota

import (
	"errors"
	"net/http"
	"strings"
	"time"

	opendota "github.com/jasonodonnell/go-opendota"
)

type Dota struct {
	client *opendota.Client
	teams  map[int]string
}

type Match struct {
	Radiant string
	Dire    string
	League  string
	MatchID int64
}

func New(teams []string, httpClient *http.Client) (*Dota, error) {
	var dota Dota
	dota.teams = make(map[int]string)
	dota.client = opendota.NewClient(httpClient)
	if err := dota.getTeams(teams); err != nil {
		return nil, err
	}

	return &dota, nil
}

func (d Dota) GetMatches() ([]Match, error) {
	var results []Match
	if len(d.teams) == 0 {
		return nil, errors.New("No teams whitelisted, aborting")
	}

	games, _, err := d.client.LiveService.Live()
	if err != nil {
		return nil, err
	}

	for _, game := range games {
		now := time.Now().Unix()
		if (now - game.ActivateTime) > 60 {
			continue
		}

		if _, ok := d.teams[game.RadiantTeamID]; !ok {
			if _, ok := d.teams[game.DireTeamID]; !ok {
				continue
			}
		}

		var match Match
		leagues, _, err := d.client.LeagueService.Leagues()
		if err != nil {
			return nil, err
		}

		for _, league := range leagues {
			if league.LeagueID == game.LeagueID {
				match.League = league.Name
				if match.League == "" {
					match.League = "Unknown League"
				}
				break
			}
		}

		match.Radiant = game.RadiantTeamName
		match.Dire = game.DireTeamName
		match.MatchID = game.MatchID
		results = append(results, match)
	}
	return results, nil
}

func (d Dota) getTeams(names []string) error {
	teams, _, err := d.client.TeamService.Teams()
	if err != nil {
		return err
	}
	for _, team := range teams {
		if lookupTeam(team.Name, names) {
			d.teams[team.TeamID] = team.Name
		}
	}
	return nil
}

func lookupTeam(name string, whitelist []string) bool {
	for _, whitelisted := range whitelist {
		if strings.ToLower(whitelisted) == strings.ToLower(name) {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package seccomp

import (
	"syscall"
)

type RenderingFunctions map[int]func(int, RegisterArgs) (string, error)

func getRenderingFunctions() RenderingFunctions {
	r := map[int]func(pid int, args RegisterArgs) (string, error){
		syscall.SYS_ACCESS:   render_access,
		syscall.SYS_MPROTECT: render_mprotect,
		syscall.SYS_MMAP:     render_mmap,
		syscall.SYS_MREMAP:   render_mremap,
		syscall.SYS_FUTEX:    render_futex,
	}
	return r
}

func renderFlags(flags map[uint]string, val uint) string {
	found := false
	flagstr := ""

	for flag := range flags {
		if val&uint(flag) == uint(flag) {
			if found == true {
				flagstr += "|"
			}
			flagstr += flags[flag]
			found = true
		}
	}
	return flagstr

}

func allFlagsTest(flags []uint, val uint) bool {
	var i uint = 0

	for flag := range flags {
		i |= uint(flag)
	}
	return i == val
}
<commit_msg>Add render_openat() to the list of system call renderers.<commit_after>package seccomp

import (
	"syscall"
)

type RenderingFunctions map[int]func(int, RegisterArgs) (string, error)

func getRenderingFunctions() RenderingFunctions {
	r := map[int]func(pid int, args RegisterArgs) (string, error){
		syscall.SYS_ACCESS:   render_access,
		syscall.SYS_MPROTECT: render_mprotect,
		syscall.SYS_MMAP:     render_mmap,
		syscall.SYS_MREMAP:   render_mremap,
		syscall.SYS_FUTEX:    render_futex,
		syscall.SYS_OPENAT:   render_openat,
	}
	return r
}

func renderFlags(flags map[uint]string, val uint) string {
	found := false
	flagstr := ""

	for flag := range flags {
		if val&uint(flag) == uint(flag) {
			if found == true {
				flagstr += "|"
			}
			flagstr += flags[flag]
			found = true
		}
	}
	return flagstr

}

func allFlagsTest(flags []uint, val uint) bool {
	var i uint = 0

	for flag := range flags {
		i |= uint(flag)
	}
	return i == val
}
<|endoftext|>


<commit_before>// Package dual provides an implementaiton of a split or "dual" dht, where two parallel instances
// are maintained for the global internet and the local LAN respectively.
package dual

import (
	"context"
	"sync"

	dht "github.com/libp2p/go-libp2p-kad-dht"

	"github.com/ipfs/go-cid"
	ci "github.com/libp2p/go-libp2p-core/crypto"
	"github.com/libp2p/go-libp2p-core/host"
	"github.com/libp2p/go-libp2p-core/peer"
	"github.com/libp2p/go-libp2p-core/protocol"
	"github.com/libp2p/go-libp2p-core/routing"
	kb "github.com/libp2p/go-libp2p-kbucket"
	helper "github.com/libp2p/go-libp2p-routing-helpers"
	ma "github.com/multiformats/go-multiaddr"

	"github.com/hashicorp/go-multierror"
)

// DHT implements the routing interface to provide two concrete DHT implementationts for use
// in IPFS that are used to support both global network users and disjoint LAN usecases.
type DHT struct {
	WAN *dht.IpfsDHT
	LAN *dht.IpfsDHT
}

// LanExtension is used to differentiate local protocol requests from those on the WAN DHT.
const LanExtension protocol.ID = "/lan"

// Assert that IPFS assumptions about interfaces aren't broken. These aren't a
// guarantee, but we can use them to aid refactoring.
var (
	_ routing.ContentRouting = (*DHT)(nil)
	_ routing.Routing        = (*DHT)(nil)
	_ routing.PeerRouting    = (*DHT)(nil)
	_ routing.PubKeyFetcher  = (*DHT)(nil)
	_ routing.ValueStore     = (*DHT)(nil)
)

// New creates a new DualDHT instance. Options provided are forwarded on to the two concrete
// IpfsDHT internal constructions, modulo additional options used by the Dual DHT to enforce
// the LAN-vs-WAN distinction.
// Note: query or routing table functional options provided as arguments to this function
// will be overriden by this constructor.
func New(ctx context.Context, h host.Host, options ...dht.Option) (*DHT, error) {
	wanOpts := append(options,
		dht.QueryFilter(dht.PublicQueryFilter),
		dht.RoutingTableFilter(dht.PublicRoutingTableFilter),
	)
	wan, err := dht.New(ctx, h, wanOpts...)
	if err != nil {
		return nil, err
	}

	// Unless overridden by user supplied options, the LAN DHT should default
	// to 'AutoServer' mode.
	lanOpts := append(options,
		dht.ProtocolExtension(LanExtension),
		dht.QueryFilter(dht.PrivateQueryFilter),
		dht.RoutingTableFilter(dht.PrivateRoutingTableFilter),
	)
	if wan.Mode() != dht.ModeClient {
		lanOpts = append(lanOpts, dht.Mode(dht.ModeServer))
	}
	lan, err := dht.New(ctx, h, lanOpts...)
	if err != nil {
		return nil, err
	}

	impl := DHT{wan, lan}
	return &impl, nil
}

// Close closes the DHT context.
func (dht *DHT) Close() error {
	return combineErrors(dht.WAN.Close(), dht.LAN.Close())
}

// WANActive returns true when the WAN DHT is active (has peers).
func (dht *DHT) WANActive() bool {
	return dht.WAN.RoutingTable().Size() > 0
}

// Provide adds the given cid to the content routing system.
func (dht *DHT) Provide(ctx context.Context, key cid.Cid, announce bool) error {
	if dht.WANActive() {
		return dht.WAN.Provide(ctx, key, announce)
	}
	return dht.LAN.Provide(ctx, key, announce)
}

// FindProvidersAsync searches for peers who are able to provide a given key
func (dht *DHT) FindProvidersAsync(ctx context.Context, key cid.Cid, count int) <-chan peer.AddrInfo {
	reqCtx, cancel := context.WithCancel(ctx)
	outCh := make(chan peer.AddrInfo)
	subCtx, errCh := routing.RegisterForQueryEvents(reqCtx)
	wanCh := dht.WAN.FindProvidersAsync(subCtx, key, count)
	lanCh := dht.LAN.FindProvidersAsync(subCtx, key, count)
	zeroCount := (count == 0)
	go func() {
		defer cancel()
		defer close(outCh)

		found := make(map[peer.ID]struct{}, count)
		var pi peer.AddrInfo
		var qEv *routing.QueryEvent
		for (zeroCount || count > 0) && (wanCh != nil || lanCh != nil) {
			var ok bool
			select {
			case qEv, ok = <-errCh:
				if ok && qEv != nil && qEv.Type != routing.QueryError {
					routing.PublishQueryEvent(reqCtx, qEv)
				}
				continue
			case pi, ok = <-wanCh:
				if !ok {
					wanCh = nil
					continue
				}
			case pi, ok = <-lanCh:
				if !ok {
					lanCh = nil
					continue
				}
			}
			// already found
			if _, ok = found[pi.ID]; ok {
				continue
			}

			select {
			case outCh <- pi:
				found[pi.ID] = struct{}{}
				count--
			case <-ctx.Done():
				return
			}
		}
		if qEv != nil && qEv.Type == routing.QueryError && len(found) == 0 {
			routing.PublishQueryEvent(reqCtx, qEv)
		}
	}()
	return outCh
}

// FindPeer searches for a peer with given ID
// Note: with signed peer records, we can change this to short circuit once either DHT returns.
func (dht *DHT) FindPeer(ctx context.Context, pid peer.ID) (peer.AddrInfo, error) {
	var wg sync.WaitGroup
	wg.Add(2)
	var wanInfo, lanInfo peer.AddrInfo
	var wanErr, lanErr error
	go func() {
		defer wg.Done()
		wanInfo, wanErr = dht.WAN.FindPeer(ctx, pid)
	}()
	go func() {
		defer wg.Done()
		lanInfo, lanErr = dht.LAN.FindPeer(ctx, pid)
	}()

	wg.Wait()

	// Combine addresses. Try to avoid doing unnecessary work while we're at
	// it. Note: We're ignoring the errors for now as many of our DHT
	// commands can return both a result and an error.
	ai := peer.AddrInfo{ID: pid}
	if len(wanInfo.Addrs) == 0 {
		ai.Addrs = lanInfo.Addrs
	} else if len(lanInfo.Addrs) == 0 {
		ai.Addrs = wanInfo.Addrs
	} else {
		// combine addresses
		deduped := make(map[string]ma.Multiaddr, len(wanInfo.Addrs)+len(lanInfo.Addrs))
		for _, addr := range wanInfo.Addrs {
			deduped[string(addr.Bytes())] = addr
		}
		for _, addr := range lanInfo.Addrs {
			deduped[string(addr.Bytes())] = addr
		}
		ai.Addrs = make([]ma.Multiaddr, 0, len(deduped))
		for _, addr := range deduped {
			ai.Addrs = append(ai.Addrs, addr)
		}
	}

	// If one of the commands succeeded, don't return an error.
	if wanErr == nil || lanErr == nil {
		return ai, nil
	}

	// Otherwise, return what we have _and_ return the error.
	return ai, combineErrors(wanErr, lanErr)
}

func combineErrors(erra, errb error) error {
	// if the errors are the same, just return one.
	if erra == errb {
		return erra
	}

	// If one of the errors is a kb lookup failure (no peers in routing
	// table), return the other.
	if erra == kb.ErrLookupFailure {
		return errb
	} else if errb == kb.ErrLookupFailure {
		return erra
	}
	return multierror.Append(erra, errb).ErrorOrNil()
}

// Bootstrap allows callers to hint to the routing system to get into a
// Boostrapped state and remain there.
func (dht *DHT) Bootstrap(ctx context.Context) error {
	erra := dht.WAN.Bootstrap(ctx)
	errb := dht.LAN.Bootstrap(ctx)
	return combineErrors(erra, errb)
}

// PutValue adds value corresponding to given Key.
func (dht *DHT) PutValue(ctx context.Context, key string, val []byte, opts ...routing.Option) error {
	if dht.WANActive() {
		return dht.WAN.PutValue(ctx, key, val, opts...)
	}
	return dht.LAN.PutValue(ctx, key, val, opts...)
}

// GetValue searches for the value corresponding to given Key.
func (d *DHT) GetValue(ctx context.Context, key string, opts ...routing.Option) ([]byte, error) {
	lanCtx, cancelLan := context.WithCancel(ctx)
	defer cancelLan()

	var (
		lanVal    []byte
		lanErr    error
		lanWaiter sync.WaitGroup
	)
	lanWaiter.Add(1)
	go func() {
		defer lanWaiter.Done()
		lanVal, lanErr = d.LAN.GetValue(lanCtx, key, opts...)
	}()

	wanVal, wanErr := d.WAN.GetValue(ctx, key, opts...)
	if wanErr == nil {
		cancelLan()
	}
	lanWaiter.Wait()
	if wanErr == nil {
		return wanVal, nil
	}
	if lanErr == nil {
		return lanVal, nil
	}
	return nil, combineErrors(wanErr, lanErr)
}

// SearchValue searches for better values from this value
func (dht *DHT) SearchValue(ctx context.Context, key string, opts ...routing.Option) (<-chan []byte, error) {
	p := helper.Parallel{Routers: []routing.Routing{dht.WAN, dht.LAN}, Validator: dht.WAN.Validator}
	return p.SearchValue(ctx, key, opts...)
}

// GetPublicKey returns the public key for the given peer.
func (dht *DHT) GetPublicKey(ctx context.Context, pid peer.ID) (ci.PubKey, error) {
	p := helper.Parallel{Routers: []routing.Routing{dht.WAN, dht.LAN}, Validator: dht.WAN.Validator}
	return p.GetPublicKey(ctx, pid)
}
<commit_msg>fix: don't spin when the event channel is closed<commit_after>// Package dual provides an implementaiton of a split or "dual" dht, where two parallel instances
// are maintained for the global internet and the local LAN respectively.
package dual

import (
	"context"
	"sync"

	dht "github.com/libp2p/go-libp2p-kad-dht"

	"github.com/ipfs/go-cid"
	ci "github.com/libp2p/go-libp2p-core/crypto"
	"github.com/libp2p/go-libp2p-core/host"
	"github.com/libp2p/go-libp2p-core/peer"
	"github.com/libp2p/go-libp2p-core/protocol"
	"github.com/libp2p/go-libp2p-core/routing"
	kb "github.com/libp2p/go-libp2p-kbucket"
	helper "github.com/libp2p/go-libp2p-routing-helpers"
	ma "github.com/multiformats/go-multiaddr"

	"github.com/hashicorp/go-multierror"
)

// DHT implements the routing interface to provide two concrete DHT implementationts for use
// in IPFS that are used to support both global network users and disjoint LAN usecases.
type DHT struct {
	WAN *dht.IpfsDHT
	LAN *dht.IpfsDHT
}

// LanExtension is used to differentiate local protocol requests from those on the WAN DHT.
const LanExtension protocol.ID = "/lan"

// Assert that IPFS assumptions about interfaces aren't broken. These aren't a
// guarantee, but we can use them to aid refactoring.
var (
	_ routing.ContentRouting = (*DHT)(nil)
	_ routing.Routing        = (*DHT)(nil)
	_ routing.PeerRouting    = (*DHT)(nil)
	_ routing.PubKeyFetcher  = (*DHT)(nil)
	_ routing.ValueStore     = (*DHT)(nil)
)

// New creates a new DualDHT instance. Options provided are forwarded on to the two concrete
// IpfsDHT internal constructions, modulo additional options used by the Dual DHT to enforce
// the LAN-vs-WAN distinction.
// Note: query or routing table functional options provided as arguments to this function
// will be overriden by this constructor.
func New(ctx context.Context, h host.Host, options ...dht.Option) (*DHT, error) {
	wanOpts := append(options,
		dht.QueryFilter(dht.PublicQueryFilter),
		dht.RoutingTableFilter(dht.PublicRoutingTableFilter),
	)
	wan, err := dht.New(ctx, h, wanOpts...)
	if err != nil {
		return nil, err
	}

	// Unless overridden by user supplied options, the LAN DHT should default
	// to 'AutoServer' mode.
	lanOpts := append(options,
		dht.ProtocolExtension(LanExtension),
		dht.QueryFilter(dht.PrivateQueryFilter),
		dht.RoutingTableFilter(dht.PrivateRoutingTableFilter),
	)
	if wan.Mode() != dht.ModeClient {
		lanOpts = append(lanOpts, dht.Mode(dht.ModeServer))
	}
	lan, err := dht.New(ctx, h, lanOpts...)
	if err != nil {
		return nil, err
	}

	impl := DHT{wan, lan}
	return &impl, nil
}

// Close closes the DHT context.
func (dht *DHT) Close() error {
	return combineErrors(dht.WAN.Close(), dht.LAN.Close())
}

// WANActive returns true when the WAN DHT is active (has peers).
func (dht *DHT) WANActive() bool {
	return dht.WAN.RoutingTable().Size() > 0
}

// Provide adds the given cid to the content routing system.
func (dht *DHT) Provide(ctx context.Context, key cid.Cid, announce bool) error {
	if dht.WANActive() {
		return dht.WAN.Provide(ctx, key, announce)
	}
	return dht.LAN.Provide(ctx, key, announce)
}

// FindProvidersAsync searches for peers who are able to provide a given key
func (dht *DHT) FindProvidersAsync(ctx context.Context, key cid.Cid, count int) <-chan peer.AddrInfo {
	reqCtx, cancel := context.WithCancel(ctx)
	outCh := make(chan peer.AddrInfo)
	subCtx, evtCh := routing.RegisterForQueryEvents(reqCtx)
	wanCh := dht.WAN.FindProvidersAsync(subCtx, key, count)
	lanCh := dht.LAN.FindProvidersAsync(subCtx, key, count)
	zeroCount := (count == 0)
	go func() {
		defer cancel()
		defer close(outCh)

		found := make(map[peer.ID]struct{}, count)
		var pi peer.AddrInfo
		var qEv *routing.QueryEvent
		for (zeroCount || count > 0) && (wanCh != nil || lanCh != nil) {
			var ok bool
			select {
			case qEv, ok = <-evtCh:
				if !ok {
					evtCh = nil
				} else if qEv != nil && qEv.Type != routing.QueryError {
					routing.PublishQueryEvent(reqCtx, qEv)
				}
				continue
			case pi, ok = <-wanCh:
				if !ok {
					wanCh = nil
					continue
				}
			case pi, ok = <-lanCh:
				if !ok {
					lanCh = nil
					continue
				}
			}
			// already found
			if _, ok = found[pi.ID]; ok {
				continue
			}

			select {
			case outCh <- pi:
				found[pi.ID] = struct{}{}
				count--
			case <-ctx.Done():
				return
			}
		}
		if qEv != nil && qEv.Type == routing.QueryError && len(found) == 0 {
			routing.PublishQueryEvent(reqCtx, qEv)
		}
	}()
	return outCh
}

// FindPeer searches for a peer with given ID
// Note: with signed peer records, we can change this to short circuit once either DHT returns.
func (dht *DHT) FindPeer(ctx context.Context, pid peer.ID) (peer.AddrInfo, error) {
	var wg sync.WaitGroup
	wg.Add(2)
	var wanInfo, lanInfo peer.AddrInfo
	var wanErr, lanErr error
	go func() {
		defer wg.Done()
		wanInfo, wanErr = dht.WAN.FindPeer(ctx, pid)
	}()
	go func() {
		defer wg.Done()
		lanInfo, lanErr = dht.LAN.FindPeer(ctx, pid)
	}()

	wg.Wait()

	// Combine addresses. Try to avoid doing unnecessary work while we're at
	// it. Note: We're ignoring the errors for now as many of our DHT
	// commands can return both a result and an error.
	ai := peer.AddrInfo{ID: pid}
	if len(wanInfo.Addrs) == 0 {
		ai.Addrs = lanInfo.Addrs
	} else if len(lanInfo.Addrs) == 0 {
		ai.Addrs = wanInfo.Addrs
	} else {
		// combine addresses
		deduped := make(map[string]ma.Multiaddr, len(wanInfo.Addrs)+len(lanInfo.Addrs))
		for _, addr := range wanInfo.Addrs {
			deduped[string(addr.Bytes())] = addr
		}
		for _, addr := range lanInfo.Addrs {
			deduped[string(addr.Bytes())] = addr
		}
		ai.Addrs = make([]ma.Multiaddr, 0, len(deduped))
		for _, addr := range deduped {
			ai.Addrs = append(ai.Addrs, addr)
		}
	}

	// If one of the commands succeeded, don't return an error.
	if wanErr == nil || lanErr == nil {
		return ai, nil
	}

	// Otherwise, return what we have _and_ return the error.
	return ai, combineErrors(wanErr, lanErr)
}

func combineErrors(erra, errb error) error {
	// if the errors are the same, just return one.
	if erra == errb {
		return erra
	}

	// If one of the errors is a kb lookup failure (no peers in routing
	// table), return the other.
	if erra == kb.ErrLookupFailure {
		return errb
	} else if errb == kb.ErrLookupFailure {
		return erra
	}
	return multierror.Append(erra, errb).ErrorOrNil()
}

// Bootstrap allows callers to hint to the routing system to get into a
// Boostrapped state and remain there.
func (dht *DHT) Bootstrap(ctx context.Context) error {
	erra := dht.WAN.Bootstrap(ctx)
	errb := dht.LAN.Bootstrap(ctx)
	return combineErrors(erra, errb)
}

// PutValue adds value corresponding to given Key.
func (dht *DHT) PutValue(ctx context.Context, key string, val []byte, opts ...routing.Option) error {
	if dht.WANActive() {
		return dht.WAN.PutValue(ctx, key, val, opts...)
	}
	return dht.LAN.PutValue(ctx, key, val, opts...)
}

// GetValue searches for the value corresponding to given Key.
func (d *DHT) GetValue(ctx context.Context, key string, opts ...routing.Option) ([]byte, error) {
	lanCtx, cancelLan := context.WithCancel(ctx)
	defer cancelLan()

	var (
		lanVal    []byte
		lanErr    error
		lanWaiter sync.WaitGroup
	)
	lanWaiter.Add(1)
	go func() {
		defer lanWaiter.Done()
		lanVal, lanErr = d.LAN.GetValue(lanCtx, key, opts...)
	}()

	wanVal, wanErr := d.WAN.GetValue(ctx, key, opts...)
	if wanErr == nil {
		cancelLan()
	}
	lanWaiter.Wait()
	if wanErr == nil {
		return wanVal, nil
	}
	if lanErr == nil {
		return lanVal, nil
	}
	return nil, combineErrors(wanErr, lanErr)
}

// SearchValue searches for better values from this value
func (dht *DHT) SearchValue(ctx context.Context, key string, opts ...routing.Option) (<-chan []byte, error) {
	p := helper.Parallel{Routers: []routing.Routing{dht.WAN, dht.LAN}, Validator: dht.WAN.Validator}
	return p.SearchValue(ctx, key, opts...)
}

// GetPublicKey returns the public key for the given peer.
func (dht *DHT) GetPublicKey(ctx context.Context, pid peer.ID) (ci.PubKey, error) {
	p := helper.Parallel{Routers: []routing.Routing{dht.WAN, dht.LAN}, Validator: dht.WAN.Validator}
	return p.GetPublicKey(ctx, pid)
}
<|endoftext|>