<commit_before>// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package websocket

import (
	"crypto/tls"
	"crypto/x509"
	"io"
	"net"
	"net/http"
	"net/http/httptest"
	"net/url"
	"reflect"
	"testing"
	"time"
)

type handshakeHandler struct {
	*testing.T
}

func (t handshakeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		http.Error(w, "Method not allowed", 405)
		t.Logf("method = %s, want GET", r.Method)
		return
	}
	if origin := r.Header.Get("Origin"); origin != "http://"+r.Host {
		http.Error(w, "Origin not allowed", 403)
		t.Logf("Origin = %s, want %s", origin, r.Host)
		return
	}
	subprotos := Subprotocols(r)
	if !reflect.DeepEqual(subprotos, handshakeDialer.Subprotocols) {
		http.Error(w, "bad protocol", 400)
		t.Logf("Subprotocols = %v, want %v", subprotos, handshakeDialer.Subprotocols)
		return
	}
	ws, err := Upgrade(w, r, http.Header{
		"Set-Cookie":             {"sessionID=1234"},
		"Sec-Websocket-Protocol": {subprotos[0]},
	}, 1024, 1024)
	if _, ok := err.(HandshakeError); ok {
		t.Logf("bad handshake: %v", err)
		http.Error(w, "Not a websocket handshake", 400)
		return
	} else if err != nil {
		t.Logf("upgrade error: %v", err)
		return
	}
	defer ws.Close()

	if ws.Subprotocol() != subprotos[0] {
		t.Logf("ws.Subprotocol() = %s, want %s", ws.Subprotocol(), subprotos[0])
		return
	}

	for {
		op, r, err := ws.NextReader()
		if err != nil {
			if err != io.EOF {
				t.Logf("NextReader: %v", err)
			}
			return
		}
		w, err := ws.NextWriter(op)
		if err != nil {
			t.Logf("NextWriter: %v", err)
			return
		}
		if _, err = io.Copy(w, r); err != nil {
			t.Logf("Copy: %v", err)
			return
		}
		if err := w.Close(); err != nil {
			t.Logf("Close: %v", err)
			return
		}
	}
}

var handshakeDialer = &Dialer{
	Subprotocols:    []string{"p1", "p2"},
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
}

func TestHandshake(t *testing.T) {
	s := httptest.NewServer(handshakeHandler{t})
	defer s.Close()
	ws, resp, err := handshakeDialer.Dial(httpToWs(s.URL), http.Header{"Origin": {s.URL}})
	if err != nil {
		t.Fatalf("Dial: %v", err)
	}
	defer ws.Close()

	var sessionID string
	for _, c := range resp.Cookies() {
		if c.Name == "sessionID" {
			sessionID = c.Value
		}
	}
	if sessionID != "1234" {
		t.Error("Set-Cookie not received from the server.")
	}

	if ws.Subprotocol() != handshakeDialer.Subprotocols[0] {
		t.Errorf("ws.Subprotocol() = %s, want %s", ws.Subprotocol(), handshakeDialer.Subprotocols[0])
	}

	sendRecv(t, ws)
}

type dialHandler struct {
	*testing.T
}

func (t dialHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	ws, err := Upgrade(w, r, nil, 1024, 1024)
	if _, ok := err.(HandshakeError); ok {
		t.Logf("bad handshake: %v", err)
		http.Error(w, "Not a websocket handshake", 400)
		return
	} else if err != nil {
		t.Logf("upgrade error: %v", err)
		return
	}
	defer ws.Close()
	for {
		mt, p, err := ws.ReadMessage()
		if err != nil {
			if err != io.EOF {
				t.Logf("ReadMessage: %v", err)
			}
			return
		}
		if err := ws.WriteMessage(mt, p); err != nil {
			t.Logf("WriteMessage: %v", err)
			return
		}
	}
}

func sendRecv(t *testing.T, ws *Conn) {
	const message = "Hello World!"
	if err := ws.SetWriteDeadline(time.Now().Add(time.Second)); err != nil {
		t.Fatalf("SetWriteDeadline: %v", err)
	}
	if err := ws.WriteMessage(TextMessage, []byte(message)); err != nil {
		t.Fatalf("WriteMessage: %v", err)
	}
	if err := ws.SetReadDeadline(time.Now().Add(time.Second)); err != nil {
		t.Fatalf("SetReadDeadline: %v", err)
	}
	_, p, err := ws.ReadMessage()
	if err != nil {
		t.Fatalf("ReadMessage: %v", err)
	}
	if string(p) != message {
		t.Fatalf("message=%s, want %s", p, message)
	}
}

func httpToWs(u string) string {
	return "ws" + u[len("http"):]
}

func TestDial(t *testing.T) {
	s := httptest.NewServer(dialHandler{t})
	defer s.Close()
	ws, _, err := DefaultDialer.Dial(httpToWs(s.URL), nil)
	if err != nil {
		t.Fatalf("Dial() returned error %v", err)
	}
	defer ws.Close()
	sendRecv(t, ws)
}

func TestDialTLS(t *testing.T) {
	s := httptest.NewTLSServer(dialHandler{t})
	defer s.Close()

	certs := x509.NewCertPool()
	for _, c := range s.TLS.Certificates {
		roots, err := x509.ParseCertificates(c.Certificate[len(c.Certificate)-1])
		if err != nil {
			t.Fatalf("error parsing server's root cert: %v", err)
		}
		for _, root := range roots {
			certs.AddCert(root)
		}
	}

	u, _ := url.Parse(s.URL)
	d := &Dialer{
		NetDial:         func(network, addr string) (net.Conn, error) { return net.Dial(network, u.Host) },
		TLSClientConfig: &tls.Config{RootCAs: certs},
	}
	ws, _, err := d.Dial("wss://example.com/", nil)
	if err != nil {
		t.Fatalf("Dial() returned error %v", err)
	}
	defer ws.Close()
	sendRecv(t, ws)
}

func TestDialTLSBadCert(t *testing.T) {
	s := httptest.NewTLSServer(dialHandler{t})
	defer s.Close()
	_, _, err := DefaultDialer.Dial(httpToWs(s.URL), nil)
	if err == nil {
		t.Fatalf("Dial() did not return error")
	}
}

func TestDialTLSNoVerify(t *testing.T) {
	s := httptest.NewTLSServer(dialHandler{t})
	defer s.Close()
	d := &Dialer{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
	ws, _, err := d.Dial(httpToWs(s.URL), nil)
	if err != nil {
		t.Fatalf("Dial() returned error %v", err)
	}
	defer ws.Close()
	sendRecv(t, ws)
}

func TestDialTimeout(t *testing.T) {
	s := httptest.NewServer(dialHandler{t})
	defer s.Close()
	d := &Dialer{
		HandshakeTimeout: -1,
	}
	_, _, err := d.Dial(httpToWs(s.URL), nil)
	if err == nil {
		t.Fatalf("Dial() did not return error")
	}
}

func TestDialBadScheme(t *testing.T) {
	s := httptest.NewServer(dialHandler{t})
	defer s.Close()
	_, _, err := DefaultDialer.Dial(s.URL, nil)
	if err == nil {
		t.Fatalf("Dial() did not return error")
	}
}
<commit_msg>Use Upgrader in client/server test.<commit_after>// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package websocket

import (
	"crypto/tls"
	"crypto/x509"
	"io"
	"net"
	"net/http"
	"net/http/httptest"
	"net/url"
	"reflect"
	"testing"
	"time"
)

func sendRecv(t *testing.T, ws *Conn) {
	const message = "Hello World!"
	if err := ws.SetWriteDeadline(time.Now().Add(time.Second)); err != nil {
		t.Fatalf("SetWriteDeadline: %v", err)
	}
	if err := ws.WriteMessage(TextMessage, []byte(message)); err != nil {
		t.Fatalf("WriteMessage: %v", err)
	}
	if err := ws.SetReadDeadline(time.Now().Add(time.Second)); err != nil {
		t.Fatalf("SetReadDeadline: %v", err)
	}
	_, p, err := ws.ReadMessage()
	if err != nil {
		t.Fatalf("ReadMessage: %v", err)
	}
	if string(p) != message {
		t.Fatalf("message=%s, want %s", p, message)
	}
}

func httpToWs(u string) string {
	return "ws" + u[len("http"):]
}

var handshakeUpgrader = &Upgrader{
	Subprotocols:    []string{"p0", "p1"},
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
}

var handshakeDialer = &Dialer{
	Subprotocols:    []string{"p1", "p2"},
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
}

type handshakeHandler struct {
	*testing.T
}

func (t handshakeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		http.Error(w, "Method not allowed", 405)
		t.Logf("method = %s, want GET", r.Method)
		return
	}
	subprotos := Subprotocols(r)
	if !reflect.DeepEqual(subprotos, handshakeDialer.Subprotocols) {
		http.Error(w, "bad protocol", 400)
		t.Logf("Subprotocols = %v, want %v", subprotos, handshakeDialer.Subprotocols)
		return
	}
	ws, err := handshakeUpgrader.Upgrade(w, r, http.Header{"Set-Cookie": {"sessionID=1234"}})
	if err != nil {
		t.Logf("upgrade error: %v", err)
		return
	}
	defer ws.Close()

	if ws.Subprotocol() != "p1" {
		t.Logf("ws.Subprotocol() = %s, want p1", ws.Subprotocol())
		return
	}

	for {
		op, r, err := ws.NextReader()
		if err != nil {
			if err != io.EOF {
				t.Logf("NextReader: %v", err)
			}
			return
		}
		w, err := ws.NextWriter(op)
		if err != nil {
			t.Logf("NextWriter: %v", err)
			return
		}
		if _, err = io.Copy(w, r); err != nil {
			t.Logf("Copy: %v", err)
			return
		}
		if err := w.Close(); err != nil {
			t.Logf("Close: %v", err)
			return
		}
	}
}

func TestHandshake(t *testing.T) {
	s := httptest.NewServer(handshakeHandler{t})
	defer s.Close()
	ws, resp, err := handshakeDialer.Dial(httpToWs(s.URL), http.Header{"Origin": {s.URL}})
	if err != nil {
		t.Fatalf("Dial: %v", err)
	}
	defer ws.Close()

	var sessionID string
	for _, c := range resp.Cookies() {
		if c.Name == "sessionID" {
			sessionID = c.Value
		}
	}
	if sessionID != "1234" {
		t.Error("Set-Cookie not received from the server.")
	}

	if ws.Subprotocol() != "p1" {
		t.Errorf("ws.Subprotocol() = %s, want p1", ws.Subprotocol())
	}
	sendRecv(t, ws)
}

type dialHandler struct {
	*testing.T
}

var dialUpgrader = &Upgrader{
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
	CheckOrigin:     func(r *http.Request) bool { return true },
}

func (t dialHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	ws, err := dialUpgrader.Upgrade(w, r, nil)
	if err != nil {
		t.Logf("upgrade error: %v", err)
		return
	}
	defer ws.Close()
	for {
		mt, p, err := ws.ReadMessage()
		if err != nil {
			if err != io.EOF {
				t.Logf("ReadMessage: %v", err)
			}
			return
		}
		if err := ws.WriteMessage(mt, p); err != nil {
			t.Logf("WriteMessage: %v", err)
			return
		}
	}
}

func TestDial(t *testing.T) {
	s := httptest.NewServer(dialHandler{t})
	defer s.Close()
	ws, _, err := DefaultDialer.Dial(httpToWs(s.URL), nil)
	if err != nil {
		t.Fatalf("Dial() returned error %v", err)
	}
	defer ws.Close()
	sendRecv(t, ws)
}

func TestDialTLS(t *testing.T) {
	s := httptest.NewTLSServer(dialHandler{t})
	defer s.Close()

	certs := x509.NewCertPool()
	for _, c := range s.TLS.Certificates {
		roots, err := x509.ParseCertificates(c.Certificate[len(c.Certificate)-1])
		if err != nil {
			t.Fatalf("error parsing server's root cert: %v", err)
		}
		for _, root := range roots {
			certs.AddCert(root)
		}
	}

	u, _ := url.Parse(s.URL)
	d := &Dialer{
		NetDial:         func(network, addr string) (net.Conn, error) { return net.Dial(network, u.Host) },
		TLSClientConfig: &tls.Config{RootCAs: certs},
	}
	ws, _, err := d.Dial("wss://example.com/", nil)
	if err != nil {
		t.Fatalf("Dial() returned error %v", err)
	}
	defer ws.Close()
	sendRecv(t, ws)
}

func TestDialTLSBadCert(t *testing.T) {
	s := httptest.NewTLSServer(dialHandler{t})
	defer s.Close()
	_, _, err := DefaultDialer.Dial(httpToWs(s.URL), nil)
	if err == nil {
		t.Fatalf("Dial() did not return error")
	}
}

func TestDialTLSNoVerify(t *testing.T) {
	s := httptest.NewTLSServer(dialHandler{t})
	defer s.Close()
	d := &Dialer{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
	ws, _, err := d.Dial(httpToWs(s.URL), nil)
	if err != nil {
		t.Fatalf("Dial() returned error %v", err)
	}
	defer ws.Close()
	sendRecv(t, ws)
}

func TestDialTimeout(t *testing.T) {
	s := httptest.NewServer(dialHandler{t})
	defer s.Close()
	d := &Dialer{
		HandshakeTimeout: -1,
	}
	_, _, err := d.Dial(httpToWs(s.URL), nil)
	if err == nil {
		t.Fatalf("Dial() did not return error")
	}
}

func TestDialBadScheme(t *testing.T) {
	s := httptest.NewServer(dialHandler{t})
	defer s.Close()
	_, _, err := DefaultDialer.Dial(s.URL, nil)
	if err == nil {
		t.Fatalf("Dial() did not return error")
	}
}
<|endoftext|>


<commit_before>package instana

import (
	"sync"
	"time"
)

// A SpanRecorder handles all of the `RawSpan` data generated via an
// associated `Tracer` (see `NewStandardTracer`) instance. It also names
// the containing process and provides access to a straightforward tag map.
type SpanRecorder interface {
	// Implementations must determine whether and where to store `span`.
	RecordSpan(span *spanS)
}

// Recorder accepts spans, processes and queues them
// for delivery to the backend.
type Recorder struct {
	sync.RWMutex
	spans    []jsonSpan
	testMode bool
}

// NewRecorder Establish a Recorder span recorder
func NewRecorder() *Recorder {
	r := new(Recorder)
	r.init()
	return r
}

// NewTestRecorder Establish a new span recorder used for testing
func NewTestRecorder() *Recorder {
	r := new(Recorder)
	r.testMode = true
	r.init()
	return r
}

func (r *Recorder) init() {
	r.clearQueuedSpans()

	if r.testMode {
		return
	}

	ticker := time.NewTicker(1 * time.Second)
	go func() {
		for range ticker.C {
			if sensor.agent.canSend() {
				r.send()
			}
		}
	}()
}

// RecordSpan accepts spans to be recorded and and added to the span queue
// for eventual reporting to the host agent.
func (r *Recorder) RecordSpan(span *spanS) {
	// If we're not announced and not in test mode then just
	// return
	if !r.testMode && !sensor.agent.canSend() {
		return
	}

	var data = &jsonData{}
	kind := span.getSpanKind()

	data.SDK = &jsonSDKData{
		Name:   span.Operation,
		Type:   kind,
		Custom: &jsonCustomData{Tags: span.Tags, Logs: span.collectLogs()}}

	baggage := make(map[string]string)
	span.context.ForeachBaggageItem(func(k string, v string) bool {
		baggage[k] = v

		return true
	})

	if len(baggage) > 0 {
		data.SDK.Custom.Baggage = baggage
	}

	data.Service = sensor.serviceName

	var parentID *int64
	if span.ParentSpanID == 0 {
		parentID = nil
	} else {
		parentID = &span.ParentSpanID
	}

	r.Lock()
	defer r.Unlock()

	if len(r.spans) == sensor.options.MaxBufferedSpans {
		r.spans = r.spans[1:]
	}

	r.spans = append(r.spans, jsonSpan{
		TraceID:   span.context.TraceID,
		ParentID:  parentID,
		SpanID:    span.context.SpanID,
		Timestamp: uint64(span.Start.UnixNano()) / uint64(time.Millisecond),
		Duration:  uint64(span.Duration) / uint64(time.Millisecond),
		Name:      "sdk",
		Error:     span.Error,
		Ec:        span.Ec,
		Lang:      "go",
		From:      sensor.agent.from,
		Data:      data})

	if r.testMode || !sensor.agent.canSend() {
		return
	}

	if len(r.spans) >= sensor.options.ForceTransmissionStartingAt {
		log.debug("Forcing spans to agent.  Count:", len(r.spans))
		r.send()
	}
}

// QueuedSpansCount returns the number of queued spans
//   Used only in tests currently.
func (r *Recorder) QueuedSpansCount() int {
	r.RLock()
	defer r.RUnlock()
	return len(r.spans)
}

// GetQueuedSpans returns a copy of the queued spans and clears the queue.
func (r *Recorder) GetQueuedSpans() []jsonSpan {
	r.Lock()
	defer r.Unlock()

	// Copy queued spans
	queuedSpans := make([]jsonSpan, len(r.spans))
	copy(queuedSpans, r.spans)

	// and clear out the source
	r.clearQueuedSpans()
	return queuedSpans
}

// clearQueuedSpans brings the span queue to empty/0/nada
//   This function doesn't take the Lock so make sure to have
//   the write lock before calling.
//   This is meant to be called from GetQueuedSpans which handles
//   locking.
func (r *Recorder) clearQueuedSpans() {
	var mbs int

	if len(r.spans) > 0 {
		if sensor != nil {
			mbs = sensor.options.MaxBufferedSpans
		} else {
			mbs = DefaultMaxBufferedSpans
		}
		r.spans = make([]jsonSpan, 0, mbs)
	}
}

// Retrieve the queued spans and post them to the host agent asynchronously.
func (r *Recorder) send() {
	spansToSend := r.GetQueuedSpans()
	if len(spansToSend) > 0 {
		go func() {
			_, err := sensor.agent.request(sensor.agent.makeURL(agentTracesURL), "POST", spansToSend)
			if err != nil {
				log.debug("Posting traces failed in send(): ", err)
				sensor.agent.reset()
			}
		}()
	}
}
<commit_msg>Force transmission as a goroutine<commit_after>package instana

import (
	"sync"
	"time"
)

// A SpanRecorder handles all of the `RawSpan` data generated via an
// associated `Tracer` (see `NewStandardTracer`) instance. It also names
// the containing process and provides access to a straightforward tag map.
type SpanRecorder interface {
	// Implementations must determine whether and where to store `span`.
	RecordSpan(span *spanS)
}

// Recorder accepts spans, processes and queues them
// for delivery to the backend.
type Recorder struct {
	sync.RWMutex
	spans    []jsonSpan
	testMode bool
}

// NewRecorder Establish a Recorder span recorder
func NewRecorder() *Recorder {
	r := new(Recorder)
	r.init()
	return r
}

// NewTestRecorder Establish a new span recorder used for testing
func NewTestRecorder() *Recorder {
	r := new(Recorder)
	r.testMode = true
	r.init()
	return r
}

func (r *Recorder) init() {
	r.clearQueuedSpans()

	if r.testMode {
		return
	}

	ticker := time.NewTicker(1 * time.Second)
	go func() {
		for range ticker.C {
			if sensor.agent.canSend() {
				r.send()
			}
		}
	}()
}

// RecordSpan accepts spans to be recorded and and added to the span queue
// for eventual reporting to the host agent.
func (r *Recorder) RecordSpan(span *spanS) {
	// If we're not announced and not in test mode then just
	// return
	if !r.testMode && !sensor.agent.canSend() {
		return
	}

	var data = &jsonData{}
	kind := span.getSpanKind()

	data.SDK = &jsonSDKData{
		Name:   span.Operation,
		Type:   kind,
		Custom: &jsonCustomData{Tags: span.Tags, Logs: span.collectLogs()}}

	baggage := make(map[string]string)
	span.context.ForeachBaggageItem(func(k string, v string) bool {
		baggage[k] = v

		return true
	})

	if len(baggage) > 0 {
		data.SDK.Custom.Baggage = baggage
	}

	data.Service = sensor.serviceName

	var parentID *int64
	if span.ParentSpanID == 0 {
		parentID = nil
	} else {
		parentID = &span.ParentSpanID
	}

	r.Lock()
	defer r.Unlock()

	if len(r.spans) == sensor.options.MaxBufferedSpans {
		r.spans = r.spans[1:]
	}

	r.spans = append(r.spans, jsonSpan{
		TraceID:   span.context.TraceID,
		ParentID:  parentID,
		SpanID:    span.context.SpanID,
		Timestamp: uint64(span.Start.UnixNano()) / uint64(time.Millisecond),
		Duration:  uint64(span.Duration) / uint64(time.Millisecond),
		Name:      "sdk",
		Error:     span.Error,
		Ec:        span.Ec,
		Lang:      "go",
		From:      sensor.agent.from,
		Data:      data})

	if r.testMode || !sensor.agent.canSend() {
		return
	}

	if len(r.spans) >= sensor.options.ForceTransmissionStartingAt {
		log.debug("Forcing spans to agent.  Count:", len(r.spans))
		go r.send()
	}
}

// QueuedSpansCount returns the number of queued spans
//   Used only in tests currently.
func (r *Recorder) QueuedSpansCount() int {
	r.RLock()
	defer r.RUnlock()
	return len(r.spans)
}

// GetQueuedSpans returns a copy of the queued spans and clears the queue.
func (r *Recorder) GetQueuedSpans() []jsonSpan {
	r.Lock()
	defer r.Unlock()

	// Copy queued spans
	queuedSpans := make([]jsonSpan, len(r.spans))
	copy(queuedSpans, r.spans)

	// and clear out the source
	r.clearQueuedSpans()
	return queuedSpans
}

// clearQueuedSpans brings the span queue to empty/0/nada
//   This function doesn't take the Lock so make sure to have
//   the write lock before calling.
//   This is meant to be called from GetQueuedSpans which handles
//   locking.
func (r *Recorder) clearQueuedSpans() {
	var mbs int

	if len(r.spans) > 0 {
		if sensor != nil {
			mbs = sensor.options.MaxBufferedSpans
		} else {
			mbs = DefaultMaxBufferedSpans
		}
		r.spans = make([]jsonSpan, 0, mbs)
	}
}

// Retrieve the queued spans and post them to the host agent asynchronously.
func (r *Recorder) send() {
	spansToSend := r.GetQueuedSpans()
	if len(spansToSend) > 0 {
		go func() {
			_, err := sensor.agent.request(sensor.agent.makeURL(agentTracesURL), "POST", spansToSend)
			if err != nil {
				log.debug("Posting traces failed in send(): ", err)
				sensor.agent.reset()
			}
		}()
	}
}
<|endoftext|>


<commit_before>package instana

import (
	"os"
	"time"

	"github.com/opentracing/basictracer-go"
	ext "github.com/opentracing/opentracing-go/ext"
)

type InstanaSpanRecorder struct {
}

type InstanaSpan struct {
	TraceId   uint64      `json:"t"`
	ParentId  *uint64     `json:"p,omitempty"`
	SpanId    uint64      `json:"s"`
	Timestamp uint64      `json:"ts"`
	Duration  uint64      `json:"d"`
	Name      string      `json:"n"`
	From      *FromS      `json:"f"`
	Data      interface{} `json:"data"`
}

func NewRecorder() *InstanaSpanRecorder {
	return new(InstanaSpanRecorder)
}

func getSpanLogField(rawSpan basictracer.RawSpan, field string) interface{} {
	for _, log := range rawSpan.Logs {
		for _, f := range log.Fields {
			if f.Key() == field {
				return f.Value()
			}
		}
	}

	return nil
}

func getStringSpanLogField(rawSpan basictracer.RawSpan, field string) string {
	d := getSpanLogField(rawSpan, field)
	if d == nil {
		return ""
	}

	return d.(string)
}

func getDataLogField(rawSpan basictracer.RawSpan) *Data {
	d := getSpanLogField(rawSpan, "data")
	if d != nil {
		return getSpanLogField(rawSpan, "data").(*Data)
	}

	return nil
}

func getTag(rawSpan basictracer.RawSpan, tag string) interface{} {
	return rawSpan.Tags[tag]
}

func getStringTag(rawSpan basictracer.RawSpan, tag string) string {
	d := getTag(rawSpan, tag)
	if d == nil {
		return ""
	}

	return d.(string)
}

func getHostName(rawSpan basictracer.RawSpan) string {
	hostTag := getStringTag(rawSpan, string(ext.PeerHostname))
	if hostTag != "" {
		return hostTag
	}

	h, err := os.Hostname()
	if err != nil {
		h = "localhost"
	}

	return h
}

func getServiceName(rawSpan basictracer.RawSpan) string {
	s := getStringTag(rawSpan, string(ext.Component))
	if s == "" {
		s = getStringTag(rawSpan, string(ext.PeerService))
		if s == "" {
			return sensor.serviceName
		}
	}

	return s
}

func getHTTPType(rawSpan basictracer.RawSpan) string {
	kind := getStringTag(rawSpan, string(ext.SpanKind))
	if kind == string(ext.SpanKindRPCServerEnum) {
		return HTTP_SERVER
	}

	return HTTP_CLIENT
}

func (r *InstanaSpanRecorder) RecordSpan(rawSpan basictracer.RawSpan) {
	//TODO: remove log field misuse after merge
	data := getDataLogField(rawSpan)
	tp := getStringSpanLogField(rawSpan, "type")
	if data == nil {
		h := getHostName(rawSpan)
		status := getTag(rawSpan, string(ext.HTTPStatusCode))
		if status != nil {
			tp = getHTTPType(rawSpan)
			data = &Data{Http: &HttpData{
				Host:   h,
				Url:    getStringTag(rawSpan, string(ext.HTTPUrl)),
				Method: getStringTag(rawSpan, string(ext.HTTPMethod)),
				Status: status.(int)}}
		} else {
			tp = RPC
			data = &Data{Rpc: &RpcData{
				Host: h,
				Call: rawSpan.Operation}}
		}
	}

	baggage := make(map[string]string)
	rawSpan.Context.ForeachBaggageItem(func(k string, v string) bool {
		baggage[k] = v

		return true
	})

	if len(baggage) > 0 {
		data.Baggage = baggage
	}

	data.Service = getServiceName(rawSpan)

	var parentId *uint64
	if rawSpan.ParentSpanID == 0 {
		parentId = nil
	} else {
		parentId = &rawSpan.ParentSpanID
	}

	if sensor.agent.canSend() {
		span := &InstanaSpan{
			TraceId:   rawSpan.Context.TraceID,
			ParentId:  parentId,
			SpanId:    rawSpan.Context.SpanID,
			Timestamp: uint64(rawSpan.Start.UnixNano()) / uint64(time.Millisecond),
			Duration:  uint64(rawSpan.Duration) / uint64(time.Millisecond),
			Name:      tp,
			From:      sensor.agent.from,
			Data:      &data}

		go sensor.agent.request(sensor.agent.makeUrl(AGENT_TRACES_URL), "POST", []interface{}{span})
	}
}
<commit_msg>actually, remove log misuse already<commit_after>package instana

import (
	"os"
	"time"

	"github.com/opentracing/basictracer-go"
	ext "github.com/opentracing/opentracing-go/ext"
)

type InstanaSpanRecorder struct {
}

type InstanaSpan struct {
	TraceId   uint64      `json:"t"`
	ParentId  *uint64     `json:"p,omitempty"`
	SpanId    uint64      `json:"s"`
	Timestamp uint64      `json:"ts"`
	Duration  uint64      `json:"d"`
	Name      string      `json:"n"`
	From      *FromS      `json:"f"`
	Data      interface{} `json:"data"`
}

func NewRecorder() *InstanaSpanRecorder {
	return new(InstanaSpanRecorder)
}

func getTag(rawSpan basictracer.RawSpan, tag string) interface{} {
	return rawSpan.Tags[tag]
}

func getStringTag(rawSpan basictracer.RawSpan, tag string) string {
	d := getTag(rawSpan, tag)
	if d == nil {
		return ""
	}

	return d.(string)
}

func getHostName(rawSpan basictracer.RawSpan) string {
	hostTag := getStringTag(rawSpan, string(ext.PeerHostname))
	if hostTag != "" {
		return hostTag
	}

	h, err := os.Hostname()
	if err != nil {
		h = "localhost"
	}

	return h
}

func getServiceName(rawSpan basictracer.RawSpan) string {
	s := getStringTag(rawSpan, string(ext.Component))
	if s == "" {
		s = getStringTag(rawSpan, string(ext.PeerService))
		if s == "" {
			return sensor.serviceName
		}
	}

	return s
}

func getHTTPType(rawSpan basictracer.RawSpan) string {
	kind := getStringTag(rawSpan, string(ext.SpanKind))
	if kind == string(ext.SpanKindRPCServerEnum) {
		return HTTP_SERVER
	}

	return HTTP_CLIENT
}

func (r *InstanaSpanRecorder) RecordSpan(rawSpan basictracer.RawSpan) {
	var data = &Data{}
	var tp string
	h := getHostName(rawSpan)
	status := getTag(rawSpan, string(ext.HTTPStatusCode))
	if status != nil {
		tp = getHTTPType(rawSpan)
		data = &Data{Http: &HttpData{
			Host:   h,
			Url:    getStringTag(rawSpan, string(ext.HTTPUrl)),
			Method: getStringTag(rawSpan, string(ext.HTTPMethod)),
			Status: status.(int)}}
	} else {
		tp = RPC
		data = &Data{Rpc: &RpcData{
			Host: h,
			Call: rawSpan.Operation}}
	}

	baggage := make(map[string]string)
	rawSpan.Context.ForeachBaggageItem(func(k string, v string) bool {
		baggage[k] = v

		return true
	})

	if len(baggage) > 0 {
		data.Baggage = baggage
	}

	data.Service = getServiceName(rawSpan)

	var parentId *uint64
	if rawSpan.ParentSpanID == 0 {
		parentId = nil
	} else {
		parentId = &rawSpan.ParentSpanID
	}

	if sensor.agent.canSend() {
		span := &InstanaSpan{
			TraceId:   rawSpan.Context.TraceID,
			ParentId:  parentId,
			SpanId:    rawSpan.Context.SpanID,
			Timestamp: uint64(rawSpan.Start.UnixNano()) / uint64(time.Millisecond),
			Duration:  uint64(rawSpan.Duration) / uint64(time.Millisecond),
			Name:      tp,
			From:      sensor.agent.from,
			Data:      &data}

		go sensor.agent.request(sensor.agent.makeUrl(AGENT_TRACES_URL), "POST", []interface{}{span})
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 Manu Martinez-Almeida.  All rights reserved.
// Use of this source code is governed by a MIT style
// license that can be found in the LICENSE file.

package ego

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http/httputil"
	"runtime"
)

var (
	dunno     = []byte("???")
	centerDot = []byte("·")
	dot       = []byte(".")
	slash     = []byte("/")
)

// Recovery returns a middleware that recovers from any panics and writes a 500 if there was one.
func Recovery() HandlerFunc {
	return RecoveryWithWriter(DefaultErrorWriter)
}

func RecoveryWithWriter(out io.Writer) HandlerFunc {
	var logger *log.Logger
	if out != nil {
		logger = log.New(out, "\n\n\x1b[31m", log.LstdFlags)
	}
	return func(c *Context) {
		defer func() {
			if err := recover(); err != nil {
				if logger != nil {
					stack := stack(3)
					httprequest, _ := httputil.DumpRequest(c.Request, false)
					logger.Printf("[Recovery] panic recovered:\n%s\n%s\n%s%s", string(httprequest), err, stack, reset)
				}
				c.AbortWithStatus(500)
			}
		}()
		c.Next()
	}
}

// stack returns a nicely formated stack frame, skipping skip frames.
func stack(skip int) []byte {
	buf := new(bytes.Buffer) // the returned data
	// As we loop, we open files and read them. These variables record the currently
	// loaded file.
	var lines [][]byte
	var lastFile string
	for i := skip; ; i++ { // Skip the expected number of frames
		pc, file, line, ok := runtime.Caller(i)
		if !ok {
			break
		}
		// Print this much at least.  If we can't find the source, it won't show.
		fmt.Fprintf(buf, "%s:%d (0x%x)\n", file, line, pc)
		if file != lastFile {
			data, err := ioutil.ReadFile(file)
			if err != nil {
				continue
			}
			lines = bytes.Split(data, []byte{'\n'})
			lastFile = file
		}
		fmt.Fprintf(buf, "\t%s: %s\n", function(pc), source(lines, line))
	}
	return buf.Bytes()
}

// source returns a space-trimmed slice of the n'th line.
func source(lines [][]byte, n int) []byte {
	n-- // in stack trace, lines are 1-indexed but our array is 0-indexed
	if n < 0 || n >= len(lines) {
		return dunno
	}
	return bytes.TrimSpace(lines[n])
}

// function returns, if possible, the name of the function containing the PC.
func function(pc uintptr) []byte {
	fn := runtime.FuncForPC(pc)
	if fn == nil {
		return dunno
	}
	name := []byte(fn.Name())
	// The name includes the path name to the package, which is unnecessary
	// since the file name is already included.  Plus, it has center dots.
	// That is, we see
	//	runtime/debug.*T·ptrmethod
	// and want
	//	*T.ptrmethod
	// Also the package path might contains dot (e.g. code.google.com/...),
	// so first eliminate the path prefix
	if lastslash := bytes.LastIndex(name, slash); lastslash >= 0 {
		name = name[lastslash+1:]
	}
	if period := bytes.Index(name, dot); period >= 0 {
		name = name[period+1:]
	}
	name = bytes.Replace(name, centerDot, dot, -1)
	return name
}
<commit_msg>fix doc<commit_after>// Copyright 2014 Manu Martinez-Almeida.  All rights reserved.
// Use of this source code is governed by a MIT style
// license that can be found in the LICENSE file.

package ego

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http/httputil"
	"runtime"
)

var (
	dunno     = []byte("???")
	centerDot = []byte("·")
	dot       = []byte(".")
	slash     = []byte("/")
)

// Recovery returns a middleware that recovers from any panics and writes a 500 if there was one.
func Recovery() HandlerFunc {
	return RecoveryWithWriter(DefaultErrorWriter)
}

func RecoveryWithWriter(out io.Writer) HandlerFunc {
	var logger *log.Logger
	if out != nil {
		logger = log.New(out, "\n\n\x1b[31m", log.LstdFlags)
	}
	return func(c *Context) {
		defer func() {
			if err := recover(); err != nil {
				if logger != nil {
					stack := stack(3)
					httprequest, _ := httputil.DumpRequest(c.Request, false)
					logger.Printf("[Recovery] panic recovered:\n%s\n%s\n%s%s", string(httprequest), err, stack, reset)
				}
				c.AbortWithStatus(500)
			}
		}()
		c.Next()
	}
}

// stack returns a nicely formatted stack frame, skipping skip frames.
func stack(skip int) []byte {
	buf := new(bytes.Buffer) // the returned data
	// As we loop, we open files and read them. These variables record the currently
	// loaded file.
	var lines [][]byte
	var lastFile string
	for i := skip; ; i++ { // Skip the expected number of frames
		pc, file, line, ok := runtime.Caller(i)
		if !ok {
			break
		}
		// Print this much at least.  If we can't find the source, it won't show.
		fmt.Fprintf(buf, "%s:%d (0x%x)\n", file, line, pc)
		if file != lastFile {
			data, err := ioutil.ReadFile(file)
			if err != nil {
				continue
			}
			lines = bytes.Split(data, []byte{'\n'})
			lastFile = file
		}
		fmt.Fprintf(buf, "\t%s: %s\n", function(pc), source(lines, line))
	}
	return buf.Bytes()
}

// source returns a space-trimmed slice of the n'th line.
func source(lines [][]byte, n int) []byte {
	n-- // in stack trace, lines are 1-indexed but our array is 0-indexed
	if n < 0 || n >= len(lines) {
		return dunno
	}
	return bytes.TrimSpace(lines[n])
}

// function returns, if possible, the name of the function containing the PC.
func function(pc uintptr) []byte {
	fn := runtime.FuncForPC(pc)
	if fn == nil {
		return dunno
	}
	name := []byte(fn.Name())
	// The name includes the path name to the package, which is unnecessary
	// since the file name is already included.  Plus, it has center dots.
	// That is, we see
	//	runtime/debug.*T·ptrmethod
	// and want
	//	*T.ptrmethod
	// Also the package path might contains dot (e.g. code.google.com/...),
	// so first eliminate the path prefix
	if lastslash := bytes.LastIndex(name, slash); lastslash >= 0 {
		name = name[lastslash+1:]
	}
	if period := bytes.Index(name, dot); period >= 0 {
		name = name[period+1:]
	}
	name = bytes.Replace(name, centerDot, dot, -1)
	return name
}
<|endoftext|>


<commit_before>// Copyright 2012 Jimmy Zelinskie. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package geddit

import (
	"fmt"
)

type Redditor struct {
	ID      string  `json:"id"`
	Name    string  `json:"name"`
	Created float32 `json:"created_utc"`
	Gold    bool    `json:"is_gold"`
	Mod     bool    `json:"is_mod"`
	Mail    bool    `json:"has_mail"`
	ModMail bool    `json:"has_mod_mail"`
	Karma
}

type Preferences struct {
	Research               bool   `json:"research"`
	ShowStylesheets        bool   `json:"show_stylesheets"`
	ShowLinkFlair          bool   `json:"show_link_flair"`
	ShowTrending           bool   `json:"show_trending"`
	PrivateFeeds           bool   `json:"private_feeds"`
	IgnoreSuggestedSort    bool   `json:"ignore_suggested_sort"`
	Media                  string `json:"media"`
	ClickGadget            bool   `json:"clickgadget"`
	LabelNSFW              bool   `json:"label_nsfw"`
	Over18                 bool   `json:"over_18"`
	EmailMessages          bool   `json:"email_messages"`
	HighlightControversial bool   `json:"highlight_controversial"`
	ForceHTTPS             bool   `json:"force_https"`
	Language               string `json:"lang"`
	HideFromRobots         bool   `json:"hide_from_robots"`
	PublicVotes            bool   `json:"public_votes"`
	ShowFlair              bool   `json:"show_flair"`
	HideAds                bool   `json:"hide_ads"`
	Beta                   bool   `json:"beta"`
	NewWindow              bool   `json:"newwindow"`
	LegacySearch           bool   `json:"legacy_search"`
}

type Friend struct {
	Date float32 `json:"date"`
	Name string  `json:"name"`
	ID   string  `json:"id"`
}

type Karma struct {
	CommentKarma int `json:"comment_karma"`
	LinkKarma    int `json:"link_karma"`
}

type Trophy struct {
	Name        string `json:"name"`
	Description string `json:"description"`
	Icon        string `json:"icon_70"`
}

// String returns the string representation of a reddit user.
func (r *Redditor) String() string {
	return fmt.Sprintf("%s (%d-%d)", r.Name, r.LinkKarma, r.CommentKarma)
}
<commit_msg>fixup! Use OAuth2 to make reddit API requests<commit_after>// Copyright 2012 Jimmy Zelinskie. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package geddit

import (
	"fmt"
)

type Redditor struct {
	ID      string  `json:"id"`
	Name    string  `json:"name"`
	Created float64 `json:"created_utc"`
	Gold    bool    `json:"is_gold"`
	Mod     bool    `json:"is_mod"`
	Mail    bool    `json:"has_mail"`
	ModMail bool    `json:"has_mod_mail"`
	Karma
}

type Preferences struct {
	Research               bool   `json:"research"`
	ShowStylesheets        bool   `json:"show_stylesheets"`
	ShowLinkFlair          bool   `json:"show_link_flair"`
	ShowTrending           bool   `json:"show_trending"`
	PrivateFeeds           bool   `json:"private_feeds"`
	IgnoreSuggestedSort    bool   `json:"ignore_suggested_sort"`
	Media                  string `json:"media"`
	ClickGadget            bool   `json:"clickgadget"`
	LabelNSFW              bool   `json:"label_nsfw"`
	Over18                 bool   `json:"over_18"`
	EmailMessages          bool   `json:"email_messages"`
	HighlightControversial bool   `json:"highlight_controversial"`
	ForceHTTPS             bool   `json:"force_https"`
	Language               string `json:"lang"`
	HideFromRobots         bool   `json:"hide_from_robots"`
	PublicVotes            bool   `json:"public_votes"`
	ShowFlair              bool   `json:"show_flair"`
	HideAds                bool   `json:"hide_ads"`
	Beta                   bool   `json:"beta"`
	NewWindow              bool   `json:"newwindow"`
	LegacySearch           bool   `json:"legacy_search"`
}

type Friend struct {
	Date float32 `json:"date"`
	Name string  `json:"name"`
	ID   string  `json:"id"`
}

type Karma struct {
	CommentKarma int `json:"comment_karma"`
	LinkKarma    int `json:"link_karma"`
}

type Trophy struct {
	Name        string `json:"name"`
	Description string `json:"description"`
	Icon        string `json:"icon_70"`
}

// String returns the string representation of a reddit user.
func (r *Redditor) String() string {
	return fmt.Sprintf("%s (%d-%d)", r.Name, r.LinkKarma, r.CommentKarma)
}
<|endoftext|>


<commit_before>package jira

import (
	// Stdlib
	"bytes"
	"errors"
	"fmt"
	"io"
	"text/tabwriter"

	// Internal
	"github.com/salsaflow/salsaflow/action"
	"github.com/salsaflow/salsaflow/errs"
	"github.com/salsaflow/salsaflow/log"
	"github.com/salsaflow/salsaflow/modules/common"
	"github.com/salsaflow/salsaflow/version"

	// Vendor
	"github.com/salsita/go-jira/v2/jira"
)

type runningRelease struct {
	tracker        *issueTracker
	releaseVersion *version.Version
	issues         []*jira.Issue
}

func newRunningRelease(
	tracker *issueTracker,
	releaseVersion *version.Version,
) (*runningRelease, error) {

	// Fetch relevant issues from JIRA.
	var (
		key          = tracker.config.ProjectKey()
		releaseLabel = releaseVersion.ReleaseTagString()
		task         = fmt.Sprintf("Fetch issues labeled with '%v'", releaseLabel)
	)
	log.Run(task)
	issues, err := issuesByLabel(newClient(tracker.config), key, releaseLabel)
	if err != nil {
		return nil, errs.NewError(task, err)
	}

	// Return a new release instance.
	return &runningRelease{tracker, releaseVersion, issues}, nil
}

func (release *runningRelease) Version() *version.Version {
	return release.releaseVersion
}

func (release *runningRelease) Stories() ([]common.Story, error) {
	return toCommonStories(release.issues, release.tracker), nil
}

func (release *runningRelease) EnsureStageable() error {
	versionString := release.releaseVersion.BaseString()

	var task = fmt.Sprintf(
		"Make sure that release %v can be staged", versionString)
	log.Run(task)

	var details bytes.Buffer
	tw := tabwriter.NewWriter(&details, 0, 8, 4, '\t', 0)
	io.WriteString(tw, "\n")
	io.WriteString(tw, "Issue Key\tError\n")
	io.WriteString(tw, "=========\t=====\n")

	var (
		err             error
		errNotStageable = errors.New("release cannot be staged")
	)
	for _, issue := range release.issues {
		if ex := ensureStageableIssue(issue); ex != nil {
			fmt.Fprintf(tw, "%v\t%v\n", issue.Key, ex)
			err = errNotStageable
		}
	}

	if err != nil {
		io.WriteString(tw, "\n")
		tw.Flush()
		return &common.ErrNotStageable{
			E: errs.NewErrorWithHint(task, err, details.String()),
		}
	}
	return nil
}

func (release *runningRelease) Stage() (action.Action, error) {

	var (
		api           = newClient(release.tracker.config)
		versionString = release.releaseVersion.BaseString()
		stageTask     = fmt.Sprintf("Stage JIRA issues associated with release %v", versionString)
	)
	log.Run(stageTask)

	// Make sure we only try to stage the issues that are in Tested.
	var issuesToStage []*jira.Issue
	for _, issue := range release.issues {
		if issue.Fields.Status.Id == stateIdTested {
			issuesToStage = append(issuesToStage, issue)
		}
	}

	// Perform the transition.
	err := performBulkTransition(api, issuesToStage, transitionIdStage, transitionIdUnstage)
	if err != nil {
		return nil, errs.NewError(stageTask, err)
	}

	return action.ActionFunc(func() error {
		log.Rollback(stageTask)
		unstageTask := fmt.Sprintf("Unstage JIRA issues associated with release %v", versionString)
		if err := performBulkTransition(api, issuesToStage, transitionIdUnstage, ""); err != nil {
			return errs.NewError(unstageTask, err)
		}
		return nil
	}), nil
}

func (release *runningRelease) EnsureReleasable() error {
	// Drop accepted issues.
	var notAccepted []*jira.Issue
IssueLoop:
	for _, issue := range release.issues {
		for _, id := range acceptedStateIds {
			if id == issue.Fields.Status.Id {
				continue IssueLoop
			}
		}
		notAccepted = append(notAccepted, issue)
	}

	// In case there is no open story, we are done.
	if len(notAccepted) == 0 {
		return nil
	}

	// Generate the error hint.
	var hint bytes.Buffer
	tw := tabwriter.NewWriter(&hint, 0, 8, 2, '\t', 0)
	fmt.Fprintf(tw, "\nThe following issues cannot be released:\n\n")
	fmt.Fprintf(tw, "Issue Key\tStatus\n")
	fmt.Fprintf(tw, "=========\t======\n")
	for _, issue := range notAccepted {
		fmt.Fprintf(tw, "%v\t%v\n", issue.Key, issue.Fields.Status.Name)
	}
	fmt.Fprintf(tw, "\n")
	tw.Flush()

	versionString := release.releaseVersion.BaseString()
	return &common.ErrNotReleasable{
		errs.NewErrorWithHint(
			fmt.Sprintf("Make sure release %v can be released", versionString),
			fmt.Errorf("release %v cannot be released", versionString),
			hint.String()),
	}
}

func (release *runningRelease) Release() error {
	if release.issues == nil {
		panic("bug(release.issues == nil)")
	}
	return performBulkTransition(
		newClient(release.tracker.config), release.issues, transitionIdRelease, "")
}

func ensureStageableIssue(issue *jira.Issue) error {
	// Check subtasks recursively.
	for _, subtask := range issue.Fields.Subtasks {
		if err := ensureStageableIssue(subtask); err != nil {
			return err
		}
	}

	// Check the issue itself.
	for _, id := range stageableStateIds {
		if issue.Fields.Status.Id == id {
			return nil
		}
	}
	return fmt.Errorf("issue %v: invalid state: %v", issue.Key, issue.Fields.Status.Name)
}
<commit_msg>jira: Release only stories where it makes sense<commit_after>package jira

import (
	// Stdlib
	"bytes"
	"errors"
	"fmt"
	"io"
	"text/tabwriter"

	// Internal
	"github.com/salsaflow/salsaflow/action"
	"github.com/salsaflow/salsaflow/errs"
	"github.com/salsaflow/salsaflow/log"
	"github.com/salsaflow/salsaflow/modules/common"
	"github.com/salsaflow/salsaflow/version"

	// Vendor
	"github.com/salsita/go-jira/v2/jira"
)

type runningRelease struct {
	tracker        *issueTracker
	releaseVersion *version.Version
	issues         []*jira.Issue
}

func newRunningRelease(
	tracker *issueTracker,
	releaseVersion *version.Version,
) (*runningRelease, error) {

	// Fetch relevant issues from JIRA.
	var (
		key          = tracker.config.ProjectKey()
		releaseLabel = releaseVersion.ReleaseTagString()
		task         = fmt.Sprintf("Fetch issues labeled with '%v'", releaseLabel)
	)
	log.Run(task)
	issues, err := issuesByLabel(newClient(tracker.config), key, releaseLabel)
	if err != nil {
		return nil, errs.NewError(task, err)
	}

	// Return a new release instance.
	return &runningRelease{tracker, releaseVersion, issues}, nil
}

func (release *runningRelease) Version() *version.Version {
	return release.releaseVersion
}

func (release *runningRelease) Stories() ([]common.Story, error) {
	return toCommonStories(release.issues, release.tracker), nil
}

func (release *runningRelease) EnsureStageable() error {
	versionString := release.releaseVersion.BaseString()

	var task = fmt.Sprintf(
		"Make sure that release %v can be staged", versionString)
	log.Run(task)

	var details bytes.Buffer
	tw := tabwriter.NewWriter(&details, 0, 8, 4, '\t', 0)
	io.WriteString(tw, "\n")
	io.WriteString(tw, "Issue Key\tError\n")
	io.WriteString(tw, "=========\t=====\n")

	var (
		err             error
		errNotStageable = errors.New("release cannot be staged")
	)
	for _, issue := range release.issues {
		if ex := ensureStageableIssue(issue); ex != nil {
			fmt.Fprintf(tw, "%v\t%v\n", issue.Key, ex)
			err = errNotStageable
		}
	}

	if err != nil {
		io.WriteString(tw, "\n")
		tw.Flush()
		return &common.ErrNotStageable{
			E: errs.NewErrorWithHint(task, err, details.String()),
		}
	}
	return nil
}

func (release *runningRelease) Stage() (action.Action, error) {

	var (
		api           = newClient(release.tracker.config)
		versionString = release.releaseVersion.BaseString()
		stageTask     = fmt.Sprintf("Stage JIRA issues associated with release %v", versionString)
	)
	log.Run(stageTask)

	// Make sure we only try to stage the issues that are in Tested.
	var issuesToStage []*jira.Issue
	for _, issue := range release.issues {
		if issue.Fields.Status.Id == stateIdTested {
			issuesToStage = append(issuesToStage, issue)
		}
	}

	// Perform the transition.
	err := performBulkTransition(api, issuesToStage, transitionIdStage, transitionIdUnstage)
	if err != nil {
		return nil, errs.NewError(stageTask, err)
	}

	return action.ActionFunc(func() error {
		log.Rollback(stageTask)
		unstageTask := fmt.Sprintf("Unstage JIRA issues associated with release %v", versionString)
		if err := performBulkTransition(api, issuesToStage, transitionIdUnstage, ""); err != nil {
			return errs.NewError(unstageTask, err)
		}
		return nil
	}), nil
}

func (release *runningRelease) EnsureReleasable() error {
	// Drop accepted issues.
	var notAccepted []*jira.Issue
IssueLoop:
	for _, issue := range release.issues {
		for _, id := range acceptedStateIds {
			if id == issue.Fields.Status.Id {
				continue IssueLoop
			}
		}
		notAccepted = append(notAccepted, issue)
	}

	// In case there is no open story, we are done.
	if len(notAccepted) == 0 {
		return nil
	}

	// Generate the error hint.
	var hint bytes.Buffer
	tw := tabwriter.NewWriter(&hint, 0, 8, 2, '\t', 0)
	fmt.Fprintf(tw, "\nThe following issues cannot be released:\n\n")
	fmt.Fprintf(tw, "Issue Key\tStatus\n")
	fmt.Fprintf(tw, "=========\t======\n")
	for _, issue := range notAccepted {
		fmt.Fprintf(tw, "%v\t%v\n", issue.Key, issue.Fields.Status.Name)
	}
	fmt.Fprintf(tw, "\n")
	tw.Flush()

	versionString := release.releaseVersion.BaseString()
	return &common.ErrNotReleasable{
		errs.NewErrorWithHint(
			fmt.Sprintf("Make sure release %v can be released", versionString),
			fmt.Errorf("release %v cannot be released", versionString),
			hint.String()),
	}
}

func (release *runningRelease) Release() error {
	// TODO: Get rid of this unholy ugliness.
	if release.issues == nil {
		panic("bug(release.issues == nil)")
	}

	// Release all issues that are accepted.
	issues := make([]*jira.Issue, 0, len(release.issues))
	for _, issue := range release.issues {
		if issue.Fields.Status.Id == stateIdAccepted {
			issues = append(issues, issue)
		}
	}
	if len(issues) == 0 {
		return nil
	}

	return performBulkTransition(
		newClient(release.tracker.config), issues, transitionIdRelease, "")
}

func ensureStageableIssue(issue *jira.Issue) error {
	// Check subtasks recursively.
	for _, subtask := range issue.Fields.Subtasks {
		if err := ensureStageableIssue(subtask); err != nil {
			return err
		}
	}

	// Check the issue itself.
	for _, id := range stageableStateIds {
		if issue.Fields.Status.Id == id {
			return nil
		}
	}
	return fmt.Errorf("issue %v: invalid state: %v", issue.Key, issue.Fields.Status.Name)
}
<|endoftext|>


<commit_before>package mail

import (
	"net/mail"
	"strings"
)

// Address represents a single mail address.
type Address mail.Address

// String formats the address as a valid RFC 5322 address. If the address's name
// contains non-ASCII characters the name will be rendered according to
// RFC 2047.
func (a *Address) String() string {
	return ((*mail.Address)(a)).String()
}

func parseAddressList(s string) ([]*Address, error) {
	list, err := mail.ParseAddressList(s)
	if err != nil {
		return nil, err
	}

	addrs := make([]*Address, len(list))
	for i, a := range list {
		addrs[i] = (*Address)(a)
	}
	return addrs, nil
}

func formatAddressList(l []*Address) string {
	formatted := make([]string, len(l))
	for i, a := range l {
		formatted[i] = a.String()
	}
	return strings.Join(formatted, ", ")
}
<commit_msg>mail: use message.CharsetReader in parseAddressList<commit_after>package mail

import (
	"mime"
	"net/mail"
	"strings"

	"github.com/emersion/go-message"
)

// Address represents a single mail address.
type Address mail.Address

// String formats the address as a valid RFC 5322 address. If the address's name
// contains non-ASCII characters the name will be rendered according to
// RFC 2047.
func (a *Address) String() string {
	return ((*mail.Address)(a)).String()
}

func parseAddressList(s string) ([]*Address, error) {
	parser := mail.AddressParser{
		&mime.WordDecoder{message.CharsetReader},
	}
	list, err := parser.ParseList(s)
	if err != nil {
		return nil, err
	}

	addrs := make([]*Address, len(list))
	for i, a := range list {
		addrs[i] = (*Address)(a)
	}
	return addrs, nil
}

func formatAddressList(l []*Address) string {
	formatted := make([]string, len(l))
	for i, a := range l {
		formatted[i] = a.String()
	}
	return strings.Join(formatted, ", ")
}
<|endoftext|>


<commit_before>package dropp

import (
	"bytes"
	"encoding/csv"
	"log"
	"net/http"
	"strconv"
	"time"

	gae "google.golang.org/appengine"
	db "google.golang.org/appengine/datastore"
	mail "google.golang.org/appengine/mail"
	tq "google.golang.org/appengine/taskqueue"
	ufe "google.golang.org/appengine/urlfetch"
	usr "google.golang.org/appengine/user"
	tmpl "html/template"
)

// API contains all data needed for the API
type API struct {
	CurrentUser *usr.User
	LogoutURL   string
}

// Item models items sold by Dropp
type Item struct {
	SourceURL string
	DataURL   string
	EbayID    string
	ItemName  string
	CreatedAt time.Time
	UpdatedAt time.Time
	IsActive  bool
}

// HomeData contains all data needed by the home page template
type HomeData struct {
	CurrentUser *usr.User
	LogoutURL   string
	ItemCount   int
}

func (a *API) newHomeData() HomeData {
	return HomeData{
		CurrentUser: a.CurrentUser,
		LogoutURL:   a.LogoutURL,
	}
}

// EmailData contains all data needed by the availability report email
type EmailData struct {
	LastUpdate string
	SnapDiffs  []SnapshotDiff
}

func (a *API) newEmailData(r *http.Request) EmailData {

	var (
		items            []Item
		snapshotDiffs    []SnapshotDiff
		currentSnapshots []Snapshot
		newDiff          SnapshotDiff
	)

	ctx := gae.NewContext(r)

	_, err := db.NewQuery("Item").GetAll(ctx, &items)
	for _, item := range items {

		currentSnapshots = []Snapshot{}
		// Get last two snaphots
		itemKey := db.NewKey(ctx, "Item", item.SourceURL, 0, nil)
		_, err =
			db.
				NewQuery("Snapshot").
				Ancestor(itemKey).
				Order("-CreatedAt").
				Limit(2).
				GetAll(ctx, &currentSnapshots)
		if err != nil {
			panic(err.Error())
		}

		// If item does not have 2 snaphots (it was just added) skip it
		if len(currentSnapshots) < 2 {
			continue
		}

		// Check if a new diff is needed
		//
		// ATTENTION!!!!!
		// As the qurey is ordered in descending order the first element of the
		// list is the MOST RECENT!!!!
		if currentSnapshots[0].Availability != currentSnapshots[1].Availability ||
			currentSnapshots[0].OnEbay != currentSnapshots[1].OnEbay ||
			currentSnapshots[0].Price != currentSnapshots[1].Price {

			newDiff = SnapshotDiff{
				ItemName:        item.ItemName,
				ItemURL:         item.SourceURL,
				EbayID:          item.EbayID,
				PreviousAva:     currentSnapshots[1].Availability,
				PreviousAvaComp: NewAva(currentSnapshots[1].Availability),
				CurrentAva:      currentSnapshots[0].Availability,
				CurrentAvaComp:  NewAva(currentSnapshots[0].Availability),
				PreviousStatus:  currentSnapshots[1].OnEbay,
				CurrentStatus:   currentSnapshots[0].OnEbay,
				PreviousPrice:   currentSnapshots[1].Price,
				CurrentPrice:    currentSnapshots[0].Price,
			}

			snapshotDiffs = append(snapshotDiffs, newDiff)

		}
	}
	if err != nil {
		panic(err.Error())
	}

	return EmailData{
		LastUpdate: time.Now().Format("02/01/2006 - 15:04"),
		SnapDiffs:  snapshotDiffs,
	}
}

func newAPI() *API {
	return &API{}
}

func (a *API) snapshot(w http.ResponseWriter, r *http.Request) {
	emailData := a.newEmailData(r)

	t, err := getEmailTemplate()

	if err != nil {
		panic(err.Error())
	}
	err = t.Execute(w, emailData)
	if err != nil {
		panic(err.Error())
	}
}

func (a *API) homePage(w http.ResponseWriter, r *http.Request) {
	ctx := gae.NewContext(r)
	homeData := a.newHomeData()
	itemCount, err := db.NewQuery("Item").Count(ctx)
	homeData.ItemCount = itemCount

	t, err := tmpl.ParseFiles("templates/home.html")
	if err != nil {
		panic(err.Error())
	}
	err = t.Execute(w, homeData)
	if err != nil {
		panic(err.Error())
	}
}

func (a *API) uploadCSV(w http.ResponseWriter, r *http.Request) {
	var (
		item    Item
		itemKey *db.Key
		items   []Item
		keys    []*db.Key
	)
	ctx := gae.NewContext(r)
	err := r.ParseMultipartForm(32 << 20)
	if err != nil {
		panic(err.Error())
	}

	file, _, err := r.FormFile("uploadfile")
	if err != nil {
		panic(err.Error())
	}
	defer func() {
		err = file.Close()
		if err != nil {
			log.Printf("Error while trying to close CSV file: %s", err)
			panic(err.Error())
		}
	}()

	csvFileContent, err := csv.NewReader(file).ReadAll()
	if err != nil {
		panic(err.Error())
	}

	for _, row := range csvFileContent {

		isActive, err := strconv.ParseBool(row[3])
		if err != nil {
			log.Printf("Error while trying to load row %s, error: %s", row[0], err)
			panic(err.Error())
		}

		// Create Item and append it to the array
		item = Item{
			ItemName:  row[0],
			SourceURL: row[1],
			EbayID:    row[2],
			IsActive:  isActive,
			DataURL:   row[4],
		}
		items = append(items, item)

		// Create item key
		itemKey = db.NewKey(ctx, "Item", item.SourceURL, 0, nil)
		keys = append(keys, itemKey)
	}

	_, err = db.PutMulti(ctx, keys, items)
	if err != nil {
		panic(err.Error())
	}
	http.Redirect(w, r, "/", http.StatusFound)
}

func (a *API) createSnapshotsTasks(w http.ResponseWriter, r *http.Request) {
	ctx := gae.NewContext(r)

	createSnapshotsTask := tq.NewPOSTTask(
		"/create_snapshots", map[string][]string{})

	sendReportEmailTask := tq.NewPOSTTask(
		"/send_report_email", map[string][]string{})

	allTasks := []*tq.Task{createSnapshotsTask, sendReportEmailTask}
	_, err := tq.AddMulti(ctx, allTasks, "snapshot")

	if err != nil {
		log.Printf("Error while trying to add task: %s", err)
	} else {
		log.Printf("Starting new update snapshot task %s", createSnapshotsTask.Name)
	}

	if err != nil {
		panic(err.Error())
	}
	http.Redirect(w, r, "/", http.StatusFound)
}

func (a *API) sendReportEmail(w http.ResponseWriter, r *http.Request) {
	ctx := gae.NewContext(r)

	var body bytes.Buffer

	t, err := getEmailTemplate()
	if err != nil {
		log.Printf("Error in generating template: %s", err)
		return
	}

	err = t.Execute(&body, a.newEmailData(r))
	if err != nil {
		log.Printf("Error in executing template: %s", err)
		return
	}

	mgs := &mail.Message{
		Sender:   conf.EmailSender,
		To:       conf.EmailRecipients,
		Subject:  conf.EmailSubject,
		HTMLBody: body.String(),
	}

	err = mail.Send(ctx, mgs)
	if err != nil {
		panic(err.Error())
	}
}

func (a *API) createSnapshots(w http.ResponseWriter, r *http.Request) {

	ebayServURL := conf.EbayServURL

	// Fetch the page
	ctx := gae.NewContext(r)
	items := make([]Item, 0, 10)
	_, err := db.
		NewQuery("Item").
		Filter("IsActive =", true).
		GetAll(ctx, &items)

	for _, item := range items {
		log.Printf("Updating snapshot for item %s", item.SourceURL)
		client := ufe.Client(ctx)
		sourceResp, err := client.Get(item.DataURL)

		if err != nil {
			log.Printf("Error while fetching item data %s", err)
			continue
		} else {

			defer func() {
				err := sourceResp.Body.Close()
				if err != nil {
					log.Printf("Error while closing BG page %s", err)
				}
			}()

		}

		ebayResp, err := client.Get(ebayServURL + item.EbayID)
		if err != nil {
			log.Printf("Error while fetching Ebay item status %s", err)
			continue
		} else {

			defer func() {
				err := ebayResp.Body.Close()
				if err != nil {
					log.Printf("Error while closing BG page %s", err)
				}
			}()
		}

		itemKey := db.NewKey(ctx, "Item", item.SourceURL, 0, nil)
		snap := &Snapshot{}
		snap.OnEbay = OffEbay
		snap.CreatedAt = time.Now()
		snap.getSourceData(sourceResp)
		snap.getEbayStatus(ebayResp)
		snapKey := db.NewIncompleteKey(ctx, "Snapshot", itemKey)
		_, err = db.Put(ctx, snapKey, snap)
	}
	if err != nil {
		panic(err.Error())
	}
}

func getEmailTemplate() (*tmpl.Template, error) {
	emailFunctions := tmpl.FuncMap{
		"avaCol":  AvaColor,
		"ebayCol": EbayStatusColor,
	}

	return tmpl.
		New("email.html").
		Funcs(emailFunctions).
		ParseFiles("templates/email.html")
}
<commit_msg>[DPP-112] Changed snapshot creation predicate<commit_after>package dropp

import (
	"bytes"
	"encoding/csv"
	"log"
	"net/http"
	"strconv"
	"time"

	gae "google.golang.org/appengine"
	db "google.golang.org/appengine/datastore"
	mail "google.golang.org/appengine/mail"
	tq "google.golang.org/appengine/taskqueue"
	ufe "google.golang.org/appengine/urlfetch"
	usr "google.golang.org/appengine/user"
	tmpl "html/template"
)

// API contains all data needed for the API
type API struct {
	CurrentUser *usr.User
	LogoutURL   string
}

// Item models items sold by Dropp
type Item struct {
	SourceURL string
	DataURL   string
	EbayID    string
	ItemName  string
	CreatedAt time.Time
	UpdatedAt time.Time
	IsActive  bool
}

// HomeData contains all data needed by the home page template
type HomeData struct {
	CurrentUser *usr.User
	LogoutURL   string
	ItemCount   int
}

func (a *API) newHomeData() HomeData {
	return HomeData{
		CurrentUser: a.CurrentUser,
		LogoutURL:   a.LogoutURL,
	}
}

// EmailData contains all data needed by the availability report email
type EmailData struct {
	LastUpdate string
	SnapDiffs  []SnapshotDiff
}

func (a *API) newEmailData(r *http.Request) EmailData {

	var (
		items            []Item
		snapshotDiffs    []SnapshotDiff
		currentSnapshots []Snapshot
		newDiff          SnapshotDiff
	)

	ctx := gae.NewContext(r)

	_, err := db.NewQuery("Item").GetAll(ctx, &items)
	for _, item := range items {

		currentSnapshots = []Snapshot{}
		// Get last two snaphots
		itemKey := db.NewKey(ctx, "Item", item.SourceURL, 0, nil)
		_, err = db.
			NewQuery("Snapshot").
			Ancestor(itemKey).
			Order("-CreatedAt").
			Limit(2).
			GetAll(ctx, &currentSnapshots)

		if err != nil {
			panic(err.Error())
		}

		// If item does not have 2 snaphots (it was just added) skip it
		if len(currentSnapshots) < 2 {
			continue
		}

		// Check if a new diff is needed
		//
		// ATTENTION!!!!!
		// As the qurey is ordered in descending order the first element of the
		// list is the MOST RECENT!!!!
		currentAva := NewAva(currentSnapshots[0].Availability)
		previousAva := NewAva(currentSnapshots[1].Availability)

		if currentAva != previousAva ||
			currentSnapshots[0].OnEbay != currentSnapshots[1].OnEbay ||
			currentSnapshots[0].Price != currentSnapshots[1].Price {

			newDiff = SnapshotDiff{
				ItemName:        item.ItemName,
				ItemURL:         item.SourceURL,
				EbayID:          item.EbayID,
				PreviousAva:     currentSnapshots[1].Availability,
				PreviousAvaComp: NewAva(currentSnapshots[1].Availability),
				CurrentAva:      currentSnapshots[0].Availability,
				CurrentAvaComp:  NewAva(currentSnapshots[0].Availability),
				PreviousStatus:  currentSnapshots[1].OnEbay,
				CurrentStatus:   currentSnapshots[0].OnEbay,
				PreviousPrice:   currentSnapshots[1].Price,
				CurrentPrice:    currentSnapshots[0].Price,
			}

			snapshotDiffs = append(snapshotDiffs, newDiff)

		}
	}
	if err != nil {
		panic(err.Error())
	}

	return EmailData{
		LastUpdate: time.Now().Format("02/01/2006 - 15:04"),
		SnapDiffs:  snapshotDiffs,
	}
}

func newAPI() *API {
	return &API{}
}

func (a *API) snapshot(w http.ResponseWriter, r *http.Request) {
	emailData := a.newEmailData(r)

	t, err := getEmailTemplate()

	if err != nil {
		panic(err.Error())
	}
	err = t.Execute(w, emailData)
	if err != nil {
		panic(err.Error())
	}
}

func (a *API) homePage(w http.ResponseWriter, r *http.Request) {
	ctx := gae.NewContext(r)
	homeData := a.newHomeData()
	itemCount, err := db.NewQuery("Item").Count(ctx)
	homeData.ItemCount = itemCount

	t, err := tmpl.ParseFiles("templates/home.html")
	if err != nil {
		panic(err.Error())
	}
	err = t.Execute(w, homeData)
	if err != nil {
		panic(err.Error())
	}
}

func (a *API) uploadCSV(w http.ResponseWriter, r *http.Request) {
	var (
		item    Item
		itemKey *db.Key
		items   []Item
		keys    []*db.Key
	)
	ctx := gae.NewContext(r)
	err := r.ParseMultipartForm(32 << 20)
	if err != nil {
		panic(err.Error())
	}

	file, _, err := r.FormFile("uploadfile")
	if err != nil {
		panic(err.Error())
	}
	defer func() {
		err = file.Close()
		if err != nil {
			log.Printf("Error while trying to close CSV file: %s", err)
			panic(err.Error())
		}
	}()

	csvFileContent, err := csv.NewReader(file).ReadAll()
	if err != nil {
		panic(err.Error())
	}

	for _, row := range csvFileContent {

		isActive, err := strconv.ParseBool(row[3])
		if err != nil {
			log.Printf("Error while trying to load row %s, error: %s", row[0], err)
			panic(err.Error())
		}

		// Create Item and append it to the array
		item = Item{
			ItemName:  row[0],
			SourceURL: row[1],
			EbayID:    row[2],
			IsActive:  isActive,
			DataURL:   row[4],
		}
		items = append(items, item)

		// Create item key
		itemKey = db.NewKey(ctx, "Item", item.SourceURL, 0, nil)
		keys = append(keys, itemKey)
	}

	_, err = db.PutMulti(ctx, keys, items)
	if err != nil {
		panic(err.Error())
	}
	http.Redirect(w, r, "/", http.StatusFound)
}

func (a *API) createSnapshotsTasks(w http.ResponseWriter, r *http.Request) {
	ctx := gae.NewContext(r)

	createSnapshotsTask := tq.NewPOSTTask(
		"/create_snapshots", map[string][]string{})

	sendReportEmailTask := tq.NewPOSTTask(
		"/send_report_email", map[string][]string{})

	allTasks := []*tq.Task{createSnapshotsTask, sendReportEmailTask}
	_, err := tq.AddMulti(ctx, allTasks, "snapshot")

	if err != nil {
		log.Printf("Error while trying to add task: %s", err)
	} else {
		log.Printf("Starting new update snapshot task %s", createSnapshotsTask.Name)
	}

	if err != nil {
		panic(err.Error())
	}
	http.Redirect(w, r, "/", http.StatusFound)
}

func (a *API) sendReportEmail(w http.ResponseWriter, r *http.Request) {
	ctx := gae.NewContext(r)

	var body bytes.Buffer

	t, err := getEmailTemplate()
	if err != nil {
		log.Printf("Error in generating template: %s", err)
		return
	}

	err = t.Execute(&body, a.newEmailData(r))
	if err != nil {
		log.Printf("Error in executing template: %s", err)
		return
	}

	mgs := &mail.Message{
		Sender:   conf.EmailSender,
		To:       conf.EmailRecipients,
		Subject:  conf.EmailSubject,
		HTMLBody: body.String(),
	}

	err = mail.Send(ctx, mgs)
	if err != nil {
		panic(err.Error())
	}
}

func (a *API) createSnapshots(w http.ResponseWriter, r *http.Request) {

	ebayServURL := conf.EbayServURL

	// Fetch the page
	ctx := gae.NewContext(r)
	items := make([]Item, 0, 10)
	_, err := db.
		NewQuery("Item").
		Filter("IsActive =", true).
		GetAll(ctx, &items)

	for _, item := range items {
		log.Printf("Updating snapshot for item %s", item.SourceURL)
		client := ufe.Client(ctx)
		sourceResp, err := client.Get(item.DataURL)

		if err != nil {
			log.Printf("Error while fetching item data %s", err)
			continue
		} else {

			defer func() {
				err := sourceResp.Body.Close()
				if err != nil {
					log.Printf("Error while closing BG page %s", err)
				}
			}()

		}

		ebayResp, err := client.Get(ebayServURL + item.EbayID)
		if err != nil {
			log.Printf("Error while fetching Ebay item status %s", err)
			continue
		} else {

			defer func() {
				err := ebayResp.Body.Close()
				if err != nil {
					log.Printf("Error while closing BG page %s", err)
				}
			}()
		}

		itemKey := db.NewKey(ctx, "Item", item.SourceURL, 0, nil)
		snap := &Snapshot{}
		snap.OnEbay = OffEbay
		snap.CreatedAt = time.Now()
		snap.getSourceData(sourceResp)
		snap.getEbayStatus(ebayResp)
		snapKey := db.NewIncompleteKey(ctx, "Snapshot", itemKey)
		_, err = db.Put(ctx, snapKey, snap)
	}
	if err != nil {
		panic(err.Error())
	}
}

func getEmailTemplate() (*tmpl.Template, error) {
	emailFunctions := tmpl.FuncMap{
		"avaCol":  AvaColor,
		"ebayCol": EbayStatusColor,
	}

	return tmpl.
		New("email.html").
		Funcs(emailFunctions).
		ParseFiles("templates/email.html")
}
<|endoftext|>


<commit_before>package main

import (
	"golang.org/x/net/icmp"
	"golang.org/x/net/ipv4"
	"io/ioutil"
	"log"
	"math"
	"math/rand"
	"net"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"
)

type networkMessage struct {
	msg       []byte
	msgType   uint8
	queueable bool
	ts        time.Time
}

type networkConnection struct {
	Conn         *net.UDPConn
	Ip           string
	Port         uint32
	Capability   uint8
	messageQueue [][]byte // Device message queue.
	/*
		Sleep mode/throttle variables. "sleep mode" is actually now just a very reduced packet rate, since we don't know positively
		 when a client is ready to accept packets - we just assume so if we don't receive ICMP Unreachable packets in 5 secs.
	*/
	lastUnreachable time.Time // Last time the device sent an ICMP Unreachable packet.
	nextMessageTime time.Time // The next time that the device is "able" to receive a message.
	numOverflows    uint32    // Number of times the queue has overflowed - for calculating the amount to chop off from the queue.
}

var messageQueue chan networkMessage
var outSockets map[string]networkConnection
var dhcpLeases map[string]string
var netMutex *sync.Mutex

var totalNetworkMessagesSent uint32

var pingResponse map[string]time.Time // Last time an IP responded to an "echo" response.

const (
	NETWORK_GDL90_STANDARD = 1
	NETWORK_AHRS_FFSIM     = 2
	NETWORK_AHRS_GDL90     = 4
	dhcp_lease_file        = "/var/lib/dhcp/dhcpd.leases"
)

// Read the "dhcpd.leases" file and parse out IP/hostname.
func getDHCPLeases() (map[string]string, error) {
	dat, err := ioutil.ReadFile(dhcp_lease_file)
	ret := make(map[string]string)
	if err != nil {
		return ret, err
	}
	lines := strings.Split(string(dat), "\n")
	open_block := false
	block_ip := ""
	for _, line := range lines {
		spaced := strings.Split(line, " ")
		if len(spaced) > 2 && spaced[0] == "lease" {
			open_block = true
			block_ip = spaced[1]
		} else if open_block && len(spaced) >= 4 && spaced[2] == "client-hostname" {
			hostname := strings.TrimRight(strings.TrimLeft(strings.Join(spaced[3:], " "), "\""), "\";")
			ret[block_ip] = hostname
			open_block = false
		} else if open_block && strings.HasPrefix(spaced[0], "}") { // No hostname.
			open_block = false
			ret[block_ip] = ""
		}
	}
	return ret, nil
}

func isSleeping(k string) bool {
	ipAndPort := strings.Split(k, ":")
	lastPing, ok := pingResponse[ipAndPort[0]]
	// No ping response. Assume disconnected/sleeping device.
	if !ok || time.Since(lastPing) > (10*time.Second) {
		return true
	}
	if time.Since(outSockets[k].lastUnreachable) < (5 * time.Second) {
		return true
	}
	return false
}

// Throttle mode for testing port open and giving some start-up time to the app.
// Throttling is 0.1% data rate for first 15 seconds.
func isThrottled(k string) bool {
	return (rand.Int()%1000 != 0) && time.Since(outSockets[k].lastUnreachable) < (15*time.Second)
}

func sendToAllConnectedClients(msg networkMessage) {
	netMutex.Lock()
	defer netMutex.Unlock()
	for k, netconn := range outSockets {
		// Check if this port is able to accept the type of message we're sending.
		if (netconn.Capability & msg.msgType) == 0 {
			continue
		}
		// Send non-queueable messages immediately, or discard if the client is in sleep mode.
		if !msg.queueable {
			if !isSleeping(k) {
				netconn.Conn.Write(msg.msg) // Write immediately.
				totalNetworkMessagesSent++
			}
		} else {
			if !isSleeping(k) {
				netconn.numOverflows = 0 // Reset the overflow counter whenever the client is not sleeping so that we're not penalizing future sleepmodes.
			}
			// Queue the message if the message is "queueable".
			if len(netconn.messageQueue) >= maxUserMsgQueueSize { // Too many messages queued? Drop the oldest.
				log.Printf("%s:%d - message queue overflow.\n", netconn.Ip, netconn.Port)
				netconn.numOverflows++
				s := 2 * netconn.numOverflows // Double the amount we chop off on each overflow.
				if int(s) >= len(netconn.messageQueue) {
					netconn.messageQueue = make([][]byte, 0)
				} else {
					netconn.messageQueue = netconn.messageQueue[s:]
				}
			}
			netconn.messageQueue = append(netconn.messageQueue, msg.msg)
			outSockets[k] = netconn
		}
	}
}

// Just returns the number of DHCP leases for now.
func getNetworkStats() uint {
	ret := uint(len(dhcpLeases))
	globalStatus.Connected_Users = ret
	return ret
}

// See who has a DHCP lease and make a UDP connection to each of them.
func refreshConnectedClients() {
	netMutex.Lock()
	defer netMutex.Unlock()
	validConnections := make(map[string]bool)
	t, err := getDHCPLeases()
	if err != nil {
		log.Printf("getDHCPLeases(): %s\n", err.Error())
		return
	}
	dhcpLeases = t
	// Client connected that wasn't before.
	for ip, hostname := range dhcpLeases {
		for _, networkOutput := range globalSettings.NetworkOutputs {
			ipAndPort := ip + ":" + strconv.Itoa(int(networkOutput.Port))
			if _, ok := outSockets[ipAndPort]; !ok {
				log.Printf("client connected: %s:%d (%s).\n", ip, networkOutput.Port, hostname)
				addr, err := net.ResolveUDPAddr("udp", ipAndPort)
				if err != nil {
					log.Printf("ResolveUDPAddr(%s): %s\n", ipAndPort, err.Error())
					continue
				}
				outConn, err := net.DialUDP("udp", nil, addr)
				if err != nil {
					log.Printf("DialUDP(%s): %s\n", ipAndPort, err.Error())
					continue
				}
				newq := make([][]byte, 0)
				outSockets[ipAndPort] = networkConnection{Conn: outConn, Ip: ip, Port: networkOutput.Port, Capability: networkOutput.Capability, messageQueue: newq}
			}
			validConnections[ipAndPort] = true
		}
	}
	// Client that was connected before that isn't.
	for ipAndPort, conn := range outSockets {
		if _, ok := validConnections[ipAndPort]; !ok {
			log.Printf("removed connection %s.\n", ipAndPort)
			conn.Conn.Close()
			delete(outSockets, ipAndPort)
		}
	}
}

func messageQueueSender() {
	secondTimer := time.NewTicker(5 * time.Second)

	queueTimer := time.NewTicker(1 * time.Second)
	//	queueTimer := time.NewTicker(400 * time.Microsecond) // 2500 msg/sec

	var lastQueueTimeChange time.Time // Reevaluate	send frequency every 5 seconds.
	for {
		select {
		case msg := <-messageQueue:
			sendToAllConnectedClients(msg)
		case <-queueTimer.C:
			netMutex.Lock()

			averageSendableQueueSize := float64(0.0)
			for k, netconn := range outSockets {
				if len(netconn.messageQueue) > 0 && !isSleeping(k) && !isThrottled(k) {
					averageSendableQueueSize += float64(len(netconn.messageQueue)) // Add num sendable messages.

					tmpConn := netconn
					tmpConn.Conn.Write(tmpConn.messageQueue[0])
					totalNetworkMessagesSent++
					tmpConn.messageQueue = tmpConn.messageQueue[1:]
					outSockets[k] = tmpConn
				}
			}
			if time.Since(lastQueueTimeChange) >= 5 * time.Second {
				var pd float64
				if averageSendableQueueSize > 0.0 && len(outSockets) > 0 {
					averageSendableQueueSize = averageSendableQueueSize / float64(len(outSockets)) // It's a total, not an average, up until this point.
					pd = math.Max(float64(1.0/2500.0), float64(1.0/(4.0*averageSendableQueueSize))) // Say 250msg is enough to get through the whole queue.
				} else {
					pd = float64(1.0/2500.0)
				}
				queueTimer.Stop()
				queueTimer = time.NewTicker(int64(pd * 1000000000.0) * time.Nanosecond)
				lastQueueTimeChange = time.Now()
			}
			averageSendableQueueSize
			netMutex.Unlock()
		case <-secondTimer.C:
			getNetworkStats()
		}
	}
}

func sendMsg(msg []byte, msgType uint8, queueable bool) {
	messageQueue <- networkMessage{msg: msg, msgType: msgType, queueable: queueable, ts: time.Now()}
}

func sendGDL90(msg []byte, queueable bool) {
	sendMsg(msg, NETWORK_GDL90_STANDARD, queueable)
}

func monitorDHCPLeases() {
	//TODO: inotify or dhcp event hook.
	timer := time.NewTicker(30 * time.Second)
	for {
		select {
		case <-timer.C:
			refreshConnectedClients()
		}
	}
}

func icmpEchoSender(c *icmp.PacketConn) {
	timer := time.NewTicker(5 * time.Second)
	for {
		<-timer.C
		// Collect IPs.
		ips := make(map[string]bool)
		for k, _ := range outSockets {
			ipAndPort := strings.Split(k, ":")
			ips[ipAndPort[0]] = true
		}
		// Send to all IPs.
		for ip, _ := range ips {
			wm := icmp.Message{
				Type: ipv4.ICMPTypeEcho, Code: 0,
				Body: &icmp.Echo{
					ID: os.Getpid() & 0xffff, Seq: 1,
					Data: []byte("STRATUX"),
				},
			}
			wb, err := wm.Marshal(nil)
			if err != nil {
				log.Printf("couldn't send ICMP Echo: %s\n", err.Error())
				continue
			}
			if _, err := c.WriteTo(wb, &net.IPAddr{IP: net.ParseIP(ip)}); err != nil {
				log.Printf("couldn't send ICMP Echo: %s\n", err.Error())
				continue
			}
			totalNetworkMessagesSent++
		}
	}
}

// Monitor clients going in/out of sleep mode via ICMP unreachable packets.
func sleepMonitor() {
	c, err := icmp.ListenPacket("ip4:icmp", "0.0.0.0")
	if err != nil {
		log.Printf("error listening for udp - sending data to all ports for all connected clients. err: %s", err)
		return
	}
	go icmpEchoSender(c)
	defer c.Close()
	for {
		buf := make([]byte, 1500)
		n, peer, err := c.ReadFrom(buf)
		if err != nil {
			log.Printf("%s\n", err.Error())
			continue
		}
		msg, err := icmp.ParseMessage(1, buf[:n])
		if err != nil {
			continue
		}

		ip := peer.String()

		// Look for echo replies, mark it as received.
		if msg.Type == ipv4.ICMPTypeEchoReply {
			pingResponse[ip] = time.Now()
			continue // No further processing needed.
		}

		// Only deal with ICMP Unreachable packets (since that's what iOS and Android seem to be sending whenever the apps are not available).
		if msg.Type != ipv4.ICMPTypeDestinationUnreachable {
			continue
		}
		// Packet parsing.
		mb, err := msg.Body.Marshal(1)
		if err != nil {
			continue
		}
		if len(mb) < 28 {
			continue
		}

		// The unreachable port.
		port := (uint16(mb[26]) << 8) | uint16(mb[27])
		ipAndPort := ip + ":" + strconv.Itoa(int(port))

		netMutex.Lock()
		p, ok := outSockets[ipAndPort]
		if !ok {
			// Can't do anything, the client isn't even technically connected.
			netMutex.Unlock()
			continue
		}
		p.lastUnreachable = time.Now()
		outSockets[ipAndPort] = p
		netMutex.Unlock()
	}
}

func initNetwork() {
	messageQueue = make(chan networkMessage, 1024) // Buffered channel, 1024 messages.
	outSockets = make(map[string]networkConnection)
	pingResponse = make(map[string]time.Time)
	netMutex = &sync.Mutex{}
	refreshConnectedClients()
	go monitorDHCPLeases()
	go messageQueueSender()
	go sleepMonitor()
}
<commit_msg>Type fix.<commit_after>package main

import (
	"golang.org/x/net/icmp"
	"golang.org/x/net/ipv4"
	"io/ioutil"
	"log"
	"math"
	"math/rand"
	"net"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"
)

type networkMessage struct {
	msg       []byte
	msgType   uint8
	queueable bool
	ts        time.Time
}

type networkConnection struct {
	Conn         *net.UDPConn
	Ip           string
	Port         uint32
	Capability   uint8
	messageQueue [][]byte // Device message queue.
	/*
		Sleep mode/throttle variables. "sleep mode" is actually now just a very reduced packet rate, since we don't know positively
		 when a client is ready to accept packets - we just assume so if we don't receive ICMP Unreachable packets in 5 secs.
	*/
	lastUnreachable time.Time // Last time the device sent an ICMP Unreachable packet.
	nextMessageTime time.Time // The next time that the device is "able" to receive a message.
	numOverflows    uint32    // Number of times the queue has overflowed - for calculating the amount to chop off from the queue.
}

var messageQueue chan networkMessage
var outSockets map[string]networkConnection
var dhcpLeases map[string]string
var netMutex *sync.Mutex

var totalNetworkMessagesSent uint32

var pingResponse map[string]time.Time // Last time an IP responded to an "echo" response.

const (
	NETWORK_GDL90_STANDARD = 1
	NETWORK_AHRS_FFSIM     = 2
	NETWORK_AHRS_GDL90     = 4
	dhcp_lease_file        = "/var/lib/dhcp/dhcpd.leases"
)

// Read the "dhcpd.leases" file and parse out IP/hostname.
func getDHCPLeases() (map[string]string, error) {
	dat, err := ioutil.ReadFile(dhcp_lease_file)
	ret := make(map[string]string)
	if err != nil {
		return ret, err
	}
	lines := strings.Split(string(dat), "\n")
	open_block := false
	block_ip := ""
	for _, line := range lines {
		spaced := strings.Split(line, " ")
		if len(spaced) > 2 && spaced[0] == "lease" {
			open_block = true
			block_ip = spaced[1]
		} else if open_block && len(spaced) >= 4 && spaced[2] == "client-hostname" {
			hostname := strings.TrimRight(strings.TrimLeft(strings.Join(spaced[3:], " "), "\""), "\";")
			ret[block_ip] = hostname
			open_block = false
		} else if open_block && strings.HasPrefix(spaced[0], "}") { // No hostname.
			open_block = false
			ret[block_ip] = ""
		}
	}
	return ret, nil
}

func isSleeping(k string) bool {
	ipAndPort := strings.Split(k, ":")
	lastPing, ok := pingResponse[ipAndPort[0]]
	// No ping response. Assume disconnected/sleeping device.
	if !ok || time.Since(lastPing) > (10*time.Second) {
		return true
	}
	if time.Since(outSockets[k].lastUnreachable) < (5 * time.Second) {
		return true
	}
	return false
}

// Throttle mode for testing port open and giving some start-up time to the app.
// Throttling is 0.1% data rate for first 15 seconds.
func isThrottled(k string) bool {
	return (rand.Int()%1000 != 0) && time.Since(outSockets[k].lastUnreachable) < (15*time.Second)
}

func sendToAllConnectedClients(msg networkMessage) {
	netMutex.Lock()
	defer netMutex.Unlock()
	for k, netconn := range outSockets {
		// Check if this port is able to accept the type of message we're sending.
		if (netconn.Capability & msg.msgType) == 0 {
			continue
		}
		// Send non-queueable messages immediately, or discard if the client is in sleep mode.
		if !msg.queueable {
			if !isSleeping(k) {
				netconn.Conn.Write(msg.msg) // Write immediately.
				totalNetworkMessagesSent++
			}
		} else {
			if !isSleeping(k) {
				netconn.numOverflows = 0 // Reset the overflow counter whenever the client is not sleeping so that we're not penalizing future sleepmodes.
			}
			// Queue the message if the message is "queueable".
			if len(netconn.messageQueue) >= maxUserMsgQueueSize { // Too many messages queued? Drop the oldest.
				log.Printf("%s:%d - message queue overflow.\n", netconn.Ip, netconn.Port)
				netconn.numOverflows++
				s := 2 * netconn.numOverflows // Double the amount we chop off on each overflow.
				if int(s) >= len(netconn.messageQueue) {
					netconn.messageQueue = make([][]byte, 0)
				} else {
					netconn.messageQueue = netconn.messageQueue[s:]
				}
			}
			netconn.messageQueue = append(netconn.messageQueue, msg.msg)
			outSockets[k] = netconn
		}
	}
}

// Just returns the number of DHCP leases for now.
func getNetworkStats() uint {
	ret := uint(len(dhcpLeases))
	globalStatus.Connected_Users = ret
	return ret
}

// See who has a DHCP lease and make a UDP connection to each of them.
func refreshConnectedClients() {
	netMutex.Lock()
	defer netMutex.Unlock()
	validConnections := make(map[string]bool)
	t, err := getDHCPLeases()
	if err != nil {
		log.Printf("getDHCPLeases(): %s\n", err.Error())
		return
	}
	dhcpLeases = t
	// Client connected that wasn't before.
	for ip, hostname := range dhcpLeases {
		for _, networkOutput := range globalSettings.NetworkOutputs {
			ipAndPort := ip + ":" + strconv.Itoa(int(networkOutput.Port))
			if _, ok := outSockets[ipAndPort]; !ok {
				log.Printf("client connected: %s:%d (%s).\n", ip, networkOutput.Port, hostname)
				addr, err := net.ResolveUDPAddr("udp", ipAndPort)
				if err != nil {
					log.Printf("ResolveUDPAddr(%s): %s\n", ipAndPort, err.Error())
					continue
				}
				outConn, err := net.DialUDP("udp", nil, addr)
				if err != nil {
					log.Printf("DialUDP(%s): %s\n", ipAndPort, err.Error())
					continue
				}
				newq := make([][]byte, 0)
				outSockets[ipAndPort] = networkConnection{Conn: outConn, Ip: ip, Port: networkOutput.Port, Capability: networkOutput.Capability, messageQueue: newq}
			}
			validConnections[ipAndPort] = true
		}
	}
	// Client that was connected before that isn't.
	for ipAndPort, conn := range outSockets {
		if _, ok := validConnections[ipAndPort]; !ok {
			log.Printf("removed connection %s.\n", ipAndPort)
			conn.Conn.Close()
			delete(outSockets, ipAndPort)
		}
	}
}

func messageQueueSender() {
	secondTimer := time.NewTicker(5 * time.Second)

	queueTimer := time.NewTicker(1 * time.Second)
	//	queueTimer := time.NewTicker(400 * time.Microsecond) // 2500 msg/sec

	var lastQueueTimeChange time.Time // Reevaluate	send frequency every 5 seconds.
	for {
		select {
		case msg := <-messageQueue:
			sendToAllConnectedClients(msg)
		case <-queueTimer.C:
			netMutex.Lock()

			averageSendableQueueSize := float64(0.0)
			for k, netconn := range outSockets {
				if len(netconn.messageQueue) > 0 && !isSleeping(k) && !isThrottled(k) {
					averageSendableQueueSize += float64(len(netconn.messageQueue)) // Add num sendable messages.

					tmpConn := netconn
					tmpConn.Conn.Write(tmpConn.messageQueue[0])
					totalNetworkMessagesSent++
					tmpConn.messageQueue = tmpConn.messageQueue[1:]
					outSockets[k] = tmpConn
				}
			}
			if time.Since(lastQueueTimeChange) >= 5 * time.Second {
				var pd float64
				if averageSendableQueueSize > 0.0 && len(outSockets) > 0 {
					averageSendableQueueSize = averageSendableQueueSize / float64(len(outSockets)) // It's a total, not an average, up until this point.
					pd = math.Max(float64(1.0/2500.0), float64(1.0/(4.0*averageSendableQueueSize))) // Say 250msg is enough to get through the whole queue.
				} else {
					pd = float64(1.0/2500.0)
				}
				queueTimer.Stop()
				queueTimer = time.NewTicker(int(pd * 1000000000.0) * time.Nanosecond)
				lastQueueTimeChange = time.Now()
			}
			averageSendableQueueSize
			netMutex.Unlock()
		case <-secondTimer.C:
			getNetworkStats()
		}
	}
}

func sendMsg(msg []byte, msgType uint8, queueable bool) {
	messageQueue <- networkMessage{msg: msg, msgType: msgType, queueable: queueable, ts: time.Now()}
}

func sendGDL90(msg []byte, queueable bool) {
	sendMsg(msg, NETWORK_GDL90_STANDARD, queueable)
}

func monitorDHCPLeases() {
	//TODO: inotify or dhcp event hook.
	timer := time.NewTicker(30 * time.Second)
	for {
		select {
		case <-timer.C:
			refreshConnectedClients()
		}
	}
}

func icmpEchoSender(c *icmp.PacketConn) {
	timer := time.NewTicker(5 * time.Second)
	for {
		<-timer.C
		// Collect IPs.
		ips := make(map[string]bool)
		for k, _ := range outSockets {
			ipAndPort := strings.Split(k, ":")
			ips[ipAndPort[0]] = true
		}
		// Send to all IPs.
		for ip, _ := range ips {
			wm := icmp.Message{
				Type: ipv4.ICMPTypeEcho, Code: 0,
				Body: &icmp.Echo{
					ID: os.Getpid() & 0xffff, Seq: 1,
					Data: []byte("STRATUX"),
				},
			}
			wb, err := wm.Marshal(nil)
			if err != nil {
				log.Printf("couldn't send ICMP Echo: %s\n", err.Error())
				continue
			}
			if _, err := c.WriteTo(wb, &net.IPAddr{IP: net.ParseIP(ip)}); err != nil {
				log.Printf("couldn't send ICMP Echo: %s\n", err.Error())
				continue
			}
			totalNetworkMessagesSent++
		}
	}
}

// Monitor clients going in/out of sleep mode via ICMP unreachable packets.
func sleepMonitor() {
	c, err := icmp.ListenPacket("ip4:icmp", "0.0.0.0")
	if err != nil {
		log.Printf("error listening for udp - sending data to all ports for all connected clients. err: %s", err)
		return
	}
	go icmpEchoSender(c)
	defer c.Close()
	for {
		buf := make([]byte, 1500)
		n, peer, err := c.ReadFrom(buf)
		if err != nil {
			log.Printf("%s\n", err.Error())
			continue
		}
		msg, err := icmp.ParseMessage(1, buf[:n])
		if err != nil {
			continue
		}

		ip := peer.String()

		// Look for echo replies, mark it as received.
		if msg.Type == ipv4.ICMPTypeEchoReply {
			pingResponse[ip] = time.Now()
			continue // No further processing needed.
		}

		// Only deal with ICMP Unreachable packets (since that's what iOS and Android seem to be sending whenever the apps are not available).
		if msg.Type != ipv4.ICMPTypeDestinationUnreachable {
			continue
		}
		// Packet parsing.
		mb, err := msg.Body.Marshal(1)
		if err != nil {
			continue
		}
		if len(mb) < 28 {
			continue
		}

		// The unreachable port.
		port := (uint16(mb[26]) << 8) | uint16(mb[27])
		ipAndPort := ip + ":" + strconv.Itoa(int(port))

		netMutex.Lock()
		p, ok := outSockets[ipAndPort]
		if !ok {
			// Can't do anything, the client isn't even technically connected.
			netMutex.Unlock()
			continue
		}
		p.lastUnreachable = time.Now()
		outSockets[ipAndPort] = p
		netMutex.Unlock()
	}
}

func initNetwork() {
	messageQueue = make(chan networkMessage, 1024) // Buffered channel, 1024 messages.
	outSockets = make(map[string]networkConnection)
	pingResponse = make(map[string]time.Time)
	netMutex = &sync.Mutex{}
	refreshConnectedClients()
	go monitorDHCPLeases()
	go messageQueueSender()
	go sleepMonitor()
}
<|endoftext|>


<commit_before>// Package hostdb provides a HostDB object that implements the renter.hostDB
// interface. The blockchain is scanned for host announcements and hosts that
// are found get added to the host database. The database continually scans the
// set of hosts it has found and updates who is online.
package hostdb

import (
	"errors"
	"os"
	"sync"

	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

const (
	// scanPoolSize sets the buffer size of the channel that holds hosts which
	// need to be scanned. A thread pool pulls from the scan pool to query
	// hosts that are due for an update.
	scanPoolSize = 1000
)

var (
	errNilCS = errors.New("cannot create hostdb with nil consensus set")
)

// The HostDB is a database of potential hosts. It assigns a weight to each
// host based on their hosting parameters, and then can select hosts at random
// for uploading files.
type HostDB struct {
	// dependencies
	dialer  dialer
	log     logger
	persist persister
	sleeper sleeper

	// The hostTree is the root node of the tree that organizes hosts by
	// weight. The tree is necessary for selecting weighted hosts at
	// random. 'activeHosts' provides a lookup from hostname to the the
	// corresponding node, as the hostTree is unsorted. A host is active if
	// it is currently responding to queries about price and other
	// settings.
	hostTree    *hostNode
	activeHosts map[modules.NetAddress]*hostNode

	// allHosts is a simple list of all known hosts by their network address,
	// including hosts that are currently offline.
	allHosts map[modules.NetAddress]*hostEntry

	// the scanPool is a set of hosts that need to be scanned. There are a
	// handful of goroutines constantly waiting on the channel for hosts to
	// scan.
	scanPool chan *hostEntry

	blockHeight types.BlockHeight
	lastChange  modules.ConsensusChangeID

	mu sync.RWMutex
}

// New returns a new HostDB.
func New(cs consensusSet, persistDir string) (*HostDB, error) {
	// Check for nil inputs.
	if cs == nil {
		return nil, errNilCS
	}

	// Create the persist directory if it does not yet exist.
	err := os.MkdirAll(persistDir, 0700)
	if err != nil {
		return nil, err
	}
	// Create the logger.
	logger, err := newLogger(persistDir)
	if err != nil {
		return nil, err
	}

	// Create HostDB using production dependencies.
	return newHostDB(cs, stdDialer{}, stdSleeper{}, newPersist(persistDir), logger)
}

// newHostDB creates a HostDB using the provided dependencies. It loads the old
// persistence data, spawns the HostDB's scanning threads, and subscribes it to
// the consensusSet.
func newHostDB(cs consensusSet, d dialer, s sleeper, p persister, l logger) (*HostDB, error) {
	// Create the HostDB object.
	hdb := &HostDB{
		dialer:  d,
		sleeper: s,
		persist: p,
		log:     l,

		// TODO: should index by pubkey, not ip
		activeHosts: make(map[modules.NetAddress]*hostNode),
		allHosts:    make(map[modules.NetAddress]*hostEntry),
		scanPool:    make(chan *hostEntry, scanPoolSize),
	}

	// Load the prior persistance structures.
	err := hdb.load()
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}

	// Begin listening to consensus and looking for hosts.
	for i := 0; i < scanningThreads; i++ {
		go hdb.threadedProbeHosts()
	}
	go hdb.threadedScan()

	err = cs.ConsensusSetPersistentSubscribe(hdb, hdb.lastChange)
	if err == modules.ErrInvalidConsensusChangeID {
		hdb.lastChange = modules.ConsensusChangeID{}
		// clear the host sets
		hdb.activeHosts = make(map[modules.NetAddress]*hostNode)
		hdb.allHosts = make(map[modules.NetAddress]*hostEntry)
		// subscribe again using the new ID
		err = cs.ConsensusSetPersistentSubscribe(hdb, hdb.lastChange)
	}
	if err != nil {
		return nil, errors.New("hostdb subscription failed: " + err.Error())
	}

	return hdb, nil
}
<commit_msg>fix race condition in hostdb<commit_after>// Package hostdb provides a HostDB object that implements the renter.hostDB
// interface. The blockchain is scanned for host announcements and hosts that
// are found get added to the host database. The database continually scans the
// set of hosts it has found and updates who is online.
package hostdb

import (
	"errors"
	"os"
	"sync"

	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

const (
	// scanPoolSize sets the buffer size of the channel that holds hosts which
	// need to be scanned. A thread pool pulls from the scan pool to query
	// hosts that are due for an update.
	scanPoolSize = 1000
)

var (
	errNilCS = errors.New("cannot create hostdb with nil consensus set")
)

// The HostDB is a database of potential hosts. It assigns a weight to each
// host based on their hosting parameters, and then can select hosts at random
// for uploading files.
type HostDB struct {
	// dependencies
	dialer  dialer
	log     logger
	persist persister
	sleeper sleeper

	// The hostTree is the root node of the tree that organizes hosts by
	// weight. The tree is necessary for selecting weighted hosts at
	// random. 'activeHosts' provides a lookup from hostname to the the
	// corresponding node, as the hostTree is unsorted. A host is active if
	// it is currently responding to queries about price and other
	// settings.
	hostTree    *hostNode
	activeHosts map[modules.NetAddress]*hostNode

	// allHosts is a simple list of all known hosts by their network address,
	// including hosts that are currently offline.
	allHosts map[modules.NetAddress]*hostEntry

	// the scanPool is a set of hosts that need to be scanned. There are a
	// handful of goroutines constantly waiting on the channel for hosts to
	// scan.
	scanPool chan *hostEntry

	blockHeight types.BlockHeight
	lastChange  modules.ConsensusChangeID

	mu sync.RWMutex
}

// New returns a new HostDB.
func New(cs consensusSet, persistDir string) (*HostDB, error) {
	// Check for nil inputs.
	if cs == nil {
		return nil, errNilCS
	}

	// Create the persist directory if it does not yet exist.
	err := os.MkdirAll(persistDir, 0700)
	if err != nil {
		return nil, err
	}
	// Create the logger.
	logger, err := newLogger(persistDir)
	if err != nil {
		return nil, err
	}

	// Create HostDB using production dependencies.
	return newHostDB(cs, stdDialer{}, stdSleeper{}, newPersist(persistDir), logger)
}

// newHostDB creates a HostDB using the provided dependencies. It loads the old
// persistence data, spawns the HostDB's scanning threads, and subscribes it to
// the consensusSet.
func newHostDB(cs consensusSet, d dialer, s sleeper, p persister, l logger) (*HostDB, error) {
	// Create the HostDB object.
	hdb := &HostDB{
		dialer:  d,
		sleeper: s,
		persist: p,
		log:     l,

		// TODO: should index by pubkey, not ip
		activeHosts: make(map[modules.NetAddress]*hostNode),
		allHosts:    make(map[modules.NetAddress]*hostEntry),
		scanPool:    make(chan *hostEntry, scanPoolSize),
	}

	// Load the prior persistance structures.
	err := hdb.load()
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}

	err = cs.ConsensusSetPersistentSubscribe(hdb, hdb.lastChange)
	if err == modules.ErrInvalidConsensusChangeID {
		hdb.lastChange = modules.ConsensusChangeID{}
		// clear the host sets
		hdb.activeHosts = make(map[modules.NetAddress]*hostNode)
		hdb.allHosts = make(map[modules.NetAddress]*hostEntry)
		// subscribe again using the new ID
		err = cs.ConsensusSetPersistentSubscribe(hdb, hdb.lastChange)
	}
	if err != nil {
		return nil, errors.New("hostdb subscription failed: " + err.Error())
	}

	// Begin listening to consensus and looking for hosts.
	for i := 0; i < scanningThreads; i++ {
		go hdb.threadedProbeHosts()
	}
	go hdb.threadedScan()
	return hdb, nil
}
<|endoftext|>