<commit_before>package main

import (
	"bytes"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"reflect"
	"strings"

	log "github.com/sirupsen/logrus"
	"github.com/gonuts/cbor"
	"github.com/ugorji/go/codec"
	"github.com/nttdots/go-dots/coap"
	"github.com/nttdots/go-dots/dots_common/connection"
	"github.com/nttdots/go-dots/dots_common/messages"
)

type RequestInterface interface {
	LoadJson([]byte) error
	CreateRequest(messageId uint16)
	Send() error
}

/*
 * Dots requests
 */
type Request struct {
	Message     interface{}
	RequestCode messages.Code
	message     coap.Message
	coapType    coap.COAPType
	address     string
	method      string
	requestName string

	connectionFactory connection.ClientConnectionFactory
}

/*
 * Request constructor.
 */
func NewRequest(code messages.Code, coapType coap.COAPType, address, method string, requestName string, factory connection.ClientConnectionFactory) *Request {
	return &Request{
		nil,
		code,
		coap.Message{},
		coapType,
		address,
		method,
		requestName,
		factory,
	}
}

/*
 * Load a Message to this Request
 */
func (r *Request) LoadMessage(message interface{}) {
	r.Message = message
}

/*
 * convert this Request into the Cbor format.
 */
func (r *Request) dumpCbor() []byte {

	cborWriter := bytes.NewBuffer(nil)
	e := cbor.NewEncoder(cborWriter)

	err := e.Encode(r.Message)
	if err != nil {
		log.Errorf("Error decoding %s", err)
	}
	return cborWriter.Bytes()

}

/*
 * convert this Requests into the JSON format.
 */
func (r *Request) dumpJson() []byte {
	payload, _ := json.Marshal(r.Message)
	return payload
}

/*
 * Load Message from JSON data.
 */
func (r *Request) LoadJson(jsonData []byte) error {
	m := reflect.New(r.RequestCode.Type()).Interface()

	err := json.Unmarshal(jsonData, &m)
	if err != nil {
		return fmt.Errorf("Can't Convert Json to Message Object: %v\n", err)

	}
	r.Message = m
	return nil
}

/*
 * return the Request paths.
 */
func (r *Request) pathString() {
	r.RequestCode.PathString()
}

/*
 * Create CoAP requests.
 */
func (r *Request) CreateRequest(messageId uint16) {
	var code coap.COAPCode

	switch strings.ToUpper(r.method) {
	case "GET":
		code = coap.GET
	case "PUT":
		code = coap.PUT
	case "POST":
		code = coap.POST
	case "DELETE":
		code = coap.DELETE
	default:
		log.WithField("method", r.method).Error("invalid request method.")
	}

	r.message = coap.Message{
		Type:      r.coapType,
		Code:      code,
		MessageID: messageId,
	}

	if r.Message != nil {
		r.message.Payload = r.dumpCbor()
		r.message.SetOption(coap.ContentFormat, coap.AppCbor)
		log.Debugf("hex dump cbor request:\n%s", hex.Dump(r.message.Payload))
	}
	r.message.SetPathString(r.RequestCode.PathString())
}

/*
 * Set the certificate common name to the CoAP request messages.
 */
func (r *Request) SetCommonName(commonName string) {
	r.message.SetOption(coap.CommonName, commonName)
}

/*
 * Send the request to the server.
*/
func (r *Request) Send() (err error) {
	conn, err := r.connectionFactory.Connect(r.address)
	if err != nil {
		log.WithError(err).Error("dtls connect error.")
		return err
	}
	defer conn.Close()

	recv, err := coap.Send(conn, r.message)
	if err != nil {
		log.Warnf("Send() => %s", err)
		return
	}

	r.logMessage(recv)

	return
}

func (r *Request) Close() {
	r.connectionFactory.Close()
}

func (r *Request) logMessage(msg coap.Message) {
	log.Infof("Message Code: %s (%d)", msg.Code, msg.Code)

	if msg.Payload == nil {
		return
	}

	log.Infof("        Raw payload: %s", msg.Payload)
	log.Infof("        Raw payload hex: \n%s", hex.Dump(msg.Payload))

	cborDecHandle := new(codec.CborHandle)
	cborDecHandle.SetUseIntElmOfStruct(true)
	dec := codec.NewDecoder(bytes.NewReader(msg.Payload), cborDecHandle)

	var err error
	var logStr string

	switch r.requestName {
	case "mitigation_request":
		var v messages.MitigationRequest
		err = dec.Decode(&v)
		logStr = v.String()
	case "session_configuration":
		var v messages.ConfigurationResponse
		err = dec.Decode(&v)
		logStr = fmt.Sprintf("%+v", v)
	}
	if err != nil {
		log.WithError(err).Warn("CBOR Decode failed.")
		return
	}
	log.Infof("        CBOR decoded: %s", logStr)
}
<commit_msg>re-fix#2 dots client response dump for session config<commit_after>package main

import (
	"bytes"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"reflect"
	"strings"

	log "github.com/sirupsen/logrus"
	"github.com/gonuts/cbor"
	"github.com/ugorji/go/codec"
	"github.com/nttdots/go-dots/coap"
	"github.com/nttdots/go-dots/dots_common/connection"
	"github.com/nttdots/go-dots/dots_common/messages"
)

type RequestInterface interface {
	LoadJson([]byte) error
	CreateRequest(messageId uint16)
	Send() error
}

/*
 * Dots requests
 */
type Request struct {
	Message     interface{}
	RequestCode messages.Code
	message     coap.Message
	coapType    coap.COAPType
	address     string
	method      string
	requestName string

	connectionFactory connection.ClientConnectionFactory
}

/*
 * Request constructor.
 */
func NewRequest(code messages.Code, coapType coap.COAPType, address, method string, requestName string, factory connection.ClientConnectionFactory) *Request {
	return &Request{
		nil,
		code,
		coap.Message{},
		coapType,
		address,
		method,
		requestName,
		factory,
	}
}

/*
 * Load a Message to this Request
 */
func (r *Request) LoadMessage(message interface{}) {
	r.Message = message
}

/*
 * convert this Request into the Cbor format.
 */
func (r *Request) dumpCbor() []byte {

	cborWriter := bytes.NewBuffer(nil)
	e := cbor.NewEncoder(cborWriter)

	err := e.Encode(r.Message)
	if err != nil {
		log.Errorf("Error decoding %s", err)
	}
	return cborWriter.Bytes()

}

/*
 * convert this Requests into the JSON format.
 */
func (r *Request) dumpJson() []byte {
	payload, _ := json.Marshal(r.Message)
	return payload
}

/*
 * Load Message from JSON data.
 */
func (r *Request) LoadJson(jsonData []byte) error {
	m := reflect.New(r.RequestCode.Type()).Interface()

	err := json.Unmarshal(jsonData, &m)
	if err != nil {
		return fmt.Errorf("Can't Convert Json to Message Object: %v\n", err)

	}
	r.Message = m
	return nil
}

/*
 * return the Request paths.
 */
func (r *Request) pathString() {
	r.RequestCode.PathString()
}

/*
 * Create CoAP requests.
 */
func (r *Request) CreateRequest(messageId uint16) {
	var code coap.COAPCode

	switch strings.ToUpper(r.method) {
	case "GET":
		code = coap.GET
	case "PUT":
		code = coap.PUT
	case "POST":
		code = coap.POST
	case "DELETE":
		code = coap.DELETE
	default:
		log.WithField("method", r.method).Error("invalid request method.")
	}

	r.message = coap.Message{
		Type:      r.coapType,
		Code:      code,
		MessageID: messageId,
	}

	if r.Message != nil {
		r.message.Payload = r.dumpCbor()
		r.message.SetOption(coap.ContentFormat, coap.AppCbor)
		log.Debugf("hex dump cbor request:\n%s", hex.Dump(r.message.Payload))
	}
	r.message.SetPathString(r.RequestCode.PathString())
}

/*
 * Set the certificate common name to the CoAP request messages.
 */
func (r *Request) SetCommonName(commonName string) {
	r.message.SetOption(coap.CommonName, commonName)
}

/*
 * Send the request to the server.
*/
func (r *Request) Send() (err error) {
	conn, err := r.connectionFactory.Connect(r.address)
	if err != nil {
		log.WithError(err).Error("dtls connect error.")
		return err
	}
	defer conn.Close()

	recv, err := coap.Send(conn, r.message)
	if err != nil {
		log.Warnf("Send() => %s", err)
		return
	}

	r.logMessage(recv)

	return
}

func (r *Request) Close() {
	r.connectionFactory.Close()
}

func (r *Request) logMessage(msg coap.Message) {
	log.Infof("Message Code: %s (%d)", msg.Code, msg.Code)

	if msg.Payload == nil {
		return
	}

	log.Infof("        Raw payload: %s", msg.Payload)
	log.Infof("        Raw payload hex: \n%s", hex.Dump(msg.Payload))

	cborDecHandle := new(codec.CborHandle)
	cborDecHandle.SetUseIntElmOfStruct(true)
	dec := codec.NewDecoder(bytes.NewReader(msg.Payload), cborDecHandle)

	var err error
	var logStr string

	switch r.requestName {
	case "mitigation_request":
		var v messages.MitigationRequest
		err = dec.Decode(&v)
		logStr = v.String()
	case "session_configuration":
		if r.method == "Get" {
			var v messages.ConfigurationResponse
			err = dec.Decode(&v)
			logStr = fmt.Sprintf("%+v", v)
		} else {
			var v messages.SignalConfigRequest
			err = dec.Decode(&v)
			logStr = fmt.Sprintf("%+v", v)
		}
	}
	if err != nil {
		log.WithError(err).Warn("CBOR Decode failed.")
		return
	}
	log.Infof("        CBOR decoded: %s", logStr)
}
<|endoftext|>


<commit_before>package hostdb

// scan.go contians the functions which periodically scan the list of all hosts
// to see which hosts are online or offline, and to get any updates to the
// settings of the hosts.

import (
	"crypto/rand"
	"math/big"
	"net"
	"time"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

const (
	DefaultScanSleep = 2*time.Hour + 18*time.Minute
	MaxScanSleep     = 6 * time.Hour
	MinScanSleep     = 1 * time.Hour

	MaxActiveHosts              = 500
	InactiveHostCheckupQuantity = 250

	maxSettingsLen = 1024

	hostRequestTimeout = 5 * time.Second

	// scanningThreads is the number of threads that will be probing hosts for
	// their settings and checking for reliability.
	scanningThreads = 25
)

var (
	MaxReliability     = types.NewCurrency64(50) // Given the scanning defaults, about 1 week of survival.
	DefaultReliability = types.NewCurrency64(20) // Given the scanning defaults, about 3 days of survival.
	UnreachablePenalty = types.NewCurrency64(1)
)

// addHostToScanPool creates a gofunc that adds a host to the scan pool. If the
// scan pool is currently full, the blocking gofunc will not cause a deadlock.
// The gofunc is created inside of this function to eliminate the burden of
// needing to remember to call 'go addHostToScanPool'.
func (hdb *HostDB) scanHostEntry(entry *hostEntry) {
	go func() {
		hdb.scanPool <- entry
	}()
}

// decrementReliability reduces the reliability of a node, moving it out of the
// set of active hosts or deleting it entirely if necessary.
func (hdb *HostDB) decrementReliability(addr modules.NetAddress, penalty types.Currency) {
	// Look up the entry and decrement the reliability.
	entry, exists := hdb.allHosts[addr]
	if !exists {
		return
	}
	entry.reliability = entry.reliability.Sub(penalty)

	// If the entry is in the active database, remove it from the active
	// database.
	node, exists := hdb.activeHosts[addr]
	if exists {
		delete(hdb.activeHosts, entry.IPAddress)
		node.removeNode()
		hdb.notifySubscribers()
	}

	// If the reliability has fallen to 0, remove the host from the
	// database entirely.
	if entry.reliability.IsZero() {
		delete(hdb.allHosts, addr)
	}
}

// threadedProbeHost tries to fetch the settings of a host. If successful, the
// host is put in the set of active hosts. If unsuccessful, the host id deleted
// from the set of active hosts.
func (hdb *HostDB) threadedProbeHosts() {
	for hostEntry := range hdb.scanPool {
		// Request settings from the queued host entry.
		var settings modules.HostSettings
		err := func() error {
			conn, err := net.DialTimeout("tcp", string(hostEntry.IPAddress), hostRequestTimeout)
			if err != nil {
				return err
			}
			defer conn.Close()
			err = encoding.WriteObject(conn, [8]byte{'S', 'e', 't', 't', 'i', 'n', 'g', 's'})
			if err != nil {
				return err
			}
			return encoding.ReadObject(conn, &settings, maxSettingsLen)
		}()

		// Now that network communication is done, lock the hostdb to modify the
		// host entry.
		id := hdb.mu.Lock()
		{
			if err != nil {
				hdb.decrementReliability(hostEntry.IPAddress, UnreachablePenalty)
				hdb.mu.Unlock(id)
				continue
			}

			// Update the host settings, reliability, and weight. The old IPAddress
			// must be preserved.
			settings.IPAddress = hostEntry.HostSettings.IPAddress
			hostEntry.HostSettings = settings
			hostEntry.reliability = MaxReliability
			hostEntry.weight = hdb.hostWeight(*hostEntry)

			// If the host is not already in the database and 'MaxActiveHosts' has not
			// been reached, add the host to the database.
			_, exists1 := hdb.activeHosts[hostEntry.IPAddress]
			_, exists2 := hdb.allHosts[hostEntry.IPAddress]
			if !exists1 && exists2 && len(hdb.activeHosts) < MaxActiveHosts {
				hdb.insertNode(hostEntry)
				hdb.notifySubscribers()
			}
		}
		hdb.mu.Unlock(id)
	}
}

// threadedScan is an ongoing function which will query the full set of hosts
// every few hours to see who is online and available for uploading.
func (hdb *HostDB) threadedScan() {
	for {
		// Determine who to scan. At most 'MaxActiveHosts' will be scanned,
		// starting with the active hosts followed by a random selection of the
		// inactive hosts.
		id := hdb.mu.Lock()
		{
			// Scan all active hosts.
			for _, host := range hdb.activeHosts {
				hdb.scanHostEntry(host.hostEntry)
			}

			// Assemble all of the inactive hosts into a single array.
			var random []*hostEntry
			for _, entry := range hdb.allHosts {
				entry2, exists := hdb.activeHosts[entry.IPAddress]
				if !exists {
					random = append(random, entry)
				} else {
					if build.DEBUG {
						if entry2.hostEntry != entry {
							panic("allHosts + activeHosts mismatch!")
						}
					}
				}
			}

			// Randomize the slice by swapping each element with an element
			// that hasn't been visited yet.
			for i := 0; i < len(random); i++ {
				N, err := rand.Int(rand.Reader, big.NewInt(int64(len(random)-i)))
				if err != nil {
					if build.DEBUG {
						panic(err)
					}
				} else {
					break
				}

				n := int(N.Int64()) + i
				tmp := random[i]
				random[i] = random[n]
				random[n] = tmp
			}

			// Select the first InactiveHostCheckupQuantity hosts from the
			// shuffled list and scan them.
			n := InactiveHostCheckupQuantity
			if len(random) < InactiveHostCheckupQuantity {
				n = len(random)
			}
			for i := 0; i < n; i++ {
				hdb.scanHostEntry(random[i])
			}
		}
		hdb.mu.Unlock(id)

		// Sleep for a random amount of time before doing another round of
		// scanning. The minimums and maximums keep the scan time reasonable,
		// while the randomness prevents the scanning from always happening at
		// the same time of day or week.
		maxBig := big.NewInt(int64(MaxScanSleep))
		minBig := big.NewInt(int64(MinScanSleep))
		randSleep, err := rand.Int(rand.Reader, maxBig.Sub(maxBig, minBig))
		if err != nil {
			if build.DEBUG {
				panic(err)
			} else {
				// If there's an error, sleep for the default amount of time.
				defaultBig := big.NewInt(int64(DefaultScanSleep))
				randSleep = defaultBig.Sub(defaultBig, minBig)
			}
		}
		time.Sleep(time.Duration(randSleep.Int64()) + MinScanSleep) // this means the MaxScanSleep is actual Max+Min.
	}
}
<commit_msg>off-by-one error in scan.go<commit_after>package hostdb

// scan.go contians the functions which periodically scan the list of all hosts
// to see which hosts are online or offline, and to get any updates to the
// settings of the hosts.

import (
	"crypto/rand"
	"math/big"
	"net"
	"time"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

const (
	DefaultScanSleep = 2*time.Hour + 18*time.Minute
	MaxScanSleep     = 6 * time.Hour
	MinScanSleep     = 1 * time.Hour

	MaxActiveHosts              = 500
	InactiveHostCheckupQuantity = 250

	maxSettingsLen = 1024

	hostRequestTimeout = 5 * time.Second

	// scanningThreads is the number of threads that will be probing hosts for
	// their settings and checking for reliability.
	scanningThreads = 25
)

var (
	MaxReliability     = types.NewCurrency64(50) // Given the scanning defaults, about 1 week of survival.
	DefaultReliability = types.NewCurrency64(20) // Given the scanning defaults, about 3 days of survival.
	UnreachablePenalty = types.NewCurrency64(1)
)

// addHostToScanPool creates a gofunc that adds a host to the scan pool. If the
// scan pool is currently full, the blocking gofunc will not cause a deadlock.
// The gofunc is created inside of this function to eliminate the burden of
// needing to remember to call 'go addHostToScanPool'.
func (hdb *HostDB) scanHostEntry(entry *hostEntry) {
	go func() {
		hdb.scanPool <- entry
	}()
}

// decrementReliability reduces the reliability of a node, moving it out of the
// set of active hosts or deleting it entirely if necessary.
func (hdb *HostDB) decrementReliability(addr modules.NetAddress, penalty types.Currency) {
	// Look up the entry and decrement the reliability.
	entry, exists := hdb.allHosts[addr]
	if !exists {
		return
	}
	entry.reliability = entry.reliability.Sub(penalty)

	// If the entry is in the active database, remove it from the active
	// database.
	node, exists := hdb.activeHosts[addr]
	if exists {
		delete(hdb.activeHosts, entry.IPAddress)
		node.removeNode()
		hdb.notifySubscribers()
	}

	// If the reliability has fallen to 0, remove the host from the
	// database entirely.
	if entry.reliability.IsZero() {
		delete(hdb.allHosts, addr)
	}
}

// threadedProbeHost tries to fetch the settings of a host. If successful, the
// host is put in the set of active hosts. If unsuccessful, the host id deleted
// from the set of active hosts.
func (hdb *HostDB) threadedProbeHosts() {
	for hostEntry := range hdb.scanPool {
		// Request settings from the queued host entry.
		var settings modules.HostSettings
		err := func() error {
			conn, err := net.DialTimeout("tcp", string(hostEntry.IPAddress), hostRequestTimeout)
			if err != nil {
				return err
			}
			defer conn.Close()
			err = encoding.WriteObject(conn, [8]byte{'S', 'e', 't', 't', 'i', 'n', 'g', 's'})
			if err != nil {
				return err
			}
			return encoding.ReadObject(conn, &settings, maxSettingsLen)
		}()

		// Now that network communication is done, lock the hostdb to modify the
		// host entry.
		id := hdb.mu.Lock()
		{
			if err != nil {
				hdb.decrementReliability(hostEntry.IPAddress, UnreachablePenalty)
				hdb.mu.Unlock(id)
				continue
			}

			// Update the host settings, reliability, and weight. The old IPAddress
			// must be preserved.
			settings.IPAddress = hostEntry.HostSettings.IPAddress
			hostEntry.HostSettings = settings
			hostEntry.reliability = MaxReliability
			hostEntry.weight = hdb.hostWeight(*hostEntry)

			// If the host is not already in the database and 'MaxActiveHosts' has not
			// been reached, add the host to the database.
			_, exists1 := hdb.activeHosts[hostEntry.IPAddress]
			_, exists2 := hdb.allHosts[hostEntry.IPAddress]
			if !exists1 && exists2 && len(hdb.activeHosts) < MaxActiveHosts {
				hdb.insertNode(hostEntry)
				hdb.notifySubscribers()
			}
		}
		hdb.mu.Unlock(id)
	}
}

// threadedScan is an ongoing function which will query the full set of hosts
// every few hours to see who is online and available for uploading.
func (hdb *HostDB) threadedScan() {
	for {
		// Determine who to scan. At most 'MaxActiveHosts' will be scanned,
		// starting with the active hosts followed by a random selection of the
		// inactive hosts.
		id := hdb.mu.Lock()
		{
			// Scan all active hosts.
			for _, host := range hdb.activeHosts {
				hdb.scanHostEntry(host.hostEntry)
			}

			// Assemble all of the inactive hosts into a single array.
			var random []*hostEntry
			for _, entry := range hdb.allHosts {
				entry2, exists := hdb.activeHosts[entry.IPAddress]
				if !exists {
					random = append(random, entry)
				} else {
					if build.DEBUG {
						if entry2.hostEntry != entry {
							panic("allHosts + activeHosts mismatch!")
						}
					}
				}
			}

			// Randomize the slice by swapping each element with an element
			// that hasn't been visited yet.
			for i := 0; i < len(random); i++ {
				N, err := rand.Int(rand.Reader, big.NewInt(int64(len(random)-i-1)))
				if err != nil {
					if build.DEBUG {
						panic(err)
					}
				} else {
					break
				}

				n := int(N.Int64()) + i
				tmp := random[i]
				random[i] = random[n]
				random[n] = tmp
			}

			// Select the first InactiveHostCheckupQuantity hosts from the
			// shuffled list and scan them.
			n := InactiveHostCheckupQuantity
			if len(random) < InactiveHostCheckupQuantity {
				n = len(random)
			}
			for i := 0; i < n; i++ {
				hdb.scanHostEntry(random[i])
			}
		}
		hdb.mu.Unlock(id)

		// Sleep for a random amount of time before doing another round of
		// scanning. The minimums and maximums keep the scan time reasonable,
		// while the randomness prevents the scanning from always happening at
		// the same time of day or week.
		maxBig := big.NewInt(int64(MaxScanSleep))
		minBig := big.NewInt(int64(MinScanSleep))
		randSleep, err := rand.Int(rand.Reader, maxBig.Sub(maxBig, minBig))
		if err != nil {
			if build.DEBUG {
				panic(err)
			} else {
				// If there's an error, sleep for the default amount of time.
				defaultBig := big.NewInt(int64(DefaultScanSleep))
				randSleep = defaultBig.Sub(defaultBig, minBig)
			}
		}
		time.Sleep(time.Duration(randSleep.Int64()) + MinScanSleep) // this means the MaxScanSleep is actual Max+Min.
	}
}
<|endoftext|>


<commit_before>package hostdb

// scan.go contians the functions which periodically scan the list of all hosts
// to see which hosts are online or offline, and to get any updates to the
// settings of the hosts.

import (
	"crypto/rand"
	"math/big"
	"net"
	"time"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

const (
	DefaultScanSleep = 2*time.Hour + 18*time.Minute
	MaxScanSleep     = 6 * time.Hour
	MinScanSleep     = 1 * time.Hour

	MaxActiveHosts              = 500
	InactiveHostCheckupQuantity = 250

	maxSettingsLen = 1024

	hostRequestTimeout = 15 * time.Second

	// scanningThreads is the number of threads that will be probing hosts for
	// their settings and checking for reliability.
	scanningThreads = 4
)

var (
	MaxReliability     = types.NewCurrency64(50) // Given the scanning defaults, about 1 week of survival.
	DefaultReliability = types.NewCurrency64(20) // Given the scanning defaults, about 3 days of survival.
	UnreachablePenalty = types.NewCurrency64(1)
)

// decrementReliability reduces the reliability of a node, moving it out of the
// set of active hosts or deleting it entirely if necessary.
func (hdb *HostDB) decrementReliability(addr modules.NetAddress, penalty types.Currency) {
	// Look up the entry and decrement the reliability.
	entry, exists := hdb.allHosts[addr]
	if !exists {
		return
	}
	entry.reliability = entry.reliability.Sub(penalty)

	// If the entry is in the active database, remove it from the active
	// database.
	node, exists := hdb.activeHosts[addr]
	if exists {
		delete(hdb.activeHosts, entry.IPAddress)
		node.removeNode()
		hdb.notifySubscribers()
	}

	// If the reliability has fallen to 0, remove the host from the
	// database entirely.
	if entry.reliability.IsZero() {
		delete(hdb.allHosts, addr)
	}
}

// threadedProbeHost tries to fetch the settings of a host. If successful, the
// host is put in the set of active hosts. If unsuccessful, the host id deleted
// from the set of active hosts.
func (hdb *HostDB) threadedProbeHosts() {
	for hostEntry := range hdb.scanPool {
		// Request settings from the queue'd host entry.
		var settings modules.HostSettings
		err := func() error {
			conn, err := net.DialTimeout("tcp", string(hostEntry.IPAddress), hostRequestTimeout)
			if err != nil {
				return err
			}
			defer conn.Close()
			err = encoding.WriteObject(conn, [8]byte{'S', 'e', 't', 't', 'i', 'n', 'g', 's'})
			if err != nil {
				return err
			}
			return encoding.ReadObject(conn, &settings, maxSettingsLen)
		}()

		// Now that network communication is done, lock the hostdb to modify the
		// host entry.
		id := hdb.mu.Lock()
		{
			if err != nil {
				hdb.decrementReliability(hostEntry.IPAddress, UnreachablePenalty)
				hdb.mu.Unlock(id)
				continue
			}

			// Update the host settings, reliability, and weight. The old IPAddress
			// must be preserved.
			settings.IPAddress = hostEntry.HostSettings.IPAddress
			hostEntry.HostSettings = settings
			hostEntry.reliability = MaxReliability
			hostEntry.weight = hdb.hostWeight(*hostEntry)

			// If the host is not already in the database and 'MaxActiveHosts' has not
			// been reached, add the host to the database.
			_, exists1 := hdb.activeHosts[hostEntry.IPAddress]
			_, exists2 := hdb.allHosts[hostEntry.IPAddress]
			if !exists1 && exists2 && len(hdb.activeHosts) < MaxActiveHosts {
				hdb.insertNode(hostEntry)
				hdb.notifySubscribers()
			}
		}
		hdb.mu.Unlock(id)
	}
}

// threadedScan is an ongoing function which will query the full set of hosts
// every few hours to see who is online and available for uploading.
func (hdb *HostDB) threadedScan() {
	for {
		// Determine who to scan. At most 'MaxActiveHosts' will be scanned,
		// starting with the active hosts followed by a random selection of the
		// inactive hosts.
		id := hdb.mu.Lock()
		{
			// Scan all active hosts.
			for _, host := range hdb.activeHosts {
				go func() {
					hdb.scanPool <- host.hostEntry
				}()
			}

			// Assemble all of the inactive hosts into a single array.
			var random []*hostEntry
			for _, entry := range hdb.allHosts {
				entry2, exists := hdb.activeHosts[entry.IPAddress]
				if !exists {
					random = append(random, entry)
				} else {
					if build.DEBUG {
						if entry2.hostEntry != entry {
							panic("allHosts + activeHosts mismatch!")
						}
					}
				}
			}

			// Randomize the slice by swapping each element with an element
			// that hasn't been visited yet.
			for i := 0; i < len(random); i++ {
				N, err := rand.Int(rand.Reader, big.NewInt(int64(len(random)-i)))
				if err != nil {
					if build.DEBUG {
						panic(err)
					}
				} else {
					break
				}

				n := int(N.Int64()) + i
				tmp := random[i]
				random[i] = random[n]
				random[n] = tmp
			}

			// Select the first InactiveHostCheckupQuantity hosts from the
			// shuffled list and scan them.
			n := InactiveHostCheckupQuantity
			if len(random) < InactiveHostCheckupQuantity {
				n = len(random)
			}
			for i := 0; i < n; i++ {
				go func() {
					hdb.scanPool <- random[i]
				}()
			}
		}
		hdb.mu.Unlock(id)

		// Sleep for a random amount of time before doing another round of
		// scanning. The minimums and maximums keep the scan time reasonable,
		// while the randomness prevents the scanning from always happening at
		// the same time of day or week.
		maxBig := big.NewInt(int64(MaxScanSleep))
		minBig := big.NewInt(int64(MinScanSleep))
		randSleep, err := rand.Int(rand.Reader, maxBig.Sub(maxBig, minBig))
		if err != nil {
			if build.DEBUG {
				panic(err)
			} else {
				// If there's an error, sleep for the default amount of time.
				defaultBig := big.NewInt(int64(DefaultScanSleep))
				randSleep = defaultBig.Sub(defaultBig, minBig)
			}
		}
		time.Sleep(time.Duration(randSleep.Int64()) + MinScanSleep) // this means the MaxScanSleep is actual Max+Min.
	}
}
<commit_msg>faster scanning constants<commit_after>package hostdb

// scan.go contians the functions which periodically scan the list of all hosts
// to see which hosts are online or offline, and to get any updates to the
// settings of the hosts.

import (
	"crypto/rand"
	"math/big"
	"net"
	"time"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

const (
	DefaultScanSleep = 2*time.Hour + 18*time.Minute
	MaxScanSleep     = 6 * time.Hour
	MinScanSleep     = 1 * time.Hour

	MaxActiveHosts              = 500
	InactiveHostCheckupQuantity = 250

	maxSettingsLen = 1024

	hostRequestTimeout = 5 * time.Second

	// scanningThreads is the number of threads that will be probing hosts for
	// their settings and checking for reliability.
	scanningThreads = 25
)

var (
	MaxReliability     = types.NewCurrency64(50) // Given the scanning defaults, about 1 week of survival.
	DefaultReliability = types.NewCurrency64(20) // Given the scanning defaults, about 3 days of survival.
	UnreachablePenalty = types.NewCurrency64(1)
)

// decrementReliability reduces the reliability of a node, moving it out of the
// set of active hosts or deleting it entirely if necessary.
func (hdb *HostDB) decrementReliability(addr modules.NetAddress, penalty types.Currency) {
	// Look up the entry and decrement the reliability.
	entry, exists := hdb.allHosts[addr]
	if !exists {
		return
	}
	entry.reliability = entry.reliability.Sub(penalty)

	// If the entry is in the active database, remove it from the active
	// database.
	node, exists := hdb.activeHosts[addr]
	if exists {
		delete(hdb.activeHosts, entry.IPAddress)
		node.removeNode()
		hdb.notifySubscribers()
	}

	// If the reliability has fallen to 0, remove the host from the
	// database entirely.
	if entry.reliability.IsZero() {
		delete(hdb.allHosts, addr)
	}
}

// threadedProbeHost tries to fetch the settings of a host. If successful, the
// host is put in the set of active hosts. If unsuccessful, the host id deleted
// from the set of active hosts.
func (hdb *HostDB) threadedProbeHosts() {
	for hostEntry := range hdb.scanPool {
		// Request settings from the queue'd host entry.
		var settings modules.HostSettings
		err := func() error {
			conn, err := net.DialTimeout("tcp", string(hostEntry.IPAddress), hostRequestTimeout)
			if err != nil {
				return err
			}
			defer conn.Close()
			err = encoding.WriteObject(conn, [8]byte{'S', 'e', 't', 't', 'i', 'n', 'g', 's'})
			if err != nil {
				return err
			}
			return encoding.ReadObject(conn, &settings, maxSettingsLen)
		}()

		// Now that network communication is done, lock the hostdb to modify the
		// host entry.
		id := hdb.mu.Lock()
		{
			if err != nil {
				hdb.decrementReliability(hostEntry.IPAddress, UnreachablePenalty)
				hdb.mu.Unlock(id)
				continue
			}

			// Update the host settings, reliability, and weight. The old IPAddress
			// must be preserved.
			settings.IPAddress = hostEntry.HostSettings.IPAddress
			hostEntry.HostSettings = settings
			hostEntry.reliability = MaxReliability
			hostEntry.weight = hdb.hostWeight(*hostEntry)

			// If the host is not already in the database and 'MaxActiveHosts' has not
			// been reached, add the host to the database.
			_, exists1 := hdb.activeHosts[hostEntry.IPAddress]
			_, exists2 := hdb.allHosts[hostEntry.IPAddress]
			if !exists1 && exists2 && len(hdb.activeHosts) < MaxActiveHosts {
				hdb.insertNode(hostEntry)
				hdb.notifySubscribers()
			}
		}
		hdb.mu.Unlock(id)
	}
}

// threadedScan is an ongoing function which will query the full set of hosts
// every few hours to see who is online and available for uploading.
func (hdb *HostDB) threadedScan() {
	for {
		// Determine who to scan. At most 'MaxActiveHosts' will be scanned,
		// starting with the active hosts followed by a random selection of the
		// inactive hosts.
		id := hdb.mu.Lock()
		{
			// Scan all active hosts.
			for _, host := range hdb.activeHosts {
				go func() {
					hdb.scanPool <- host.hostEntry
				}()
			}

			// Assemble all of the inactive hosts into a single array.
			var random []*hostEntry
			for _, entry := range hdb.allHosts {
				entry2, exists := hdb.activeHosts[entry.IPAddress]
				if !exists {
					random = append(random, entry)
				} else {
					if build.DEBUG {
						if entry2.hostEntry != entry {
							panic("allHosts + activeHosts mismatch!")
						}
					}
				}
			}

			// Randomize the slice by swapping each element with an element
			// that hasn't been visited yet.
			for i := 0; i < len(random); i++ {
				N, err := rand.Int(rand.Reader, big.NewInt(int64(len(random)-i)))
				if err != nil {
					if build.DEBUG {
						panic(err)
					}
				} else {
					break
				}

				n := int(N.Int64()) + i
				tmp := random[i]
				random[i] = random[n]
				random[n] = tmp
			}

			// Select the first InactiveHostCheckupQuantity hosts from the
			// shuffled list and scan them.
			n := InactiveHostCheckupQuantity
			if len(random) < InactiveHostCheckupQuantity {
				n = len(random)
			}
			for i := 0; i < n; i++ {
				go func() {
					hdb.scanPool <- random[i]
				}()
			}
		}
		hdb.mu.Unlock(id)

		// Sleep for a random amount of time before doing another round of
		// scanning. The minimums and maximums keep the scan time reasonable,
		// while the randomness prevents the scanning from always happening at
		// the same time of day or week.
		maxBig := big.NewInt(int64(MaxScanSleep))
		minBig := big.NewInt(int64(MinScanSleep))
		randSleep, err := rand.Int(rand.Reader, maxBig.Sub(maxBig, minBig))
		if err != nil {
			if build.DEBUG {
				panic(err)
			} else {
				// If there's an error, sleep for the default amount of time.
				defaultBig := big.NewInt(int64(DefaultScanSleep))
				randSleep = defaultBig.Sub(defaultBig, minBig)
			}
		}
		time.Sleep(time.Duration(randSleep.Int64()) + MinScanSleep) // this means the MaxScanSleep is actual Max+Min.
	}
}
<|endoftext|>


<commit_before>package gorums

import "fmt"

type ConfigOption interface{}

type NodeListOption interface {
	ConfigOption
	newConfig(*Manager) (Configuration, error)
}

type nodeIDMap struct {
	idMap map[string]uint32
}

func (o nodeIDMap) newConfig(mgr *Manager) (nodes Configuration, err error) {
	if len(o.idMap) == 0 {
		return nil, ConfigCreationError(fmt.Errorf("node-to-ID map required: WithNodeMap"))
	}
	nodes = make(Configuration, 0)
	for naddr, id := range o.idMap {
		node, found := mgr.Node(id)
		if !found {
			node, err = NewNodeWithID(naddr, id)
			if err != nil {
				return nil, ConfigCreationError(err)
			}
			err = mgr.AddNode(node)
			if err != nil {
				return nil, ConfigCreationError(err)
			}
		}
		nodes = append(nodes, node)
	}
	// Sort nodes to ensure deterministic iteration.
	OrderedBy(ID).Sort(mgr.nodes)
	OrderedBy(ID).Sort(nodes)
	return nodes, nil
}

// WithNodeMap returns a ConfigOption containing the provided
// mapping from node addresses to application-specific IDs.
func WithNodeMap(idMap map[string]uint32) NodeListOption {
	return &nodeIDMap{idMap: idMap}
}

type nodeList struct {
	addrsList []string
}

func (o nodeList) newConfig(mgr *Manager) (nodes Configuration, err error) {
	if len(o.addrsList) == 0 {
		return nil, ConfigCreationError(fmt.Errorf("node addresses required: WithNodeList"))
	}
	nodes = make(Configuration, 0)
	for _, naddr := range o.addrsList {
		node, err := NewNode(naddr)
		if err != nil {
			return nil, ConfigCreationError(err)
		}
		if n, found := mgr.Node(node.ID()); !found {
			err = mgr.AddNode(node)
			if err != nil {
				return nil, ConfigCreationError(err)
			}
		} else {
			node = n
		}
		nodes = append(nodes, node)
	}
	// Sort nodes to ensure deterministic iteration.
	OrderedBy(ID).Sort(mgr.nodes)
	OrderedBy(ID).Sort(nodes)
	return nodes, nil
}

// WithNodeList returns a ConfigOption containing the provided list of node addresses.
// With this option, node IDs are generated by the Manager.
func WithNodeList(addrsList []string) NodeListOption {
	return &nodeList{addrsList: addrsList}
}

type nodeIDs struct {
	nodeIDs []uint32
}

func (o nodeIDs) newConfig(mgr *Manager) (nodes Configuration, err error) {
	if len(o.nodeIDs) == 0 {
		return nil, ConfigCreationError(fmt.Errorf("node IDs required: WithNodeIDs"))
	}
	nodes = make(Configuration, 0)
	for _, id := range o.nodeIDs {
		node, found := mgr.Node(id)
		if !found {
			// Node IDs must have been registered previously
			return nil, ConfigCreationError(fmt.Errorf("node ID %d not found", id))
		}
		nodes = append(nodes, node)
	}
	// Sort nodes to ensure deterministic iteration.
	OrderedBy(ID).Sort(mgr.nodes)
	OrderedBy(ID).Sort(nodes)
	return nodes, nil
}

// WithNodeIDs returns a ConfigOption containing a list of node IDs.
// This assumes that the provided node IDs have already been registered with the manager.
func WithNodeIDs(ids []uint32) NodeListOption {
	return &nodeIDs{nodeIDs: ids}
}

type addNodes struct {
	old Configuration
	new NodeListOption
}

func (o addNodes) newConfig(mgr *Manager) (nodes Configuration, err error) {
	newNodes, err := o.new.newConfig(mgr)
	if err != nil {
		return nil, err
	}
	ac := &addConfig{old: o.old, add: newNodes}
	return ac.newConfig(mgr)
}

// Add returns a NodeListOption that can be used to create a new configuration combining c
// and the nodes provided via the NodeListOption new.
func (c Configuration) AddNodes(new NodeListOption) NodeListOption {
	return &addNodes{old: c, new: new}
}

type addConfig struct {
	old Configuration
	add Configuration
}

func (o addConfig) newConfig(mgr *Manager) (nodes Configuration, err error) {
	nodes = make(Configuration, 0)
	m := make(map[uint32]bool)
	for _, n := range append(o.old, o.add...) {
		if !m[n.id] {
			m[n.id] = true
			nodes = append(nodes, n)
		}
	}
	// Sort nodes to ensure deterministic iteration.
	OrderedBy(ID).Sort(mgr.nodes)
	OrderedBy(ID).Sort(nodes)
	return nodes, err
}

// Add returns a NodeListOption that can be used to create a new configuration combining c and add.
func (c Configuration) Add(add Configuration) NodeListOption {
	return &addConfig{old: c, add: add}
}
<commit_msg>Documentation improvements<commit_after>package gorums

import "fmt"

// ConfigOption is a marker interface for options to NewConfiguration.
type ConfigOption interface{}

// NodeListOption must be implemented by node providers.
type NodeListOption interface {
	ConfigOption
	newConfig(*Manager) (Configuration, error)
}

type nodeIDMap struct {
	idMap map[string]uint32
}

func (o nodeIDMap) newConfig(mgr *Manager) (nodes Configuration, err error) {
	if len(o.idMap) == 0 {
		return nil, ConfigCreationError(fmt.Errorf("node-to-ID map required: WithNodeMap"))
	}
	nodes = make(Configuration, 0)
	for naddr, id := range o.idMap {
		node, found := mgr.Node(id)
		if !found {
			node, err = NewNodeWithID(naddr, id)
			if err != nil {
				return nil, ConfigCreationError(err)
			}
			err = mgr.AddNode(node)
			if err != nil {
				return nil, ConfigCreationError(err)
			}
		}
		nodes = append(nodes, node)
	}
	// Sort nodes to ensure deterministic iteration.
	OrderedBy(ID).Sort(mgr.nodes)
	OrderedBy(ID).Sort(nodes)
	return nodes, nil
}

// WithNodeMap returns a NodeListOption containing the provided
// mapping from node addresses to application-specific IDs.
func WithNodeMap(idMap map[string]uint32) NodeListOption {
	return &nodeIDMap{idMap: idMap}
}

type nodeList struct {
	addrsList []string
}

func (o nodeList) newConfig(mgr *Manager) (nodes Configuration, err error) {
	if len(o.addrsList) == 0 {
		return nil, ConfigCreationError(fmt.Errorf("node addresses required: WithNodeList"))
	}
	nodes = make(Configuration, 0)
	for _, naddr := range o.addrsList {
		node, err := NewNode(naddr)
		if err != nil {
			return nil, ConfigCreationError(err)
		}
		if n, found := mgr.Node(node.ID()); !found {
			err = mgr.AddNode(node)
			if err != nil {
				return nil, ConfigCreationError(err)
			}
		} else {
			node = n
		}
		nodes = append(nodes, node)
	}
	// Sort nodes to ensure deterministic iteration.
	OrderedBy(ID).Sort(mgr.nodes)
	OrderedBy(ID).Sort(nodes)
	return nodes, nil
}

// WithNodeList returns a NodeListOption containing the provided list of node addresses.
// With this option, node IDs are generated by the Manager.
func WithNodeList(addrsList []string) NodeListOption {
	return &nodeList{addrsList: addrsList}
}

type nodeIDs struct {
	nodeIDs []uint32
}

func (o nodeIDs) newConfig(mgr *Manager) (nodes Configuration, err error) {
	if len(o.nodeIDs) == 0 {
		return nil, ConfigCreationError(fmt.Errorf("node IDs required: WithNodeIDs"))
	}
	nodes = make(Configuration, 0)
	for _, id := range o.nodeIDs {
		node, found := mgr.Node(id)
		if !found {
			// Node IDs must have been registered previously
			return nil, ConfigCreationError(fmt.Errorf("node ID %d not found", id))
		}
		nodes = append(nodes, node)
	}
	// Sort nodes to ensure deterministic iteration.
	OrderedBy(ID).Sort(mgr.nodes)
	OrderedBy(ID).Sort(nodes)
	return nodes, nil
}

// WithNodeIDs returns a NodeListOption containing a list of node IDs.
// This assumes that the provided node IDs have already been registered with the manager.
func WithNodeIDs(ids []uint32) NodeListOption {
	return &nodeIDs{nodeIDs: ids}
}

type addNodes struct {
	old Configuration
	new NodeListOption
}

func (o addNodes) newConfig(mgr *Manager) (nodes Configuration, err error) {
	newNodes, err := o.new.newConfig(mgr)
	if err != nil {
		return nil, err
	}
	ac := &addConfig{old: o.old, add: newNodes}
	return ac.newConfig(mgr)
}

// Add returns a NodeListOption that can be used to create a new configuration combining c
// and the nodes provided via the NodeListOption new.
func (c Configuration) AddNodes(new NodeListOption) NodeListOption {
	return &addNodes{old: c, new: new}
}

type addConfig struct {
	old Configuration
	add Configuration
}

func (o addConfig) newConfig(mgr *Manager) (nodes Configuration, err error) {
	nodes = make(Configuration, 0)
	m := make(map[uint32]bool)
	for _, n := range append(o.old, o.add...) {
		if !m[n.id] {
			m[n.id] = true
			nodes = append(nodes, n)
		}
	}
	// Sort nodes to ensure deterministic iteration.
	OrderedBy(ID).Sort(mgr.nodes)
	OrderedBy(ID).Sort(nodes)
	return nodes, err
}

// Add returns a NodeListOption that can be used to create a new configuration combining c and add.
func (c Configuration) Add(add Configuration) NodeListOption {
	return &addConfig{old: c, add: add}
}
<|endoftext|>


<commit_before>package application

import (
	"log"
	"net/http"
	"os"
	"time"

	"github.com/newrelic/go-agent"
	"github.com/pressly/chi"
	"github.com/pressly/chi/middleware"
)

var newrelicApp newrelic.Application

func init() {
	newrelicKey := os.Getenv("NEW_RELIC_LICENSE_KEY")
	if newrelicKey == "" {
		newrelicKey = "1234567890123456789012345678901234567890" // length 40
	}
	config := newrelic.NewConfig("bubble-go", newrelicKey)
	app, err := newrelic.NewApplication(config)
	if err != nil {
		log.Fatal(err)
	}
	newrelicApp = app
}

// GetRouter return http.Handler for tests and core
func GetRouter(test bool) http.Handler {
	router := chi.NewRouter()

	if !test {
		router.Use(middleware.Logger)
		router.Use(middleware.Recoverer)
	}

	// Set a timeout value on the request context (ctx), that will signal
	// through ctx.Done() that the request has timed out and further
	// processing should be stopped.
	router.Use(middleware.Timeout(60 * time.Second))

	router.Mount("/debug", middleware.Profiler())

	router.Get(wrapHandlerFunc("/", func(w http.ResponseWriter, r *http.Request) {
		JSON(w, h{
			"foo": "bar",
		})
	}))

	router.Mount("/", func() http.Handler {
		r := chi.NewRouter()
		r.Use(AuthorizationMiddleware)
		r.Post(wrapHandlerFunc("/ReqEnter", ReqEnter))
		r.Post(wrapHandlerFunc("/ReqBuyProduct", ReqBuyProduct))
		r.Post(wrapHandlerFunc("/ReqReduceTries", ReqReduceTries))
		r.Post(wrapHandlerFunc("/ReqReduceCredits", ReqReduceCredits))
		r.Post(wrapHandlerFunc("/ReqSavePlayerProgress", ReqSavePlayerProgress))
		r.Post(wrapHandlerFunc("/ReqUsersProgress", ReqUsersProgress))
		return r
	}())
	router.Post(wrapHandlerFunc("/VkPay", VkPay))

	router.Get(wrapHandlerFunc("/crossdomain.xml", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`<?xml version="1.0"?><cross-domain-policy><allow-access-from domain="*" /></cross-domain-policy>`))
	}))
	// http://119226.selcdn.ru/bubble/ShootTheBubbleDevVK.html
	// http://bubble-srv-dev.herokuapp.com/bubble/ShootTheBubbleDevVK.html
	router.Get(wrapHandlerFunc("/bubble/*filePath", ServeStatick))
	router.Get(wrapHandlerFunc("/cache-clear", ClearStatickCache))

	router.Get(wrapHandlerFunc("/exception", func(w http.ResponseWriter, r *http.Request) {
		panic("test log.Fatal")
	}))

	router.Get(wrapHandlerFunc("/debug-vk", func(w http.ResponseWriter, r *http.Request) {
		JSON(w, h{
			"levels": VkWorker.LenLevels(),
			"events": VkWorker.LenEvents(),
		})
	}))

	loaderio := os.Getenv("LOADERIO")
	router.Get(wrapHandlerFunc("/loaderio-"+loaderio, func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("loaderio-" + loaderio))
	}))

	return router
}

func wrapHandlerFunc(route string, handler http.HandlerFunc) (string, http.HandlerFunc) {
	return newrelic.WrapHandleFunc(newrelicApp, route, handler)
}
<commit_msg>wrong url<commit_after>package application

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"time"

	"github.com/newrelic/go-agent"
	"github.com/pressly/chi"
	"github.com/pressly/chi/middleware"
)

var newrelicApp newrelic.Application

func init() {
	newrelicKey := os.Getenv("NEW_RELIC_LICENSE_KEY")
	if newrelicKey == "" {
		newrelicKey = "1234567890123456789012345678901234567890" // length 40
	}
	config := newrelic.NewConfig("bubble-go", newrelicKey)
	app, err := newrelic.NewApplication(config)
	if err != nil {
		log.Fatal(err)
	}
	newrelicApp = app
}

// GetRouter return http.Handler for tests and core
func GetRouter(test bool) http.Handler {
	router := chi.NewRouter()

	if !test {
		router.Use(middleware.Logger)
		router.Use(middleware.Recoverer)
	}

	// Set a timeout value on the request context (ctx), that will signal
	// through ctx.Done() that the request has timed out and further
	// processing should be stopped.
	router.Use(middleware.Timeout(60 * time.Second))

	router.Mount("/debug", middleware.Profiler())

	router.Get(wrapHandlerFunc("/", func(w http.ResponseWriter, r *http.Request) {
		JSON(w, h{
			"foo": "bar",
		})
	}))

	router.Mount("/", func() http.Handler {
		r := chi.NewRouter()
		r.Use(AuthorizationMiddleware)
		r.Post(wrapHandlerFunc("/ReqEnter", ReqEnter))
		r.Post(wrapHandlerFunc("/ReqBuyProduct", ReqBuyProduct))
		r.Post(wrapHandlerFunc("/ReqReduceTries", ReqReduceTries))
		r.Post(wrapHandlerFunc("/ReqReduceCredits", ReqReduceCredits))
		r.Post(wrapHandlerFunc("/ReqSavePlayerProgress", ReqSavePlayerProgress))
		r.Post(wrapHandlerFunc("/ReqUsersProgress", ReqUsersProgress))
		return r
	}())
	router.Post(wrapHandlerFunc("/VkPay", VkPay))

	router.Get(wrapHandlerFunc("/crossdomain.xml", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`<?xml version="1.0"?><cross-domain-policy><allow-access-from domain="*" /></cross-domain-policy>`))
	}))
	// http://119226.selcdn.ru/bubble/ShootTheBubbleDevVK.html
	// http://bubble-srv-dev.herokuapp.com/bubble/ShootTheBubbleDevVK.html
	router.Get(wrapHandlerFunc("/bubble/*filePath", ServeStatick))
	router.Get(wrapHandlerFunc("/cache-clear", ClearStatickCache))

	router.Get(wrapHandlerFunc("/exception", func(w http.ResponseWriter, r *http.Request) {
		panic("test log.Fatal")
	}))

	router.Get(wrapHandlerFunc("/debug-vk", func(w http.ResponseWriter, r *http.Request) {
		JSON(w, h{
			"levels": VkWorker.LenLevels(),
			"events": VkWorker.LenEvents(),
		})
	}))

	loaderio := os.Getenv("LOADERIO")
	loaderioRoute := fmt.Sprintf("/loaderio-%s/", loaderio)
	router.Get(wrapHandlerFunc(loaderioRoute, func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("loaderio-" + loaderio))
	}))

	return router
}

func wrapHandlerFunc(route string, handler http.HandlerFunc) (string, http.HandlerFunc) {
	return newrelic.WrapHandleFunc(newrelicApp, route, handler)
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"fmt"
	"github.com/gin-gonic/gin"
	"net/http"
	"path"
	"sync"

	"github.com/eirka/eirka-libs/config"
	"github.com/eirka/eirka-libs/db"
	e "github.com/eirka/eirka-libs/errors"
)

var (
	sitemap = make(map[string]*SiteData)
	mu      = new(sync.RWMutex)
)

// SiteData holds imageboard settings
type SiteData struct {
	Ib          uint
	Api         string
	Img         string
	Title       string
	Desc        string
	Nsfw        bool
	Style       string
	Logo        string
	Base        string
	Imageboards []Imageboard
}

type Imageboard struct {
	Title   string
	Address string
}

// Details gets the imageboard settings from the request for the page handler variables
func Details() gin.HandlerFunc {
	return func(c *gin.Context) {

		var host, base string

		host = c.Request.Host

		// figure out our path and host
		if path.Dir(host) == "." {
			// we're on the base in this case
			host = path.Base(host)
		} else {
			host = path.Dir(host)
		}

		mu.RLock()
		// check the sitemap to see if its cached
		site := sitemap[host]
		mu.RUnlock()

		// if not query the database
		if site == nil {

			sitedata := &SiteData{}

			// set the base for angularjs
			sitedata.Base = fmt.Sprintf("%s/", path.Base(host))

			// Get Database handle
			dbase, err := db.GetDb()
			if err != nil {
				c.JSON(e.ErrorMessage(e.ErrInternalError))
				c.Error(err).SetMeta("Details.GetDb")
				c.Abort()
				return
			}

			// get the info about the imageboard
			err = dbase.QueryRow(`SELECT ib_id,ib_title,ib_description,ib_nsfw,ib_api,ib_img,ib_style,ib_logo FROM imageboards WHERE ib_domain = ?`, host).Scan(&sitedata.Ib, &sitedata.Title, &sitedata.Desc, &sitedata.Nsfw, &sitedata.Api, &sitedata.Img, &sitedata.Style, &sitedata.Logo)
			if err == sql.ErrNoRows {
				c.JSON(e.ErrorMessage(e.ErrNotFound))
				c.Error(err).SetMeta("Details.QueryRow")
				return
			} else if err != nil {
				c.JSON(e.ErrorMessage(e.ErrInternalError))
				c.Error(err).SetMeta("Details.QueryRow")
				return
			}

			// collect the links to the other imageboards for nav menu
			rows, err := dbase.Query(`SELECT ib_title,ib_domain FROM imageboards WHERE ib_id != ?`, sitedata.Ib)
			if err != nil {
				c.JSON(e.ErrorMessage(e.ErrInternalError))
				c.Error(err).SetMeta("Details.Query")
				c.Abort()
				return
			}
			defer rows.Close()

			for rows.Next() {

				ib := Imageboard{}

				err := rows.Scan(&ib.Title, &ib.Address)
				if err != nil {
					return
				}

				sitedata.Imageboards = append(sitedata.Imageboards, ib)
			}
			if rows.Err() != nil {
				c.JSON(e.ErrorMessage(e.ErrInternalError))
				c.Error(err).SetMeta("Details.Query")
				c.Abort()
				return
			}

			mu.Lock()
			sitemap[host] = sitedata
			mu.Unlock()

		}

		c.Next()

	}

}

// IndexController generates pages for angularjs frontend
func IndexController(c *gin.Context) {

	// Get parameters from csrf middleware
	csrf_token := c.MustGet("csrf_token").(string)

	host := c.Request.Host

	mu.RLock()
	site := sitemap[host]
	mu.RUnlock()

	c.HTML(http.StatusOK, "index", gin.H{
		"primjs":      config.Settings.Prim.Js,
		"primcss":     config.Settings.Prim.Css,
		"ib":          site.Ib,
		"apisrv":      site.Api,
		"imgsrv":      site.Img,
		"title":       site.Title,
		"desc":        site.Desc,
		"nsfw":        site.Nsfw,
		"style":       site.Style,
		"logo":        site.Logo,
		"imageboards": site.Imageboards,
		"csrf":        csrf_token,
	})

	return

}

// ErrorController generates pages and a 404 response
func ErrorController(c *gin.Context) {

	// Get parameters from csrf middleware
	csrf_token := c.MustGet("csrf_token").(string)

	host := c.Request.Host

	mu.RLock()
	site := sitemap[host]
	mu.RUnlock()

	c.HTML(http.StatusNotFound, "index", gin.H{
		"primjs":      config.Settings.Prim.Js,
		"primcss":     config.Settings.Prim.Css,
		"ib":          site.Ib,
		"base":        site.Base,
		"apisrv":      site.Api,
		"imgsrv":      site.Img,
		"title":       site.Title,
		"desc":        site.Desc,
		"nsfw":        site.Nsfw,
		"style":       site.Style,
		"logo":        site.Logo,
		"imageboards": site.Imageboards,
		"csrf":        csrf_token,
	})

	return

}
<commit_msg>add subdirectory support<commit_after>package main

import (
	"database/sql"
	"fmt"
	"github.com/gin-gonic/gin"
	"net/http"
	"path"
	"sync"

	"github.com/eirka/eirka-libs/config"
	"github.com/eirka/eirka-libs/db"
	e "github.com/eirka/eirka-libs/errors"
)

var (
	sitemap = make(map[string]*SiteData)
	mu      = new(sync.RWMutex)
)

// SiteData holds imageboard settings
type SiteData struct {
	Ib          uint
	Api         string
	Img         string
	Title       string
	Desc        string
	Nsfw        bool
	Style       string
	Logo        string
	Base        string
	Imageboards []Imageboard
}

type Imageboard struct {
	Title   string
	Address string
}

// Details gets the imageboard settings from the request for the page handler variables
func Details() gin.HandlerFunc {
	return func(c *gin.Context) {

		host := c.Request.Host

		// figure out our path and host
		if path.Dir(host) == "." {
			// we're on the base in this case
			host = path.Base(host)
		} else {
			host = path.Dir(host)
		}

		mu.RLock()
		// check the sitemap to see if its cached
		site := sitemap[host]
		mu.RUnlock()

		// if not query the database
		if site == nil {

			sitedata := &SiteData{}

			// set the base for angularjs
			sitedata.Base = fmt.Sprintf("%s/", path.Base(host))

			// Get Database handle
			dbase, err := db.GetDb()
			if err != nil {
				c.JSON(e.ErrorMessage(e.ErrInternalError))
				c.Error(err).SetMeta("Details.GetDb")
				c.Abort()
				return
			}

			// get the info about the imageboard
			err = dbase.QueryRow(`SELECT ib_id,ib_title,ib_description,ib_nsfw,ib_api,ib_img,ib_style,ib_logo FROM imageboards WHERE ib_domain = ?`, host).Scan(&sitedata.Ib, &sitedata.Title, &sitedata.Desc, &sitedata.Nsfw, &sitedata.Api, &sitedata.Img, &sitedata.Style, &sitedata.Logo)
			if err == sql.ErrNoRows {
				c.JSON(e.ErrorMessage(e.ErrNotFound))
				c.Error(err).SetMeta("Details.QueryRow")
				return
			} else if err != nil {
				c.JSON(e.ErrorMessage(e.ErrInternalError))
				c.Error(err).SetMeta("Details.QueryRow")
				return
			}

			// collect the links to the other imageboards for nav menu
			rows, err := dbase.Query(`SELECT ib_title,ib_domain FROM imageboards WHERE ib_id != ?`, sitedata.Ib)
			if err != nil {
				c.JSON(e.ErrorMessage(e.ErrInternalError))
				c.Error(err).SetMeta("Details.Query")
				c.Abort()
				return
			}
			defer rows.Close()

			for rows.Next() {

				ib := Imageboard{}

				err := rows.Scan(&ib.Title, &ib.Address)
				if err != nil {
					return
				}

				sitedata.Imageboards = append(sitedata.Imageboards, ib)
			}
			if rows.Err() != nil {
				c.JSON(e.ErrorMessage(e.ErrInternalError))
				c.Error(err).SetMeta("Details.Query")
				c.Abort()
				return
			}

			mu.Lock()
			sitemap[host] = sitedata
			mu.Unlock()

		}

		c.Next()

	}

}

// IndexController generates pages for angularjs frontend
func IndexController(c *gin.Context) {

	// Get parameters from csrf middleware
	csrf_token := c.MustGet("csrf_token").(string)

	host := c.Request.Host

	mu.RLock()
	site := sitemap[host]
	mu.RUnlock()

	c.HTML(http.StatusOK, "index", gin.H{
		"primjs":      config.Settings.Prim.Js,
		"primcss":     config.Settings.Prim.Css,
		"ib":          site.Ib,
		"apisrv":      site.Api,
		"imgsrv":      site.Img,
		"title":       site.Title,
		"desc":        site.Desc,
		"nsfw":        site.Nsfw,
		"style":       site.Style,
		"logo":        site.Logo,
		"imageboards": site.Imageboards,
		"csrf":        csrf_token,
	})

	return

}

// ErrorController generates pages and a 404 response
func ErrorController(c *gin.Context) {

	// Get parameters from csrf middleware
	csrf_token := c.MustGet("csrf_token").(string)

	host := c.Request.Host

	mu.RLock()
	site := sitemap[host]
	mu.RUnlock()

	c.HTML(http.StatusNotFound, "index", gin.H{
		"primjs":      config.Settings.Prim.Js,
		"primcss":     config.Settings.Prim.Css,
		"ib":          site.Ib,
		"base":        site.Base,
		"apisrv":      site.Api,
		"imgsrv":      site.Img,
		"title":       site.Title,
		"desc":        site.Desc,
		"nsfw":        site.Nsfw,
		"style":       site.Style,
		"logo":        site.Logo,
		"imageboards": site.Imageboards,
		"csrf":        csrf_token,
	})

	return

}
<|endoftext|>


<commit_before>package bild

import (
	"image"
	"image/color"
)

// ConvolutionMatrix interface for use as an image Kernel
type ConvolutionMatrix interface {
	At(x, y int) float64
	Sum() float64
	Normalized() ConvolutionMatrix
	Size() int
}

// NewKernel returns a kernel of the provided size
func NewKernel(size int) *Kernel {
	matrix := make([][]float64, size)
	for i := 0; i < size; i++ {
		matrix[i] = make([]float64, size)
	}
	return &Kernel{matrix}
}

// Kernel is used as a convolution matrix
type Kernel struct {
	Matrix [][]float64
}

// Sum returns the cumulative value of the matrix
func (k *Kernel) Sum() float64 {
	var sum float64
	size := k.Size()
	for x := 0; x < size; x++ {
		for y := 0; y < size; y++ {
			sum += k.Matrix[x][y]
		}
	}
	return sum
}

// Normalized returns a new Kernel with normalized values
func (k *Kernel) Normalized() ConvolutionMatrix {
	sum := k.Sum()
	size := k.Size()
	nk := NewKernel(size)

	for x := 0; x < size; x++ {
		for y := 0; y < size; y++ {
			nk.Matrix[x][y] = k.Matrix[x][y] / sum
		}
	}

	return nk
}

// Size returns the row/column length for the kernel
func (k *Kernel) Size() int {
	return len(k.Matrix)
}

// At returns the matrix value at position x, y
func (k *Kernel) At(x, y int) float64 {
	return k.Matrix[x][y]
}

// convolute applies a convolution matrix (kernel) to an image
func convolute(src *image.NRGBA, k ConvolutionMatrix) *image.NRGBA {
	bounds := src.Bounds()
	w, h := bounds.Max.X, bounds.Max.Y
	dst := image.NewNRGBA(bounds)

	nk := k.Normalized()
	nksize := k.Size()

	parallelize(h, func(start, end int) {
		for x := 0; x < w; x++ {
			for y := start; y < end; y++ {

				var r, g, b, a float64
				for kx := 0; kx < nksize; kx++ {
					for ky := 0; ky < nksize; ky++ {
						ix := x + kx - (nksize / 2)
						iy := y + ky - (nksize / 2)

						// Quality threshold
						if nk.At(kx, ky) < 0.00001 {
							continue
						}

						if ix < 0 || kx >= w || iy < 0 || ky >= h {
							continue
						}

						c := src.NRGBAAt(ix, iy)
						r += float64(c.R) * nk.At(kx, ky)
						g += float64(c.G) * nk.At(kx, ky)
						b += float64(c.B) * nk.At(kx, ky)
						a += float64(c.A) * nk.At(kx, ky)
					}
				}

				dst.Set(x, y, color.NRGBA{uint8(r), uint8(g), uint8(b), uint8(a)})
			}
		}
	})

	return dst
}
<commit_msg>refactoring convolution function<commit_after>package bild

import (
	"image"
	"image/color"
	"math"
)

// ConvolutionMatrix interface for use as an image Kernel
type ConvolutionMatrix interface {
	At(x, y int) float64
	Sum() float64
	Normalized() ConvolutionMatrix
	Size() int
}

// NewKernel returns a kernel of the provided size
func NewKernel(size int) *Kernel {
	matrix := make([][]float64, size)
	for i := 0; i < size; i++ {
		matrix[i] = make([]float64, size)
	}
	return &Kernel{matrix}
}

// Kernel is used as a convolution matrix
type Kernel struct {
	Matrix [][]float64
}

// Sum returns the cumulative value of the matrix
func (k *Kernel) Sum() float64 {
	var sum float64
	size := k.Size()
	for x := 0; x < size; x++ {
		for y := 0; y < size; y++ {
			sum += k.Matrix[x][y]
		}
	}
	return sum
}

// Normalized returns a new Kernel with normalized values
func (k *Kernel) Normalized() ConvolutionMatrix {
	sum := k.Sum()
	size := k.Size()
	nk := NewKernel(size)

	for x := 0; x < size; x++ {
		for y := 0; y < size; y++ {
			nk.Matrix[x][y] = k.Matrix[x][y] / sum
		}
	}

	return nk
}

// Size returns the row/column length for the kernel
func (k *Kernel) Size() int {
	return len(k.Matrix)
}

// At returns the matrix value at position x, y
func (k *Kernel) At(x, y int) float64 {
	return k.Matrix[x][y]
}

// convolute applies a convolution matrix (kernel) to an image
func convolute(src *image.NRGBA, k ConvolutionMatrix) *image.NRGBA {
	bounds := src.Bounds()
	w, h := bounds.Max.X, bounds.Max.Y
	dst := image.NewNRGBA(bounds)

	nk := k.Normalized()
	nksize := nk.Size()

	parallelize(h, func(start, end int) {
		for x := 0; x < w; x++ {
			for y := start; y < end; y++ {

				var r, g, b, a float64
				for kx := 0; kx < nksize; kx++ {
					for ky := 0; ky < nksize; ky++ {
						ix := x + kx - (nksize / 2)
						iy := y + ky - (nksize / 2)

						// Quality threshold
						if nk.At(kx, ky) < 0.00001 {
							continue
						}
						if ix < 0 || kx >= w || iy < 0 || ky >= h {
							continue
						}

						c := src.NRGBAAt(ix, iy)
						m := nk.At(kx, ky)
						r += float64(c.R) * m
						g += float64(c.G) * m
						b += float64(c.B) * m
						a += float64(c.A) * m
					}
				}

				c := color.NRGBA{
					uint8(math.Max(0, math.Min(r, 255))),
					uint8(math.Max(0, math.Min(g, 255))),
					uint8(math.Max(0, math.Min(b, 255))),
					uint8(math.Max(0, math.Min(a, 255))),
				}

				dst.Set(x, y, c)
			}
		}
	})

	return dst
}
<|endoftext|>


<commit_before>// Copyright 2015-2016 trivago GmbH
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"github.com/sirupsen/logrus"
	"github.com/trivago/gollum/core"
	"github.com/trivago/gollum/logger"
	"github.com/trivago/tgo"
	"os"
	"os/signal"
	"reflect"
	"sync"
	"time"
)

const (
	coordinatorStateConfigure      = coordinatorState(iota)
	coordinatorStateStartProducers = coordinatorState(iota)
	coordinatorStateStartConsumers = coordinatorState(iota)
	coordinatorStateRunning        = coordinatorState(iota)
	coordinatorStateShutdown       = coordinatorState(iota)
	coordinatorStateStopConsumers  = coordinatorState(iota)
	coordinatorStateStopProducers  = coordinatorState(iota)
	coordinatorStateStopped        = coordinatorState(iota)
)

const (
	signalNone = signalType(iota)
	signalExit = signalType(iota)
	signalRoll = signalType(iota)
)

type coordinatorState byte
type signalType byte

// Coordinator is the main gollum instance taking care of starting and stopping
// plugins.
type Coordinator struct {
	consumers      []core.Consumer
	producers      []core.Producer
	routers        []core.Router
	consumerWorker *sync.WaitGroup
	producerWorker *sync.WaitGroup
	logConsumer    *core.LogConsumer
	state          coordinatorState
	signal         chan os.Signal
}

// NewCoordinator creates a new multplexer
func NewCoordinator() Coordinator {
	return Coordinator{
		consumerWorker: new(sync.WaitGroup),
		producerWorker: new(sync.WaitGroup),
		state:          coordinatorStateConfigure,
	}
}

// Configure processes the config and instantiates all valid plugins
func (co *Coordinator) Configure(conf *core.Config) {
	// Make sure the log is printed to stderr if we are stuck here
	logFallback := time.AfterFunc(time.Duration(3)*time.Second, func() {
		//logrus.SetOutput(os.Stderr)
		logrusHookBuffer.SetTargetWriter(logger.FallbackLogDevice)
		logrusHookBuffer.Purge()
	})
	defer logFallback.Stop()

	// Initialize the plugins in the order of routers > producers > consumers
	// to match the order of reference between the different types.

	co.configureRouters(conf)
	co.configureProducers(conf)
	co.configureConsumers(conf)

	// As consumers might create new fallback router this is the first position
	// where we can add the wildcard producers to all streams. No new routers
	// created beyond this point must use StreamRegistry.AddWildcardProducersToRouter.

	core.StreamRegistry.AddAllWildcardProducersToAllRouters()
}

// StartPlugins starts all plugins in the correct order.
func (co *Coordinator) StartPlugins() {

	if len(co.consumers) == 0 {
		logrus.Error("No consumers configured.")
		logrusHookBuffer.SetTargetWriter(logger.FallbackLogDevice)
		logrusHookBuffer.Purge()
		return // ### return, nothing to do ###
	}

	if len(co.producers) == 0 {
		logrus.Error("No producers configured.")
		logrusHookBuffer.SetTargetWriter(logger.FallbackLogDevice)
		logrusHookBuffer.Purge()
		return // ### return, nothing to do ###
	}

	// Launch routers
	for _, router := range co.routers {
		if err := router.Start(); err != nil {
			logrus.Error("Router was not able to start from type ", reflect.TypeOf(router), ": ", err)
		} else {
			logrus.Debug("Starting ", reflect.TypeOf(router))
		}
	}

	// Launch producers
	co.state = coordinatorStateStartProducers
	for _, producer := range co.producers {
		producer := producer
		go tgo.WithRecoverShutdown(func() {
			logrus.Debug("Starting ", reflect.TypeOf(producer))
			producer.Produce(co.producerWorker)
		})
	}

	// Set final log target and purge the intermediate buffer
	if core.StreamRegistry.IsStreamRegistered(core.LogInternalStreamID) {
		// The _GOLLUM_ stream has listeners, so use LogConsumer to write to it
		if *flagLogColors == "always" {
			logrus.SetFormatter(logger.NewConsoleFormatter())
		}
		logrusHookBuffer.SetTargetHook(co.logConsumer)
		logrusHookBuffer.Purge()

	} else {
		// _GOLLUM_ not used, so write to the fallback device
		if *flagLogColors == "always" ||
			(*flagLogColors == "auto" && logrus.IsTerminal(logger.FallbackLogDevice)) {
			// Logrus doesn't know the final log device, so we hint the color option here
			logrus.SetFormatter(logger.NewConsoleFormatter())
		}
		logrusHookBuffer.SetTargetWriter(logger.FallbackLogDevice)
		logrusHookBuffer.Purge()
	}

	// Launch consumers
	co.state = coordinatorStateStartConsumers
	for _, consumer := range co.consumers {
		consumer := consumer
		go tgo.WithRecoverShutdown(func() {
			logrus.Debug("Starting ", reflect.TypeOf(consumer))
			consumer.Consume(co.consumerWorker)
		})
	}
}

// Run is essentially the Coordinator main loop.
// It listens for shutdown signals and updates global metrics
func (co *Coordinator) Run() {
	co.signal = newSignalHandler()
	defer signal.Stop(co.signal)

	logrus.Info("We be nice to them, if they be nice to us. (startup)")

	for {
		sig := <-co.signal
		switch translateSignal(sig) {
		case signalExit:
			logrus.Info("Master betrayed us. Wicked. Tricksy, False. (signal)")
			return // ### return, exit requested ###

		case signalRoll:
			for _, consumer := range co.consumers {
				consumer.Control() <- core.PluginControlRoll
			}
			for _, producer := range co.producers {
				producer.Control() <- core.PluginControlRoll
			}

		default:
		}
	}
}

// Shutdown all consumers and producers in a clean way.
// The internal log is flushed after the consumers have been shut down so that
// consumer related messages are still in the tlog.
// Producers are flushed after flushing the log, so producer related shutdown
// messages will be posted to stdout
func (co *Coordinator) Shutdown() {
	logrus.Info("Filthy little hobbites. They stole it from us. (shutdown)")

	stateAtShutdown := co.state
	co.state = coordinatorStateShutdown

	co.shutdownConsumers(stateAtShutdown)

	// Make sure remaining warning / errors are written to stderr
	logrus.Info("I'm not listening... I'm not listening... (flushing)")
	//logrus.SetOutput(fallbackLogDevice)
	logrusHookBuffer.SetTargetWriter(logger.FallbackLogDevice)
	logrusHookBuffer.SetTargetHook(nil)

	// Shutdown producers
	co.shutdownProducers(stateAtShutdown)

	co.state = coordinatorStateStopped
}

func (co *Coordinator) configureRouters(conf *core.Config) {
	routerConfigs := conf.GetRouters()
	for _, config := range routerConfigs {
		logrus.Debugf("Instantiating router '%s'", config.ID)

		plugin, err := core.NewPluginWithConfig(config)
		if err != nil {
			logrus.Errorf("Failed to instantiate router %s: %s", config.ID, err)
			continue // ### continue ###
		}

		routerPlugin := plugin.(core.Router)
		co.routers = append(co.routers, routerPlugin)

		logrus.Debugf("Instantiated '%s' (%s) as '%s'", config.ID, core.StreamRegistry.GetStreamName(routerPlugin.GetStreamID()), config.Typename)
		core.StreamRegistry.Register(routerPlugin, routerPlugin.GetStreamID())
	}
}

func (co *Coordinator) configureProducers(conf *core.Config) {
	co.state = coordinatorStateStartProducers

	// All producers are added to the wildcard stream so that consumers can send
	// to all producers if required. The wildcard producer list is required
	// to add producers listening to all routers to all streams that are used.
	wildcardStream := core.StreamRegistry.GetRouterOrFallback(core.WildcardStreamID)
	producerConfigs := conf.GetProducers()

	for _, config := range producerConfigs {
		logrus.Debug("Instantiating ", config.ID)

		plugin, err := core.NewPluginWithConfig(config)
		if err != nil {
			logrus.Errorf("Failed to instantiate producer %s: %s", config.ID, err)
			continue // ### continue ###
		}

		producer, _ := plugin.(core.Producer)
		streams := producer.Streams()

		if len(streams) == 0 {
			logrus.Error("Producer ", config.ID, " has no streams set")
			continue // ### continue ###
		}

		co.producers = append(co.producers, producer)
		core.CountProducers()

		// Attach producer to streams

		for _, streamID := range streams {
			if streamID == core.WildcardStreamID {
				core.StreamRegistry.RegisterWildcardProducer(producer)
			} else {
				router := core.StreamRegistry.GetRouterOrFallback(streamID)
				router.AddProducer(producer)
			}
		}

		// Add producer to wildcard stream unless it only listens to internal streams
	searchinternal:
		for _, streamID := range streams {
			switch streamID {
			case core.LogInternalStreamID:
			default:
				wildcardStream.AddProducer(producer)
				break searchinternal
			}
		}
	}
}

func (co *Coordinator) configureConsumers(conf *core.Config) {
	co.state = coordinatorStateStartConsumers
	co.configureLogConsumer()

	consumerConfigs := conf.GetConsumers()
	for _, config := range consumerConfigs {
		logrus.Debug("Instantiating ", config.ID)

		plugin, err := core.NewPluginWithConfig(config)
		if err != nil {
			logrus.Errorf("Failed to instantiate producer %s: %s", config.ID, err)
			continue // ### continue ###
		}

		consumer, _ := plugin.(core.Consumer)
		co.consumers = append(co.consumers, consumer)
		core.CountConsumers()
	}
}

func (co *Coordinator) configureLogConsumer() {
	config := core.NewPluginConfig("", "core.LogConsumer")
	configReader := core.NewPluginConfigReader(&config)

	co.logConsumer = new(core.LogConsumer)
	co.logConsumer.Configure(configReader)
	co.consumers = append(co.consumers, co.logConsumer)
}

func (co *Coordinator) shutdownConsumers(stateAtShutdown coordinatorState) {
	if stateAtShutdown >= coordinatorStateStartConsumers {
		co.state = coordinatorStateStopConsumers
		waitTimeout := time.Duration(0)

		logrus.Debug("Telling consumers to stop")
		for _, cons := range co.consumers {
			timeout := cons.GetShutdownTimeout()
			if timeout > waitTimeout {
				waitTimeout = timeout
			}
			cons.Control() <- core.PluginControlStopConsumer
		}

		waitTimeout *= 10
		logrus.Debugf("Waiting for consumers to stop. Forced shutdown after %.2f seconds.", waitTimeout.Seconds())
		if !tgo.ReturnAfter(waitTimeout, co.consumerWorker.Wait) {
			logrus.Error("At least one consumer found to be blocking.")
		}
	}
}

func (co *Coordinator) shutdownProducers(stateAtShutdown coordinatorState) {
	if stateAtShutdown >= coordinatorStateStartProducers {
		co.state = coordinatorStateStopProducers
		waitTimeout := time.Duration(0)

		logrus.Debug("Telling producers to stop")
		for _, prod := range co.producers {
			timeout := prod.GetShutdownTimeout()
			if timeout > waitTimeout {
				waitTimeout = timeout
			}
			prod.Control() <- core.PluginControlStopProducer
		}

		waitTimeout *= 10
		logrus.Debugf("Waiting for producers to stop. Forced shutdown after %.2f seconds.", waitTimeout.Seconds())
		if !tgo.ReturnAfter(waitTimeout, co.producerWorker.Wait) {
			logrus.Error("At least one producer found to be blocking.")
		}
	}
}
<commit_msg>Forced log consumer to be the last consumer to shut down<commit_after>// Copyright 2015-2016 trivago GmbH
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"github.com/sirupsen/logrus"
	"github.com/trivago/gollum/core"
	"github.com/trivago/gollum/logger"
	"github.com/trivago/tgo"
	"os"
	"os/signal"
	"reflect"
	"sync"
	"time"
)

const (
	coordinatorStateConfigure      = coordinatorState(iota)
	coordinatorStateStartProducers = coordinatorState(iota)
	coordinatorStateStartConsumers = coordinatorState(iota)
	coordinatorStateRunning        = coordinatorState(iota)
	coordinatorStateShutdown       = coordinatorState(iota)
	coordinatorStateStopConsumers  = coordinatorState(iota)
	coordinatorStateStopProducers  = coordinatorState(iota)
	coordinatorStateStopped        = coordinatorState(iota)
)

const (
	signalNone = signalType(iota)
	signalExit = signalType(iota)
	signalRoll = signalType(iota)
)

type coordinatorState byte
type signalType byte

// Coordinator is the main gollum instance taking care of starting and stopping
// plugins.
type Coordinator struct {
	consumers      []core.Consumer
	producers      []core.Producer
	routers        []core.Router
	consumerWorker *sync.WaitGroup
	producerWorker *sync.WaitGroup
	logConsumer    *core.LogConsumer
	state          coordinatorState
	signal         chan os.Signal
}

// NewCoordinator creates a new multplexer
func NewCoordinator() Coordinator {
	return Coordinator{
		consumerWorker: new(sync.WaitGroup),
		producerWorker: new(sync.WaitGroup),
		state:          coordinatorStateConfigure,
	}
}

// Configure processes the config and instantiates all valid plugins
func (co *Coordinator) Configure(conf *core.Config) {
	// Make sure the log is printed to stderr if we are stuck here
	logFallback := time.AfterFunc(time.Duration(3)*time.Second, func() {
		//logrus.SetOutput(os.Stderr)
		logrusHookBuffer.SetTargetWriter(logger.FallbackLogDevice)
		logrusHookBuffer.Purge()
	})
	defer logFallback.Stop()

	// Initialize the plugins in the order of routers > producers > consumers
	// to match the order of reference between the different types.

	co.configureRouters(conf)
	co.configureProducers(conf)
	co.configureConsumers(conf)

	// As consumers might create new fallback router this is the first position
	// where we can add the wildcard producers to all streams. No new routers
	// created beyond this point must use StreamRegistry.AddWildcardProducersToRouter.

	core.StreamRegistry.AddAllWildcardProducersToAllRouters()
}

// StartPlugins starts all plugins in the correct order.
func (co *Coordinator) StartPlugins() {

	if len(co.consumers) == 0 {
		logrus.Error("No consumers configured.")
		logrusHookBuffer.SetTargetWriter(logger.FallbackLogDevice)
		logrusHookBuffer.Purge()
		return // ### return, nothing to do ###
	}

	if len(co.producers) == 0 {
		logrus.Error("No producers configured.")
		logrusHookBuffer.SetTargetWriter(logger.FallbackLogDevice)
		logrusHookBuffer.Purge()
		return // ### return, nothing to do ###
	}

	// Launch routers
	for _, router := range co.routers {
		if err := router.Start(); err != nil {
			logrus.Error("Router was not able to start from type ", reflect.TypeOf(router), ": ", err)
		} else {
			logrus.Debug("Starting ", reflect.TypeOf(router))
		}
	}

	// Launch producers
	co.state = coordinatorStateStartProducers
	for _, producer := range co.producers {
		producer := producer
		go tgo.WithRecoverShutdown(func() {
			logrus.Debug("Starting ", reflect.TypeOf(producer))
			producer.Produce(co.producerWorker)
		})
	}

	// Set final log target and purge the intermediate buffer
	if core.StreamRegistry.IsStreamRegistered(core.LogInternalStreamID) {
		// The _GOLLUM_ stream has listeners, so use LogConsumer to write to it
		if *flagLogColors == "always" {
			logrus.SetFormatter(logger.NewConsoleFormatter())
		}
		logrusHookBuffer.SetTargetHook(co.logConsumer)
		logrusHookBuffer.Purge()

	} else {
		// _GOLLUM_ not used, so write to the fallback device
		if *flagLogColors == "always" ||
			(*flagLogColors == "auto" && logrus.IsTerminal(logger.FallbackLogDevice)) {
			// Logrus doesn't know the final log device, so we hint the color option here
			logrus.SetFormatter(logger.NewConsoleFormatter())
		}
		logrusHookBuffer.SetTargetWriter(logger.FallbackLogDevice)
		logrusHookBuffer.Purge()
	}

	// Launch consumers
	co.state = coordinatorStateStartConsumers
	for _, consumer := range co.consumers {
		consumer := consumer
		go tgo.WithRecoverShutdown(func() {
			logrus.Debug("Starting ", reflect.TypeOf(consumer))
			consumer.Consume(co.consumerWorker)
		})
	}
}

// Run is essentially the Coordinator main loop.
// It listens for shutdown signals and updates global metrics
func (co *Coordinator) Run() {
	co.signal = newSignalHandler()
	defer signal.Stop(co.signal)

	logrus.Info("We be nice to them, if they be nice to us. (startup)")

	for {
		sig := <-co.signal
		switch translateSignal(sig) {
		case signalExit:
			logrus.Info("Master betrayed us. Wicked. Tricksy, False. (signal)")
			return // ### return, exit requested ###

		case signalRoll:
			for _, consumer := range co.consumers {
				consumer.Control() <- core.PluginControlRoll
			}
			for _, producer := range co.producers {
				producer.Control() <- core.PluginControlRoll
			}

		default:
		}
	}
}

// Shutdown all consumers and producers in a clean way.
// The internal log is flushed after the consumers have been shut down so that
// consumer related messages are still in the tlog.
// Producers are flushed after flushing the log, so producer related shutdown
// messages will be posted to stdout
func (co *Coordinator) Shutdown() {
	logrus.Info("Filthy little hobbites. They stole it from us. (shutdown)")

	stateAtShutdown := co.state
	co.state = coordinatorStateShutdown

	co.shutdownConsumers(stateAtShutdown)

	// Make sure remaining warning / errors are written to stderr
	logrus.Info("I'm not listening... I'm not listening... (flushing)")
	//logrus.SetOutput(fallbackLogDevice)
	logrusHookBuffer.SetTargetWriter(logger.FallbackLogDevice)
	logrusHookBuffer.SetTargetHook(nil)

	// Shutdown producers
	co.shutdownProducers(stateAtShutdown)

	co.state = coordinatorStateStopped
}

func (co *Coordinator) configureRouters(conf *core.Config) {
	routerConfigs := conf.GetRouters()
	for _, config := range routerConfigs {
		logrus.Debugf("Instantiating router '%s'", config.ID)

		plugin, err := core.NewPluginWithConfig(config)
		if err != nil {
			logrus.Errorf("Failed to instantiate router %s: %s", config.ID, err)
			continue // ### continue ###
		}

		routerPlugin := plugin.(core.Router)
		co.routers = append(co.routers, routerPlugin)

		logrus.Debugf("Instantiated '%s' (%s) as '%s'", config.ID, core.StreamRegistry.GetStreamName(routerPlugin.GetStreamID()), config.Typename)
		core.StreamRegistry.Register(routerPlugin, routerPlugin.GetStreamID())
	}
}

func (co *Coordinator) configureProducers(conf *core.Config) {
	co.state = coordinatorStateStartProducers

	// All producers are added to the wildcard stream so that consumers can send
	// to all producers if required. The wildcard producer list is required
	// to add producers listening to all routers to all streams that are used.
	wildcardStream := core.StreamRegistry.GetRouterOrFallback(core.WildcardStreamID)
	producerConfigs := conf.GetProducers()

	for _, config := range producerConfigs {
		logrus.Debug("Instantiating ", config.ID)

		plugin, err := core.NewPluginWithConfig(config)
		if err != nil {
			logrus.Errorf("Failed to instantiate producer %s: %s", config.ID, err)
			continue // ### continue ###
		}

		producer, _ := plugin.(core.Producer)
		streams := producer.Streams()

		if len(streams) == 0 {
			logrus.Error("Producer ", config.ID, " has no streams set")
			continue // ### continue ###
		}

		co.producers = append(co.producers, producer)
		core.CountProducers()

		// Attach producer to streams

		for _, streamID := range streams {
			if streamID == core.WildcardStreamID {
				core.StreamRegistry.RegisterWildcardProducer(producer)
			} else {
				router := core.StreamRegistry.GetRouterOrFallback(streamID)
				router.AddProducer(producer)
			}
		}

		// Add producer to wildcard stream unless it only listens to internal streams
	searchinternal:
		for _, streamID := range streams {
			switch streamID {
			case core.LogInternalStreamID:
			default:
				wildcardStream.AddProducer(producer)
				break searchinternal
			}
		}
	}
}

func (co *Coordinator) configureConsumers(conf *core.Config) {
	co.state = coordinatorStateStartConsumers
	co.configureLogConsumer()

	consumerConfigs := conf.GetConsumers()
	for _, config := range consumerConfigs {
		logrus.Debug("Instantiating ", config.ID)

		plugin, err := core.NewPluginWithConfig(config)
		if err != nil {
			logrus.Errorf("Failed to instantiate producer %s: %s", config.ID, err)
			continue // ### continue ###
		}

		consumer, _ := plugin.(core.Consumer)
		co.consumers = append(co.consumers, consumer)
		core.CountConsumers()
	}
}

func (co *Coordinator) configureLogConsumer() {
	config := core.NewPluginConfig("", "core.LogConsumer")
	configReader := core.NewPluginConfigReader(&config)

	co.logConsumer = new(core.LogConsumer)
	co.logConsumer.Configure(configReader)
	co.consumers = append(co.consumers, co.logConsumer)
}

func (co *Coordinator) shutdownConsumers(stateAtShutdown coordinatorState) {
	if stateAtShutdown >= coordinatorStateStartConsumers {
		co.state = coordinatorStateStopConsumers
		waitTimeout := time.Duration(0)

		logrus.Debug("Telling consumers to stop")
		for _, cons := range co.consumers {
			timeout := cons.GetShutdownTimeout()
			if timeout > waitTimeout {
				waitTimeout = timeout
			}

			// Skip log consumer so we get clean log messages for all consumers
			if cons != co.logConsumer {
				cons.Control() <- core.PluginControlStopConsumer
			}
		}

		waitTimeout *= 10
		logrus.Debugf("Waiting for consumers to stop. Forced shutdown after %.2f seconds.", waitTimeout.Seconds())
		if co.logConsumer != nil {
			co.logConsumer.Control() <- core.PluginControlStopConsumer
			// No logs in _GOLLUM_ after this point
		}

		if !tgo.ReturnAfter(waitTimeout, co.consumerWorker.Wait) {
			logrus.Error("At least one consumer found to be blocking.")
		}
	}
}

func (co *Coordinator) shutdownProducers(stateAtShutdown coordinatorState) {
	if stateAtShutdown >= coordinatorStateStartProducers {
		co.state = coordinatorStateStopProducers
		waitTimeout := time.Duration(0)

		logrus.Debug("Telling producers to stop")
		for _, prod := range co.producers {
			timeout := prod.GetShutdownTimeout()
			if timeout > waitTimeout {
				waitTimeout = timeout
			}
			prod.Control() <- core.PluginControlStopProducer
		}

		waitTimeout *= 10
		logrus.Debugf("Waiting for producers to stop. Forced shutdown after %.2f seconds.", waitTimeout.Seconds())
		if !tgo.ReturnAfter(waitTimeout, co.producerWorker.Wait) {
			logrus.Error("At least one producer found to be blocking.")
		}
	}
}
<|endoftext|>


<commit_before>package test

import (
	"fmt"
	"os"
	"os/exec"
	"testing"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/stretchr/testify/assert"

	"github.com/libopenstorage/kvdb"
	"github.com/libopenstorage/kvdb/mem"
	"github.com/libopenstorage/openstorage/api"
	"github.com/libopenstorage/openstorage/volume"
)

// Context maintains current device state. It gets passed into tests
// so that tests can build on other tests' work
type Context struct {
	volume.VolumeDriver
	volID      api.VolumeID
	snapID     api.SnapID
	mountPath  string
	tgtPath    string
	devicePath string
	Filesystem string
}

func NewContext(d volume.VolumeDriver) *Context {
	return &Context{
		VolumeDriver: d,
		volID:        api.BadVolumeID,
		snapID:       api.BadSnapID,
		Filesystem:   string(""),
	}
}

func RunShort(t *testing.T, ctx *Context) {
	create(t, ctx)
	inspect(t, ctx)
	enumerate(t, ctx)
	attach(t, ctx)
	format(t, ctx)
	mount(t, ctx)
	io(t, ctx)
	unmount(t, ctx)
	detach(t, ctx)
	delete(t, ctx)
	runEnd(t, ctx)
}

func Run(t *testing.T, ctx *Context) {
	RunShort(t, ctx)
	RunSnap(t, ctx)
	runEnd(t, ctx)
}

func runEnd(t *testing.T, ctx *Context) {
	time.Sleep(time.Second * 2)
	shutdown(t, ctx)
}

func RunSnap(t *testing.T, ctx *Context) {
	snap(t, ctx)
	snapInspect(t, ctx)
	snapEnumerate(t, ctx)
	snapDiff(t, ctx)
	snapDelete(t, ctx)
}

func create(t *testing.T, ctx *Context) {
	fmt.Println("create")

	volID, err := ctx.Create(
		api.VolumeLocator{Name: "foo"},
		&api.CreateOptions{FailIfExists: false},
		&api.VolumeSpec{
			Size:    1 * 1024 * 1024 * 1024,
			HALevel: 1,
			Format:  api.Filesystem(ctx.Filesystem),
		})

	assert.NoError(t, err, "Failed in Create")
	ctx.volID = volID
}

func inspect(t *testing.T, ctx *Context) {
	fmt.Println("inspect")

	vols, err := ctx.Inspect([]api.VolumeID{ctx.volID})
	assert.NoError(t, err, "Failed in Inspect")
	assert.NotNil(t, vols, "Nil vols")
	assert.Equal(t, len(vols), 1, "Expect 1 volume actual %v volumes", len(vols))
	assert.Equal(t, vols[0].ID, ctx.volID, "Expect volID %v actual %v", ctx.volID, vols[0].ID)

	vols, err = ctx.Inspect([]api.VolumeID{api.VolumeID("shouldNotExist")})
	assert.Equal(t, 0, len(vols), "Expect 0 volume actual %v volumes", len(vols))
}

func enumerate(t *testing.T, ctx *Context) {
	fmt.Println("enumerate")

	vols, err := ctx.Enumerate(api.VolumeLocator{}, nil)
	assert.NoError(t, err, "Failed in Enumerate")
	assert.NotNil(t, vols, "Nil vols")
	assert.Equal(t, 1, len(vols), "Expect 1 volume actual %v volumes", len(vols))
	assert.Equal(t, vols[0].ID, ctx.volID, "Expect volID %v actual %v", ctx.volID, vols[0].ID)

	vols, err = ctx.Enumerate(api.VolumeLocator{Name: "foo"}, nil)
	assert.NoError(t, err, "Failed in Enumerate")
	assert.NotNil(t, vols, "Nil vols")
	assert.Equal(t, len(vols), 1, "Expect 1 volume actual %v volumes", len(vols))
	assert.Equal(t, vols[0].ID, ctx.volID, "Expect volID %v actual %v", ctx.volID, vols[0].ID)

	vols, err = ctx.Enumerate(api.VolumeLocator{Name: "shouldNotExist"}, nil)
	assert.Equal(t, len(vols), 0, "Expect 0 volume actual %v volumes", len(vols))
}

func format(t *testing.T, ctx *Context) {
	fmt.Println("format")

	err := ctx.Format(ctx.volID)
	if err != nil {
		assert.Equal(t, err, volume.ErrNotSupported, "Error on format %v", err)
	}
}

func attach(t *testing.T, ctx *Context) {
	fmt.Println("attach")
	p, err := ctx.Attach(ctx.volID)
	if err != nil {
		assert.Equal(t, err, volume.ErrNotSupported, "Error on attach %v", err)
	}
	ctx.devicePath = p

	p, err = ctx.Attach(ctx.volID)
	if err == nil {
		assert.Equal(t, p, ctx.devicePath, "Multiple calls to attach if not errored should return the same path")
	}
}

func detach(t *testing.T, ctx *Context) {
	fmt.Println("detach")
	err := ctx.Detach(ctx.volID)
	if err != nil {
		assert.Equal(t, ctx.devicePath, "", "Error on detach %s: %v", ctx.devicePath, err)
	}
	ctx.devicePath = ""
}

func mount(t *testing.T, ctx *Context) {
	fmt.Println("mount")

	mountPath := "/mnt/voltest"
	err := os.MkdirAll(mountPath, 0755)

	tgtPath := "/mnt/foo"
	err = os.MkdirAll(tgtPath, 0755)
	assert.NoError(t, err, "Failed in mkdir")

	err = ctx.Mount(ctx.volID, tgtPath)
	assert.NoError(t, err, "Failed in mount")

	ctx.mountPath = mountPath
	ctx.tgtPath = tgtPath
}

func unmount(t *testing.T, ctx *Context) {
	fmt.Println("unmount")

	assert.NotEqual(t, ctx.mountPath, "", "Device is not mounted")

	err := ctx.Unmount(ctx.volID, ctx.mountPath)
	assert.NoError(t, err, "Failed in unmount")

	ctx.mountPath = ""
	ctx.tgtPath = ""
}

func shutdown(t *testing.T, ctx *Context) {
	fmt.Println("shutdown")
	ctx.Shutdown()
}

func io(t *testing.T, ctx *Context) {
	fmt.Println("io")
	assert.NotEqual(t, ctx.mountPath, "", "Device is not mounted")

	cmd := exec.Command("dd", "if=/dev/urandom", "of=/tmp/xx", "bs=1M", "count=10")
	err := cmd.Run()
	assert.NoError(t, err, "Failed to run dd")

	cmd = exec.Command("dd", "if=/tmp/xx", fmt.Sprintf("of=%s/xx", ctx.mountPath))
	err = cmd.Run()
	assert.NoError(t, err, "Failed to run dd on mountpoint %s/xx", ctx.mountPath)

	cmd = exec.Command("diff", "if=/tmp/xx", fmt.Sprintf("of=%s/xx", ctx.mountPath))
	assert.NoError(t, err, "data mismatch")
}

func detachBad(t *testing.T, ctx *Context) {
	err := ctx.Detach(ctx.volID)
	assert.True(t, (err == nil || err == volume.ErrNotSupported),
		"Detach on mounted device should fail")
}

func deleteBad(t *testing.T, ctx *Context) {
	fmt.Println("deleteBad")
	assert.NotEqual(t, ctx.mountPath, "", "Device is not mounted")

	err := ctx.Delete(ctx.volID)
	assert.Error(t, err, "Delete on mounted device must fail")
}

func delete(t *testing.T, ctx *Context) {
	fmt.Println("delete")
	err := ctx.Delete(ctx.volID)
	assert.NoError(t, err, "Delete failed")
	ctx.volID = api.BadVolumeID
}

func snap(t *testing.T, ctx *Context) {
	fmt.Println("snap")
	if ctx.volID == api.BadVolumeID {
		create(t, ctx)
	}
	assert.NotEqual(t, ctx.volID, api.BadVolumeID, "invalid volume ID")
	id, err := ctx.Snapshot(ctx.volID, api.Labels{"oh": "snap"})
	assert.NoError(t, err, "Failed in creating a snapshot")
	ctx.snapID = id
}

func snapInspect(t *testing.T, ctx *Context) {
	fmt.Println("snapInspect")

	snaps, err := ctx.SnapInspect([]api.SnapID{ctx.snapID})
	assert.NoError(t, err, "Failed in Inspect")
	assert.NotNil(t, snaps, "Nil snaps")
	assert.Equal(t, len(snaps), 1, "Expect 1 snaps actual %v snaps", len(snaps))
	assert.Equal(t, snaps[0].ID, ctx.snapID, "Expect snapID %v actual %v", ctx.snapID, snaps[0].ID)

	snaps, err = ctx.SnapInspect([]api.SnapID{api.SnapID("shouldNotExist")})
	assert.Equal(t, 0, len(snaps), "Expect 0 snaps actual %v snaps", len(snaps))
}

func snapEnumerate(t *testing.T, ctx *Context) {
	fmt.Println("snapEnumerate")

	snaps, err := ctx.SnapEnumerate(nil, nil)
	assert.NoError(t, err, "Failed in snapEnumerate")
	assert.NotNil(t, snaps, "Nil snaps")
	assert.Equal(t, 1, len(snaps), "Expect 1 snaps actual %v snaps", len(snaps))
	assert.Equal(t, snaps[0].ID, ctx.snapID, "Expect snapID %v actual %v", ctx.snapID, snaps[0].ID)
	labels := snaps[0].SnapLabels

	snaps, err = ctx.SnapEnumerate([]api.VolumeID{ctx.volID}, nil)
	assert.NoError(t, err, "Failed in snapEnumerate")
	assert.NotNil(t, snaps, "Nil snaps")
	assert.Equal(t, len(snaps), 1, "Expect 1 snap actual %v snaps", len(snaps))
	assert.Equal(t, snaps[0].ID, ctx.snapID, "Expect snapID %v actual %v", ctx.snapID, snaps[0].ID)

	snaps, err = ctx.SnapEnumerate([]api.VolumeID{api.VolumeID("shouldNotExist")}, nil)
	assert.Equal(t, len(snaps), 0, "Expect 0 snap actual %v snaps", len(snaps))

	snaps, err = ctx.SnapEnumerate(nil, labels)
	assert.NoError(t, err, "Failed in snapEnumerate")
	assert.NotNil(t, snaps, "Nil snaps")
	assert.Equal(t, len(snaps), 1, "Expect 1 snap actual %v snaps", len(snaps))
	assert.Equal(t, snaps[0].ID, ctx.snapID, "Expect snapID %v actual %v", ctx.snapID, snaps[0].ID)
}

func snapDiff(t *testing.T, ctx *Context) {
	fmt.Println("snapDiff")
}

func snapDelete(t *testing.T, ctx *Context) {
	fmt.Println("snapDelete")
}

func init() {
	kv, err := kvdb.New(mem.Name, "driver_test", []string{}, nil)
	if err != nil {
		log.Panicf("Failed to intialize KVDB")
	}
	err = kvdb.SetInstance(kv)
	if err != nil {
		log.Panicf("Failed to set KVDB instance")
	}
}
<commit_msg>blow away io test dir<commit_after>package test

import (
	"fmt"
	"os"
	"os/exec"
	"testing"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/stretchr/testify/assert"

	"github.com/libopenstorage/kvdb"
	"github.com/libopenstorage/kvdb/mem"
	"github.com/libopenstorage/openstorage/api"
	"github.com/libopenstorage/openstorage/volume"
)

const testPath = "/tmp/openstorage/mount"

// Context maintains current device state. It gets passed into tests
// so that tests can build on other tests' work
type Context struct {
	volume.VolumeDriver
	volID      api.VolumeID
	snapID     api.SnapID
	mountPath  string
	devicePath string
	Filesystem string
}

func NewContext(d volume.VolumeDriver) *Context {
	return &Context{
		VolumeDriver: d,
		volID:        api.BadVolumeID,
		snapID:       api.BadSnapID,
		Filesystem:   string(""),
	}
}

func RunShort(t *testing.T, ctx *Context) {
	create(t, ctx)
	inspect(t, ctx)
	enumerate(t, ctx)
	attach(t, ctx)
	format(t, ctx)
	mount(t, ctx)
	io(t, ctx)
	unmount(t, ctx)
	detach(t, ctx)
	delete(t, ctx)
	runEnd(t, ctx)
}

func Run(t *testing.T, ctx *Context) {
	RunShort(t, ctx)
	RunSnap(t, ctx)
	runEnd(t, ctx)
}

func runEnd(t *testing.T, ctx *Context) {
	time.Sleep(time.Second * 2)
	os.RemoveAll(testPath)
	shutdown(t, ctx)
}

func RunSnap(t *testing.T, ctx *Context) {
	snap(t, ctx)
	snapInspect(t, ctx)
	snapEnumerate(t, ctx)
	snapDiff(t, ctx)
	snapDelete(t, ctx)
}

func create(t *testing.T, ctx *Context) {
	fmt.Println("create")

	volID, err := ctx.Create(
		api.VolumeLocator{Name: "foo"},
		&api.CreateOptions{FailIfExists: false},
		&api.VolumeSpec{
			Size:    1 * 1024 * 1024 * 1024,
			HALevel: 1,
			Format:  api.Filesystem(ctx.Filesystem),
		})

	assert.NoError(t, err, "Failed in Create")
	ctx.volID = volID
}

func inspect(t *testing.T, ctx *Context) {
	fmt.Println("inspect")

	vols, err := ctx.Inspect([]api.VolumeID{ctx.volID})
	assert.NoError(t, err, "Failed in Inspect")
	assert.NotNil(t, vols, "Nil vols")
	assert.Equal(t, len(vols), 1, "Expect 1 volume actual %v volumes", len(vols))
	assert.Equal(t, vols[0].ID, ctx.volID, "Expect volID %v actual %v", ctx.volID, vols[0].ID)

	vols, err = ctx.Inspect([]api.VolumeID{api.VolumeID("shouldNotExist")})
	assert.Equal(t, 0, len(vols), "Expect 0 volume actual %v volumes", len(vols))
}

func enumerate(t *testing.T, ctx *Context) {
	fmt.Println("enumerate")

	vols, err := ctx.Enumerate(api.VolumeLocator{}, nil)
	assert.NoError(t, err, "Failed in Enumerate")
	assert.NotNil(t, vols, "Nil vols")
	assert.Equal(t, 1, len(vols), "Expect 1 volume actual %v volumes", len(vols))
	assert.Equal(t, vols[0].ID, ctx.volID, "Expect volID %v actual %v", ctx.volID, vols[0].ID)

	vols, err = ctx.Enumerate(api.VolumeLocator{Name: "foo"}, nil)
	assert.NoError(t, err, "Failed in Enumerate")
	assert.NotNil(t, vols, "Nil vols")
	assert.Equal(t, len(vols), 1, "Expect 1 volume actual %v volumes", len(vols))
	assert.Equal(t, vols[0].ID, ctx.volID, "Expect volID %v actual %v", ctx.volID, vols[0].ID)

	vols, err = ctx.Enumerate(api.VolumeLocator{Name: "shouldNotExist"}, nil)
	assert.Equal(t, len(vols), 0, "Expect 0 volume actual %v volumes", len(vols))
}

func format(t *testing.T, ctx *Context) {
	fmt.Println("format")

	err := ctx.Format(ctx.volID)
	if err != nil {
		assert.Equal(t, err, volume.ErrNotSupported, "Error on format %v", err)
	}
}

func attach(t *testing.T, ctx *Context) {
	fmt.Println("attach")
	p, err := ctx.Attach(ctx.volID)
	if err != nil {
		assert.Equal(t, err, volume.ErrNotSupported, "Error on attach %v", err)
	}
	ctx.devicePath = p

	p, err = ctx.Attach(ctx.volID)
	if err == nil {
		assert.Equal(t, p, ctx.devicePath, "Multiple calls to attach if not errored should return the same path")
	}
}

func detach(t *testing.T, ctx *Context) {
	fmt.Println("detach")
	err := ctx.Detach(ctx.volID)
	if err != nil {
		assert.Equal(t, ctx.devicePath, "", "Error on detach %s: %v", ctx.devicePath, err)
	}
	ctx.devicePath = ""
}

func mount(t *testing.T, ctx *Context) {
	fmt.Println("mount")

	err := os.MkdirAll(testPath, 0755)

	err = ctx.Mount(ctx.volID, testPath)
	assert.NoError(t, err, "Failed in mount")

	ctx.mountPath = testPath
}

func unmount(t *testing.T, ctx *Context) {
	fmt.Println("unmount")

	assert.NotEqual(t, ctx.mountPath, "", "Device is not mounted")

	err := ctx.Unmount(ctx.volID, ctx.mountPath)
	assert.NoError(t, err, "Failed in unmount")

	ctx.mountPath = ""
}

func shutdown(t *testing.T, ctx *Context) {
	fmt.Println("shutdown")
	ctx.Shutdown()
}

func io(t *testing.T, ctx *Context) {
	fmt.Println("io")
	assert.NotEqual(t, ctx.mountPath, "", "Device is not mounted")

	cmd := exec.Command("dd", "if=/dev/urandom", "of=/tmp/xx", "bs=1M", "count=10")
	err := cmd.Run()
	assert.NoError(t, err, "Failed to run dd")

	cmd = exec.Command("dd", "if=/tmp/xx", fmt.Sprintf("of=%s/xx", ctx.mountPath))
	err = cmd.Run()
	assert.NoError(t, err, "Failed to run dd on mountpoint %s/xx", ctx.mountPath)

	cmd = exec.Command("diff", "if=/tmp/xx", fmt.Sprintf("of=%s/xx", ctx.mountPath))
	assert.NoError(t, err, "data mismatch")
}

func detachBad(t *testing.T, ctx *Context) {
	err := ctx.Detach(ctx.volID)
	assert.True(t, (err == nil || err == volume.ErrNotSupported),
		"Detach on mounted device should fail")
}

func deleteBad(t *testing.T, ctx *Context) {
	fmt.Println("deleteBad")
	assert.NotEqual(t, ctx.mountPath, "", "Device is not mounted")

	err := ctx.Delete(ctx.volID)
	assert.Error(t, err, "Delete on mounted device must fail")
}

func delete(t *testing.T, ctx *Context) {
	fmt.Println("delete")
	err := ctx.Delete(ctx.volID)
	assert.NoError(t, err, "Delete failed")
	ctx.volID = api.BadVolumeID
}

func snap(t *testing.T, ctx *Context) {
	fmt.Println("snap")
	if ctx.volID == api.BadVolumeID {
		create(t, ctx)
	}
	assert.NotEqual(t, ctx.volID, api.BadVolumeID, "invalid volume ID")
	id, err := ctx.Snapshot(ctx.volID, api.Labels{"oh": "snap"})
	assert.NoError(t, err, "Failed in creating a snapshot")
	ctx.snapID = id
}

func snapInspect(t *testing.T, ctx *Context) {
	fmt.Println("snapInspect")

	snaps, err := ctx.SnapInspect([]api.SnapID{ctx.snapID})
	assert.NoError(t, err, "Failed in Inspect")
	assert.NotNil(t, snaps, "Nil snaps")
	assert.Equal(t, len(snaps), 1, "Expect 1 snaps actual %v snaps", len(snaps))
	assert.Equal(t, snaps[0].ID, ctx.snapID, "Expect snapID %v actual %v", ctx.snapID, snaps[0].ID)

	snaps, err = ctx.SnapInspect([]api.SnapID{api.SnapID("shouldNotExist")})
	assert.Equal(t, 0, len(snaps), "Expect 0 snaps actual %v snaps", len(snaps))
}

func snapEnumerate(t *testing.T, ctx *Context) {
	fmt.Println("snapEnumerate")

	snaps, err := ctx.SnapEnumerate(nil, nil)
	assert.NoError(t, err, "Failed in snapEnumerate")
	assert.NotNil(t, snaps, "Nil snaps")
	assert.Equal(t, 1, len(snaps), "Expect 1 snaps actual %v snaps", len(snaps))
	assert.Equal(t, snaps[0].ID, ctx.snapID, "Expect snapID %v actual %v", ctx.snapID, snaps[0].ID)
	labels := snaps[0].SnapLabels

	snaps, err = ctx.SnapEnumerate([]api.VolumeID{ctx.volID}, nil)
	assert.NoError(t, err, "Failed in snapEnumerate")
	assert.NotNil(t, snaps, "Nil snaps")
	assert.Equal(t, len(snaps), 1, "Expect 1 snap actual %v snaps", len(snaps))
	assert.Equal(t, snaps[0].ID, ctx.snapID, "Expect snapID %v actual %v", ctx.snapID, snaps[0].ID)

	snaps, err = ctx.SnapEnumerate([]api.VolumeID{api.VolumeID("shouldNotExist")}, nil)
	assert.Equal(t, len(snaps), 0, "Expect 0 snap actual %v snaps", len(snaps))

	snaps, err = ctx.SnapEnumerate(nil, labels)
	assert.NoError(t, err, "Failed in snapEnumerate")
	assert.NotNil(t, snaps, "Nil snaps")
	assert.Equal(t, len(snaps), 1, "Expect 1 snap actual %v snaps", len(snaps))
	assert.Equal(t, snaps[0].ID, ctx.snapID, "Expect snapID %v actual %v", ctx.snapID, snaps[0].ID)
}

func snapDiff(t *testing.T, ctx *Context) {
	fmt.Println("snapDiff")
}

func snapDelete(t *testing.T, ctx *Context) {
	fmt.Println("snapDelete")
}

func init() {
	kv, err := kvdb.New(mem.Name, "driver_test", []string{}, nil)
	if err != nil {
		log.Panicf("Failed to intialize KVDB")
	}
	err = kvdb.SetInstance(kv)
	if err != nil {
		log.Panicf("Failed to set KVDB instance")
	}
}
<|endoftext|>