<commit_before>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
	"github.com/sirupsen/logrus"
)

const VERSION = "0.0.199"

func main() {
	// XXX (reed): normalize
	level, err := logrus.ParseLevel(os.Getenv("LOG_LEVEL"))
	if err != nil {
		level = logrus.InfoLevel
	}
	logrus.SetLevel(level)

	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.166", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory; use k8s for kuberneted")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.StringVar(&conf.MgmtListen, "mgmt-listen", ":8081", "management port to run on")
	flag.IntVar(&conf.ShutdownTimeout, "shutdown-timeout", 0, "graceful shutdown timeout")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckHealthy, "hc-healthy", 1, "threshold of success checks to declare node healthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.StringVar(&conf.Namespace, "namespace", "", "kubernetes namespace to monitor")
	flag.StringVar(&conf.LabelSelector, "label-selector", "", "kubernetes label selector to monitor")
	flag.IntVar(&conf.TargetPort, "target-port", 8080, "kubernetes port to target on selected pods")

	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	db, err := lb.NewDB(conf) // Handles case where DBurl == "k8s"
	if err != nil {
		logrus.WithError(err).Fatal("error setting up database")
	}
	defer db.Close()

	g, err := lb.NewAllGrouper(conf, db)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	servers := make([]*http.Server, 0, 1)
	handler := lb.NewProxy(k, g, r, conf)

	// a separate mgmt listener is requested? then let's create a LB traffic only server
	if conf.Listen != conf.MgmtListen {
		servers = append(servers, &http.Server{Addr: conf.Listen, Handler: handler})
		handler = lb.NullHandler()
	}

	// add mgmt endpoints to the handler
	handler = g.Wrap(handler) // add/del/list endpoints
	handler = r.Wrap(handler) // stats / dash endpoint

	servers = append(servers, &http.Server{Addr: conf.MgmtListen, Handler: handler})
	serve(servers, &conf)
}

func serve(servers []*http.Server, conf *lb.Config) {

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)

	for i := 0; i < len(servers); i++ {
		go func(idx int) {
			err := servers[idx].ListenAndServe()
			if err != nil && err != http.ErrServerClosed {
				logrus.WithFields(logrus.Fields{"server_id": idx}).WithError(err).Fatal("server error")
			} else {
				logrus.WithFields(logrus.Fields{"server_id": idx}).Info("server stopped")
			}
		}(i)
	}

	sig := <-ch
	logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")

	for i := 0; i < len(servers); i++ {

		ctx := context.Background()

		if conf.ShutdownTimeout > 0 {
			tmpCtx, cancel := context.WithTimeout(context.Background(), time.Duration(conf.ShutdownTimeout)*time.Second)
			ctx = tmpCtx
			defer cancel()
		}

		err := servers[i].Shutdown(ctx) // safe shutdown
		if err != nil {
			logrus.WithFields(logrus.Fields{"server_id": i}).WithError(err).Fatal("server shutdown error")
		} else {
			logrus.WithFields(logrus.Fields{"server_id": i}).Info("server shutdown")
		}
	}
}
<commit_msg>fnlb: 0.0.200 release [skip ci]<commit_after>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
	"github.com/sirupsen/logrus"
)

const VERSION = "0.0.200"

func main() {
	// XXX (reed): normalize
	level, err := logrus.ParseLevel(os.Getenv("LOG_LEVEL"))
	if err != nil {
		level = logrus.InfoLevel
	}
	logrus.SetLevel(level)

	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.167", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory; use k8s for kuberneted")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.StringVar(&conf.MgmtListen, "mgmt-listen", ":8081", "management port to run on")
	flag.IntVar(&conf.ShutdownTimeout, "shutdown-timeout", 0, "graceful shutdown timeout")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckHealthy, "hc-healthy", 1, "threshold of success checks to declare node healthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.StringVar(&conf.Namespace, "namespace", "", "kubernetes namespace to monitor")
	flag.StringVar(&conf.LabelSelector, "label-selector", "", "kubernetes label selector to monitor")
	flag.IntVar(&conf.TargetPort, "target-port", 8080, "kubernetes port to target on selected pods")

	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	db, err := lb.NewDB(conf) // Handles case where DBurl == "k8s"
	if err != nil {
		logrus.WithError(err).Fatal("error setting up database")
	}
	defer db.Close()

	g, err := lb.NewAllGrouper(conf, db)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	servers := make([]*http.Server, 0, 1)
	handler := lb.NewProxy(k, g, r, conf)

	// a separate mgmt listener is requested? then let's create a LB traffic only server
	if conf.Listen != conf.MgmtListen {
		servers = append(servers, &http.Server{Addr: conf.Listen, Handler: handler})
		handler = lb.NullHandler()
	}

	// add mgmt endpoints to the handler
	handler = g.Wrap(handler) // add/del/list endpoints
	handler = r.Wrap(handler) // stats / dash endpoint

	servers = append(servers, &http.Server{Addr: conf.MgmtListen, Handler: handler})
	serve(servers, &conf)
}

func serve(servers []*http.Server, conf *lb.Config) {

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)

	for i := 0; i < len(servers); i++ {
		go func(idx int) {
			err := servers[idx].ListenAndServe()
			if err != nil && err != http.ErrServerClosed {
				logrus.WithFields(logrus.Fields{"server_id": idx}).WithError(err).Fatal("server error")
			} else {
				logrus.WithFields(logrus.Fields{"server_id": idx}).Info("server stopped")
			}
		}(i)
	}

	sig := <-ch
	logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")

	for i := 0; i < len(servers); i++ {

		ctx := context.Background()

		if conf.ShutdownTimeout > 0 {
			tmpCtx, cancel := context.WithTimeout(context.Background(), time.Duration(conf.ShutdownTimeout)*time.Second)
			ctx = tmpCtx
			defer cancel()
		}

		err := servers[i].Shutdown(ctx) // safe shutdown
		if err != nil {
			logrus.WithFields(logrus.Fields{"server_id": i}).WithError(err).Fatal("server shutdown error")
		} else {
			logrus.WithFields(logrus.Fields{"server_id": i}).Info("server shutdown")
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
	"github.com/sirupsen/logrus"
)

const VERSION = "0.0.184"

func main() {
	// XXX (reed): normalize
	level, err := logrus.ParseLevel(os.Getenv("LOG_LEVEL"))
	if err != nil {
		level = logrus.InfoLevel
	}
	logrus.SetLevel(level)

	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.151", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory; use k8s for kuberneted")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.StringVar(&conf.MgmtListen, "mgmt-listen", ":8081", "management port to run on")
	flag.IntVar(&conf.ShutdownTimeout, "shutdown-timeout", 0, "graceful shutdown timeout")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckHealthy, "hc-healthy", 1, "threshold of success checks to declare node healthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.StringVar(&conf.Namespace, "namespace", "", "kubernetes namespace to monitor")
	flag.StringVar(&conf.LabelSelector, "label-selector", "", "kubernetes label selector to monitor")
	flag.IntVar(&conf.TargetPort, "target-port", 8080, "kubernetes port to target on selected pods")

	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	db, err := lb.NewDB(conf) // Handles case where DBurl == "k8s"
	if err != nil {
		logrus.WithError(err).Fatal("error setting up database")
	}
	defer db.Close()

	g, err := lb.NewAllGrouper(conf, db)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	servers := make([]*http.Server, 0, 1)
	handler := lb.NewProxy(k, g, r, conf)

	// a separate mgmt listener is requested? then let's create a LB traffic only server
	if conf.Listen != conf.MgmtListen {
		servers = append(servers, &http.Server{Addr: conf.Listen, Handler: handler})
		handler = lb.NullHandler()
	}

	// add mgmt endpoints to the handler
	handler = g.Wrap(handler) // add/del/list endpoints
	handler = r.Wrap(handler) // stats / dash endpoint

	servers = append(servers, &http.Server{Addr: conf.MgmtListen, Handler: handler})
	serve(servers, &conf)
}

func serve(servers []*http.Server, conf *lb.Config) {

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)

	for i := 0; i < len(servers); i++ {
		go func(idx int) {
			err := servers[idx].ListenAndServe()
			if err != nil && err != http.ErrServerClosed {
				logrus.WithFields(logrus.Fields{"server_id": idx}).WithError(err).Fatal("server error")
			} else {
				logrus.WithFields(logrus.Fields{"server_id": idx}).Info("server stopped")
			}
		}(i)
	}

	sig := <-ch
	logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")

	for i := 0; i < len(servers); i++ {

		ctx := context.Background()

		if conf.ShutdownTimeout > 0 {
			tmpCtx, cancel := context.WithTimeout(context.Background(), time.Duration(conf.ShutdownTimeout)*time.Second)
			ctx = tmpCtx
			defer cancel()
		}

		err := servers[i].Shutdown(ctx) // safe shutdown
		if err != nil {
			logrus.WithFields(logrus.Fields{"server_id": i}).WithError(err).Fatal("server shutdown error")
		} else {
			logrus.WithFields(logrus.Fields{"server_id": i}).Info("server shutdown")
		}
	}
}
<commit_msg>fnlb: 0.0.185 release [skip ci]<commit_after>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
	"github.com/sirupsen/logrus"
)

const VERSION = "0.0.185"

func main() {
	// XXX (reed): normalize
	level, err := logrus.ParseLevel(os.Getenv("LOG_LEVEL"))
	if err != nil {
		level = logrus.InfoLevel
	}
	logrus.SetLevel(level)

	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.152", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory; use k8s for kuberneted")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.StringVar(&conf.MgmtListen, "mgmt-listen", ":8081", "management port to run on")
	flag.IntVar(&conf.ShutdownTimeout, "shutdown-timeout", 0, "graceful shutdown timeout")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckHealthy, "hc-healthy", 1, "threshold of success checks to declare node healthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.StringVar(&conf.Namespace, "namespace", "", "kubernetes namespace to monitor")
	flag.StringVar(&conf.LabelSelector, "label-selector", "", "kubernetes label selector to monitor")
	flag.IntVar(&conf.TargetPort, "target-port", 8080, "kubernetes port to target on selected pods")

	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	db, err := lb.NewDB(conf) // Handles case where DBurl == "k8s"
	if err != nil {
		logrus.WithError(err).Fatal("error setting up database")
	}
	defer db.Close()

	g, err := lb.NewAllGrouper(conf, db)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	servers := make([]*http.Server, 0, 1)
	handler := lb.NewProxy(k, g, r, conf)

	// a separate mgmt listener is requested? then let's create a LB traffic only server
	if conf.Listen != conf.MgmtListen {
		servers = append(servers, &http.Server{Addr: conf.Listen, Handler: handler})
		handler = lb.NullHandler()
	}

	// add mgmt endpoints to the handler
	handler = g.Wrap(handler) // add/del/list endpoints
	handler = r.Wrap(handler) // stats / dash endpoint

	servers = append(servers, &http.Server{Addr: conf.MgmtListen, Handler: handler})
	serve(servers, &conf)
}

func serve(servers []*http.Server, conf *lb.Config) {

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)

	for i := 0; i < len(servers); i++ {
		go func(idx int) {
			err := servers[idx].ListenAndServe()
			if err != nil && err != http.ErrServerClosed {
				logrus.WithFields(logrus.Fields{"server_id": idx}).WithError(err).Fatal("server error")
			} else {
				logrus.WithFields(logrus.Fields{"server_id": idx}).Info("server stopped")
			}
		}(i)
	}

	sig := <-ch
	logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")

	for i := 0; i < len(servers); i++ {

		ctx := context.Background()

		if conf.ShutdownTimeout > 0 {
			tmpCtx, cancel := context.WithTimeout(context.Background(), time.Duration(conf.ShutdownTimeout)*time.Second)
			ctx = tmpCtx
			defer cancel()
		}

		err := servers[i].Shutdown(ctx) // safe shutdown
		if err != nil {
			logrus.WithFields(logrus.Fields{"server_id": i}).WithError(err).Fatal("server shutdown error")
		} else {
			logrus.WithFields(logrus.Fields{"server_id": i}).Info("server shutdown")
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 Jeff Foley. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package amass

import (
	"strings"

	"github.com/caffix/amass/amass/stringset"
	"github.com/caffix/recon"
)

type wildcard struct {
	Sub *Subdomain
	Ans chan bool
}

type dnsWildcard struct {
	HasWildcard bool
	Answers     *stringset.StringSet
}

// Goroutine that keeps track of DNS wildcards discovered
func (a *Amass) processWildcardMatches() {
	wildcards := make(map[string]*dnsWildcard)
loop:
	for {
		select {
		case req := <-a.wildcardMatches:
			var answer bool

			labels := strings.Split(req.Sub.Name, ".")
			last := len(labels) - len(strings.Split(req.Sub.Domain, "."))
			// Iterate over all the subdomains looking for wildcards
			for i := 1; i <= last; i++ {
				sub := strings.Join(labels[i:], ".")

				w, ok := wildcards[sub]
				if !ok {
					w = a.checkDomainForWildcard(sub, req.Sub.Domain)
					wildcards[sub] = w
				}

				if w.HasWildcard && w.Answers.Contains(req.Sub.Address) {
					answer = true
					break
				}
			}
			req.Ans <- answer
		case <-a.quit:
			break loop
		}
	}
	a.done <- struct{}{}
}

func answersToStringSet(answers []recon.DNSAnswer) *stringset.StringSet {
	ss := stringset.NewStringSet()

	for _, a := range answers {
		ss.Add(a.Data)
	}
	return ss
}

// checkDomainForWildcard detects if a domain returns an IP
// address for "bad" names, and if so, which address is used
func (a *Amass) checkDomainForWildcard(sub, root string) *dnsWildcard {
	var ss1, ss2, ss3 *stringset.StringSet

	name1 := "81very92unlikely03name." + sub
	name2 := "45another34random99name." + sub
	name3 := "just555little333me." + sub
	server := "8.8.8.8:53"

	if a1, err := a.dnsQuery(root, name1, server); err == nil {
		ss1 = answersToStringSet(a1)
	}

	if a2, err := a.dnsQuery(root, name2, server); err == nil {
		ss2 = answersToStringSet(a2)
	}

	if a3, err := a.dnsQuery(root, name3, server); err == nil {
		ss3 = answersToStringSet(a3)
	}

	if ss1 != nil && ss2 != nil && ss3 != nil {
		if !ss1.Empty() && (ss1.Equal(ss2) && ss2.Equal(ss3)) {
			return &dnsWildcard{
				HasWildcard: true,
				Answers:     ss1,
			}
		}
	}
	return &dnsWildcard{
		HasWildcard: false,
		Answers:     nil,
	}
}

// matchesWildcard - Checks subdomains in the wildcard cache for matches on the IP address
func (a *Amass) matchesWildcard(subdomain *Subdomain) bool {
	answer := make(chan bool, 2)

	a.wildcardMatches <- &wildcard{
		Sub: subdomain,
		Ans: answer,
	}
	return <-answer
}
<commit_msg>a few fixes to the wildcard detection implementation<commit_after>// Copyright 2017 Jeff Foley. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package amass

import (
	"strings"

	"github.com/caffix/amass/amass/stringset"
	"github.com/caffix/recon"
)

type wildcard struct {
	Sub *Subdomain
	Ans chan bool
}

type dnsWildcard struct {
	HasWildcard bool
	Answers     *stringset.StringSet
}

// matchesWildcard - Checks subdomains in the wildcard cache for matches on the IP address
func (a *Amass) matchesWildcard(subdomain *Subdomain) bool {
	answer := make(chan bool, 2)

	a.wildcardMatches <- &wildcard{
		Sub: subdomain,
		Ans: answer,
	}
	return <-answer
}

// Goroutine that keeps track of DNS wildcards discovered
func (a *Amass) processWildcardMatches() {
	wildcards := make(map[string]*dnsWildcard)
loop:
	for {
		select {
		case req := <-a.wildcardMatches:
			var answer bool

			labels := strings.Split(req.Sub.Name, ".")
			last := len(labels) - len(strings.Split(req.Sub.Domain, "."))
			// Iterate over all the subdomains looking for wildcards
			for i := 1; i <= last; i++ {
				sub := strings.Join(labels[i:], ".")

				w, ok := wildcards[sub]
				if !ok {
					w = a.checkDomainForWildcard(sub, req.Sub.Domain)
					wildcards[sub] = w
				}

				if w.HasWildcard && w.Answers.Contains(req.Sub.Address) {
					answer = true
					break
				}
			}
			req.Ans <- answer
		case <-a.quit:
			break loop
		}
	}
	a.done <- struct{}{}
}

// checkDomainForWildcard detects if a domain returns an IP
// address for "bad" names, and if so, which address is used
func (a *Amass) checkDomainForWildcard(sub, root string) *dnsWildcard {
	var ss1, ss2, ss3 *stringset.StringSet

	name1 := "81very92unlikely03name." + sub
	name2 := "45another34random99name." + sub
	name3 := "just555little333me." + sub
	server := a.NextNameserver()

	if a1, err := a.dnsQuery(root, name1, server); err == nil {
		ss1 = answersToStringSet(a1)
	}

	if a2, err := a.dnsQuery(root, name2, server); err == nil {
		ss2 = answersToStringSet(a2)
	}

	if a3, err := a.dnsQuery(root, name3, server); err == nil {
		ss3 = answersToStringSet(a3)
	}

	if ss1 != nil && ss2 != nil && ss3 != nil {
		if !ss1.Empty() && (ss1.Equal(ss2) && ss2.Equal(ss3)) {
			return &dnsWildcard{
				HasWildcard: true,
				Answers:     ss1,
			}
		}
	}
	return &dnsWildcard{
		HasWildcard: false,
		Answers:     nil,
	}
}

func answersToStringSet(answers []recon.DNSAnswer) *stringset.StringSet {
	ss := stringset.NewStringSet()

	for _, a := range answers {
		ss.Add(a.Data)
	}
	return ss
}
<|endoftext|>


<commit_before>package analytics

import "net/http/httptest"
import "encoding/json"
import "net/http"
import "bytes"
import "time"
import "fmt"
import "io"

func mockId() string { return "I'm unique" }

func mockTime() time.Time {
	// time.Unix(0, 0) fails on Circle
	return time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC)
}

func mockServer() (chan []byte, *httptest.Server) {
	done := make(chan []byte, 1)

	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		buf := bytes.NewBuffer(nil)
		io.Copy(buf, r.Body)

		var v interface{}
		err := json.Unmarshal(buf.Bytes(), &v)
		if err != nil {
			panic(err)
		}

		b, err := json.MarshalIndent(v, "", "  ")
		if err != nil {
			panic(err)
		}

		done <- b
	}))

	return done, server
}

func ExampleTrack() {
	body, server := mockServer()
	defer server.Close()

	client := New("h97jamjwbh")
	client.Endpoint = server.URL
	client.now = mockTime
	client.uid = mockId
	client.Size = 1

	client.Track(&Track{
		Event:  "Download",
		UserId: "123456",
		Properties: map[string]interface{}{
			"application": "Segment Desktop",
			"version":     "1.1.0",
			"platform":    "osx",
		},
	})

	fmt.Printf("%s\n", <-body)
	// Output:
	// {
	//   "batch": [
	//     {
	//       "event": "Download",
	//       "messageId": "I'm unique",
	//       "properties": {
	//         "application": "Segment Desktop",
	//         "platform": "osx",
	//         "version": "1.1.0"
	//       },
	//       "timestamp": "2009-11-10T23:00:00+0000",
	//       "type": "track",
	//       "userId": "123456"
	//     }
	//   ],
	//   "context": {
	//     "library": {
	//       "name": "analytics-go",
	//       "version": "2.1.0"
	//     }
	//   },
	//   "messageId": "I'm unique",
	//   "sentAt": "2009-11-10T23:00:00+0000"
	// }
}

func ExampleClose() {
	body, server := mockServer()
	defer server.Close()

	client := New("h97jamjwbh")
	client.Endpoint = server.URL
	client.now = mockTime
	client.uid = mockId

	client.Track(&Track{
		Event:  "Download",
		UserId: "123456",
		Properties: map[string]interface{}{
			"application": "Segment Desktop",
			"version":     "1.1.0",
			"platform":    "osx",
		},
	})

	client.Close()

	fmt.Printf("%s\n", <-body)
	// Output:
	// {
	//   "batch": [
	//     {
	//       "event": "Download",
	//       "messageId": "I'm unique",
	//       "properties": {
	//         "application": "Segment Desktop",
	//         "platform": "osx",
	//         "version": "1.1.0"
	//       },
	//       "timestamp": "2009-11-10T23:00:00+0000",
	//       "type": "track",
	//       "userId": "123456"
	//     }
	//   ],
	//   "context": {
	//     "library": {
	//       "name": "analytics-go",
	//       "version": "2.1.0"
	//     }
	//   },
	//   "messageId": "I'm unique",
	//   "sentAt": "2009-11-10T23:00:00+0000"
	// }
}

func ExampleInterval() {
	body, server := mockServer()
	defer server.Close()

	client := New("h97jamjwbh")
	client.Endpoint = server.URL
	client.now = mockTime
	client.uid = mockId

	client.Track(&Track{
		Event:  "Download",
		UserId: "123456",
		Properties: map[string]interface{}{
			"application": "Segment Desktop",
			"version":     "1.1.0",
			"platform":    "osx",
		},
	})

	// Will flush in 5 seconds (default interval).
	fmt.Printf("%s\n", <-body)
	// Output:
	// {
	//   "batch": [
	//     {
	//       "event": "Download",
	//       "messageId": "I'm unique",
	//       "properties": {
	//         "application": "Segment Desktop",
	//         "platform": "osx",
	//         "version": "1.1.0"
	//       },
	//       "timestamp": "2009-11-10T23:00:00+0000",
	//       "type": "track",
	//       "userId": "123456"
	//     }
	//   ],
	//   "context": {
	//     "library": {
	//       "name": "analytics-go",
	//       "version": "2.1.0"
	//     }
	//   },
	//   "messageId": "I'm unique",
	//   "sentAt": "2009-11-10T23:00:00+0000"
	// }
}

func ExampleTrackWithTimestampSet() {
	body, server := mockServer()
	defer server.Close()

	client := New("h97jamjwbh")
	client.Endpoint = server.URL
	client.now = mockTime
	client.uid = mockId
	client.Size = 1

	client.Track(&Track{
		Event:  "Download",
		UserId: "123456",
		Properties: map[string]interface{}{
			"application": "Segment Desktop",
			"version":     "1.1.0",
			"platform":    "osx",
		},
		Message: Message{
			Timestamp: timestamp(time.Date(2015, time.July, 10, 23, 0, 0, 0, time.UTC)),
		},
	})

	fmt.Printf("%s\n", <-body)
	// Output:
	// {
	//   "batch": [
	//     {
	//       "event": "Download",
	//       "messageId": "I'm unique",
	//       "properties": {
	//         "application": "Segment Desktop",
	//         "platform": "osx",
	//         "version": "1.1.0"
	//       },
	//       "timestamp": "2015-07-10T23:00:00+0000",
	//       "type": "track",
	//       "userId": "123456"
	//     }
	//   ],
	//   "context": {
	//     "library": {
	//       "name": "analytics-go",
	//       "version": "2.1.0"
	//     }
	//   },
	//   "messageId": "I'm unique",
	//   "sentAt": "2009-11-10T23:00:00+0000"
	// }
}

func ExampleTrack_context() {
	body, server := mockServer()
	defer server.Close()

	client := New("h97jamjwbh")
	client.Endpoint = server.URL
	client.now = mockTime
	client.uid = mockId
	client.Size = 1

	client.Track(&Track{
		Event:  "Download",
		UserId: "123456",
		Properties: map[string]interface{}{
			"application": "Segment Desktop",
			"version":     "1.1.0",
			"platform":    "osx",
		},
		Context: map[string]interface{}{
			"whatever": "here",
		},
	})

	fmt.Printf("%s\n", <-body)
	// Output:
	// {
	//   "batch": [
	//     {
	//       "context": {
	//         "whatever": "here"
	//       },
	//       "event": "Download",
	//       "messageId": "I'm unique",
	//       "properties": {
	//         "application": "Segment Desktop",
	//         "platform": "osx",
	//         "version": "1.1.0"
	//       },
	//       "timestamp": "2009-11-10T23:00:00+0000",
	//       "type": "track",
	//       "userId": "123456"
	//     }
	//   ],
	//   "context": {
	//     "library": {
	//       "name": "analytics-go",
	//       "version": "2.1.0"
	//     }
	//   },
	//   "messageId": "I'm unique",
	//   "sentAt": "2009-11-10T23:00:00+0000"
	// }
}

func ExampleTrack_many() {
	body, server := mockServer()
	defer server.Close()

	client := New("h97jamjwbh")
	client.Endpoint = server.URL
	client.now = mockTime
	client.uid = mockId
	client.Size = 3

	for i := 0; i < 5; i++ {
		client.Track(&Track{
			Event:  "Download",
			UserId: "123456",
			Properties: map[string]interface{}{
				"application": "Segment Desktop",
				"version":     i,
			},
		})
	}

	fmt.Printf("%s\n", <-body)
	// Output:
	// {
	//   "batch": [
	//     {
	//       "event": "Download",
	//       "messageId": "I'm unique",
	//       "properties": {
	//         "application": "Segment Desktop",
	//         "version": 0
	//       },
	//       "timestamp": "2009-11-10T23:00:00+0000",
	//       "type": "track",
	//       "userId": "123456"
	//     },
	//     {
	//       "event": "Download",
	//       "messageId": "I'm unique",
	//       "properties": {
	//         "application": "Segment Desktop",
	//         "version": 1
	//       },
	//       "timestamp": "2009-11-10T23:00:00+0000",
	//       "type": "track",
	//       "userId": "123456"
	//     },
	//     {
	//       "event": "Download",
	//       "messageId": "I'm unique",
	//       "properties": {
	//         "application": "Segment Desktop",
	//         "version": 2
	//       },
	//       "timestamp": "2009-11-10T23:00:00+0000",
	//       "type": "track",
	//       "userId": "123456"
	//     }
	//   ],
	//   "context": {
	//     "library": {
	//       "name": "analytics-go",
	//       "version": "2.1.0"
	//     }
	//   },
	//   "messageId": "I'm unique",
	//   "sentAt": "2009-11-10T23:00:00+0000"
	// }
}

func ExampleTrackWithIntegrations() {
	body, server := mockServer()
	defer server.Close()

	client := New("h97jamjwbh")
	client.Endpoint = server.URL
	client.now = mockTime
	client.uid = mockId
	client.Size = 1

	client.Track(&Track{
		Event:  "Download",
		UserId: "123456",
		Properties: map[string]interface{}{
			"application": "Segment Desktop",
			"version":     "1.1.0",
			"platform":    "osx",
		},
		Integrations: map[string]interface{}{
			"All":      true,
			"Intercom": false,
			"Mixpanel": true,
		},
	})

	fmt.Printf("%s\n", <-body)
	// Output:
	// {
	//   "batch": [
	//     {
	//       "event": "Download",
	//       "integrations": {
	//         "All": true,
	//         "Intercom": false,
	//         "Mixpanel": true
	//       },
	//       "messageId": "I'm unique",
	//       "properties": {
	//         "application": "Segment Desktop",
	//         "platform": "osx",
	//         "version": "1.1.0"
	//       },
	//       "timestamp": "2009-11-10T23:00:00+0000",
	//       "type": "track",
	//       "userId": "123456"
	//     }
	//   ],
	//   "context": {
	//     "library": {
	//       "name": "analytics-go",
	//       "version": "2.1.0"
	//     }
	//   },
	//   "messageId": "I'm unique",
	//   "sentAt": "2009-11-10T23:00:00+0000"
	// }
}
<commit_msg>Add test<commit_after>package analytics

import "net/http/httptest"
import "encoding/json"
import "net/http"
import "bytes"
import "time"
import "fmt"
import "io"

func mockId() string { return "I'm unique" }

func mockTime() time.Time {
	// time.Unix(0, 0) fails on Circle
	return time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC)
}

func mockServer() (chan []byte, *httptest.Server) {
	done := make(chan []byte, 1)

	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		buf := bytes.NewBuffer(nil)
		io.Copy(buf, r.Body)

		var v interface{}
		err := json.Unmarshal(buf.Bytes(), &v)
		if err != nil {
			panic(err)
		}

		b, err := json.MarshalIndent(v, "", "  ")
		if err != nil {
			panic(err)
		}

		done <- b
	}))

	return done, server
}

func ExampleTrack() {
	body, server := mockServer()
	defer server.Close()

	client := New("h97jamjwbh")
	client.Endpoint = server.URL
	client.now = mockTime
	client.uid = mockId
	client.Size = 1

	client.Track(&Track{
		Event:  "Download",
		UserId: "123456",
		Properties: map[string]interface{}{
			"application": "Segment Desktop",
			"version":     "1.1.0",
			"platform":    "osx",
		},
	})

	fmt.Printf("%s\n", <-body)
	// Output:
	// {
	//   "batch": [
	//     {
	//       "event": "Download",
	//       "messageId": "I'm unique",
	//       "properties": {
	//         "application": "Segment Desktop",
	//         "platform": "osx",
	//         "version": "1.1.0"
	//       },
	//       "timestamp": "2009-11-10T23:00:00+0000",
	//       "type": "track",
	//       "userId": "123456"
	//     }
	//   ],
	//   "context": {
	//     "library": {
	//       "name": "analytics-go",
	//       "version": "2.1.0"
	//     }
	//   },
	//   "messageId": "I'm unique",
	//   "sentAt": "2009-11-10T23:00:00+0000"
	// }
}

func ExampleClose() {
	body, server := mockServer()
	defer server.Close()

	client := New("h97jamjwbh")
	client.Endpoint = server.URL
	client.now = mockTime
	client.uid = mockId

	client.Track(&Track{
		Event:  "Download",
		UserId: "123456",
		Properties: map[string]interface{}{
			"application": "Segment Desktop",
			"version":     "1.1.0",
			"platform":    "osx",
		},
	})

	client.Close()

	fmt.Printf("%s\n", <-body)
	// Output:
	// {
	//   "batch": [
	//     {
	//       "event": "Download",
	//       "messageId": "I'm unique",
	//       "properties": {
	//         "application": "Segment Desktop",
	//         "platform": "osx",
	//         "version": "1.1.0"
	//       },
	//       "timestamp": "2009-11-10T23:00:00+0000",
	//       "type": "track",
	//       "userId": "123456"
	//     }
	//   ],
	//   "context": {
	//     "library": {
	//       "name": "analytics-go",
	//       "version": "2.1.0"
	//     }
	//   },
	//   "messageId": "I'm unique",
	//   "sentAt": "2009-11-10T23:00:00+0000"
	// }
}

func ExampleInterval() {
	body, server := mockServer()
	defer server.Close()

	client := New("h97jamjwbh")
	client.Endpoint = server.URL
	client.now = mockTime
	client.uid = mockId

	client.Track(&Track{
		Event:  "Download",
		UserId: "123456",
		Properties: map[string]interface{}{
			"application": "Segment Desktop",
			"version":     "1.1.0",
			"platform":    "osx",
		},
	})

	// Will flush in 5 seconds (default interval).
	fmt.Printf("%s\n", <-body)
	// Output:
	// {
	//   "batch": [
	//     {
	//       "event": "Download",
	//       "messageId": "I'm unique",
	//       "properties": {
	//         "application": "Segment Desktop",
	//         "platform": "osx",
	//         "version": "1.1.0"
	//       },
	//       "timestamp": "2009-11-10T23:00:00+0000",
	//       "type": "track",
	//       "userId": "123456"
	//     }
	//   ],
	//   "context": {
	//     "library": {
	//       "name": "analytics-go",
	//       "version": "2.1.0"
	//     }
	//   },
	//   "messageId": "I'm unique",
	//   "sentAt": "2009-11-10T23:00:00+0000"
	// }
}

func ExampleTrackWithTimestampSet() {
	body, server := mockServer()
	defer server.Close()

	client := New("h97jamjwbh")
	client.Endpoint = server.URL
	client.now = mockTime
	client.uid = mockId
	client.Size = 1

	client.Track(&Track{
		Event:  "Download",
		UserId: "123456",
		Properties: map[string]interface{}{
			"application": "Segment Desktop",
			"version":     "1.1.0",
			"platform":    "osx",
		},
		Message: Message{
			Timestamp: timestamp(time.Date(2015, time.July, 10, 23, 0, 0, 0, time.UTC)),
		},
	})

	fmt.Printf("%s\n", <-body)
	// Output:
	// {
	//   "batch": [
	//     {
	//       "event": "Download",
	//       "messageId": "I'm unique",
	//       "properties": {
	//         "application": "Segment Desktop",
	//         "platform": "osx",
	//         "version": "1.1.0"
	//       },
	//       "timestamp": "2015-07-10T23:00:00+0000",
	//       "type": "track",
	//       "userId": "123456"
	//     }
	//   ],
	//   "context": {
	//     "library": {
	//       "name": "analytics-go",
	//       "version": "2.1.0"
	//     }
	//   },
	//   "messageId": "I'm unique",
	//   "sentAt": "2009-11-10T23:00:00+0000"
	// }
}

func ExampleTrackWithMessageIdSet() {
	body, server := mockServer()
	defer server.Close()

	client := New("h97jamjwbh")
	client.Endpoint = server.URL
	client.now = mockTime
	client.uid = mockId
	client.Size = 1

	client.Track(&Track{
		Event:  "Download",
		UserId: "123456",
		Properties: map[string]interface{}{
			"application": "Segment Desktop",
			"version":     "1.1.0",
			"platform":    "osx",
		},
		Message: Message{
			MessageId: "abc",
		},
	})

	fmt.Printf("%s\n", <-body)
	// Output:
	// {
	//   "batch": [
	//     {
	//       "event": "Download",
	//       "messageId": "abc",
	//       "properties": {
	//         "application": "Segment Desktop",
	//         "platform": "osx",
	//         "version": "1.1.0"
	//       },
	//       "timestamp": "2009-11-10T23:00:00+0000",
	//       "type": "track",
	//       "userId": "123456"
	//     }
	//   ],
	//   "context": {
	//     "library": {
	//       "name": "analytics-go",
	//       "version": "2.1.0"
	//     }
	//   },
	//   "messageId": "I'm unique",
	//   "sentAt": "2009-11-10T23:00:00+0000"
	// }
}

func ExampleTrack_context() {
	body, server := mockServer()
	defer server.Close()

	client := New("h97jamjwbh")
	client.Endpoint = server.URL
	client.now = mockTime
	client.uid = mockId
	client.Size = 1

	client.Track(&Track{
		Event:  "Download",
		UserId: "123456",
		Properties: map[string]interface{}{
			"application": "Segment Desktop",
			"version":     "1.1.0",
			"platform":    "osx",
		},
		Context: map[string]interface{}{
			"whatever": "here",
		},
	})

	fmt.Printf("%s\n", <-body)
	// Output:
	// {
	//   "batch": [
	//     {
	//       "context": {
	//         "whatever": "here"
	//       },
	//       "event": "Download",
	//       "messageId": "I'm unique",
	//       "properties": {
	//         "application": "Segment Desktop",
	//         "platform": "osx",
	//         "version": "1.1.0"
	//       },
	//       "timestamp": "2009-11-10T23:00:00+0000",
	//       "type": "track",
	//       "userId": "123456"
	//     }
	//   ],
	//   "context": {
	//     "library": {
	//       "name": "analytics-go",
	//       "version": "2.1.0"
	//     }
	//   },
	//   "messageId": "I'm unique",
	//   "sentAt": "2009-11-10T23:00:00+0000"
	// }
}

func ExampleTrack_many() {
	body, server := mockServer()
	defer server.Close()

	client := New("h97jamjwbh")
	client.Endpoint = server.URL
	client.now = mockTime
	client.uid = mockId
	client.Size = 3

	for i := 0; i < 5; i++ {
		client.Track(&Track{
			Event:  "Download",
			UserId: "123456",
			Properties: map[string]interface{}{
				"application": "Segment Desktop",
				"version":     i,
			},
		})
	}

	fmt.Printf("%s\n", <-body)
	// Output:
	// {
	//   "batch": [
	//     {
	//       "event": "Download",
	//       "messageId": "I'm unique",
	//       "properties": {
	//         "application": "Segment Desktop",
	//         "version": 0
	//       },
	//       "timestamp": "2009-11-10T23:00:00+0000",
	//       "type": "track",
	//       "userId": "123456"
	//     },
	//     {
	//       "event": "Download",
	//       "messageId": "I'm unique",
	//       "properties": {
	//         "application": "Segment Desktop",
	//         "version": 1
	//       },
	//       "timestamp": "2009-11-10T23:00:00+0000",
	//       "type": "track",
	//       "userId": "123456"
	//     },
	//     {
	//       "event": "Download",
	//       "messageId": "I'm unique",
	//       "properties": {
	//         "application": "Segment Desktop",
	//         "version": 2
	//       },
	//       "timestamp": "2009-11-10T23:00:00+0000",
	//       "type": "track",
	//       "userId": "123456"
	//     }
	//   ],
	//   "context": {
	//     "library": {
	//       "name": "analytics-go",
	//       "version": "2.1.0"
	//     }
	//   },
	//   "messageId": "I'm unique",
	//   "sentAt": "2009-11-10T23:00:00+0000"
	// }
}

func ExampleTrackWithIntegrations() {
	body, server := mockServer()
	defer server.Close()

	client := New("h97jamjwbh")
	client.Endpoint = server.URL
	client.now = mockTime
	client.uid = mockId
	client.Size = 1

	client.Track(&Track{
		Event:  "Download",
		UserId: "123456",
		Properties: map[string]interface{}{
			"application": "Segment Desktop",
			"version":     "1.1.0",
			"platform":    "osx",
		},
		Integrations: map[string]interface{}{
			"All":      true,
			"Intercom": false,
			"Mixpanel": true,
		},
	})

	fmt.Printf("%s\n", <-body)
	// Output:
	// {
	//   "batch": [
	//     {
	//       "event": "Download",
	//       "integrations": {
	//         "All": true,
	//         "Intercom": false,
	//         "Mixpanel": true
	//       },
	//       "messageId": "I'm unique",
	//       "properties": {
	//         "application": "Segment Desktop",
	//         "platform": "osx",
	//         "version": "1.1.0"
	//       },
	//       "timestamp": "2009-11-10T23:00:00+0000",
	//       "type": "track",
	//       "userId": "123456"
	//     }
	//   ],
	//   "context": {
	//     "library": {
	//       "name": "analytics-go",
	//       "version": "2.1.0"
	//     }
	//   },
	//   "messageId": "I'm unique",
	//   "sentAt": "2009-11-10T23:00:00+0000"
	// }
}
<|endoftext|>


<commit_before>package kiwi

// This file consists of realizations of default formatters.

/* Copyright (c) 2016-2018, Alexander I.Grafov <grafov@gmail.com>
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

* Redistributions of source code must retain the above copyright notice, this
  list of conditions and the following disclaimer.

* Redistributions in binary form must reproduce the above copyright notice,
  this list of conditions and the following disclaimer in the documentation
  and/or other materials provided with the distribution.

* Neither the name of kvlog nor the names of its
  contributors may be used to endorse or promote products derived from
  this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

ॐ तारे तुत्तारे तुरे स्व */

import (
	"bytes"
	"strconv"
	"strings"
)

// Formatter represents format of the output.
type Formatter interface {
	// Begin function allows to add prefix string for the output
	// or make some preparations before the output.
	Begin()
	// Pair function called for each key-value pair of the record.
	// ValueType is hint that helps the formatter decide how to output the value.
	// For example formatter can format string values in quotes but numbers without them.
	Pair(key, value string, valueType int)
	// Finish function allows to add suffix string for the output.
	// Also it returns result string for the displaying of the single record.
	// It may be multiline if you wish. Result has no restrictions for you imagination :)
	Finish() []byte
}

type formatLogfmt struct {
	line *bytes.Buffer
}

// UseLogfmt says that a sink uses Logfmt format for records output.
func UseLogfmt() *formatLogfmt {
	return &formatLogfmt{line: bytes.NewBuffer(make([]byte, 256))}
}

func (f *formatLogfmt) Begin() {
	f.line.Reset()
}

func (f *formatLogfmt) Pair(key, val string, valType int) {
	// TODO allow multiline values output?
	// TODO extend check for all non printable chars, so it need just check for each byte>space
	if strings.ContainsAny(key, " \n\r\t") {
		f.line.WriteString(strconv.Quote(key))
	} else {
		f.line.WriteString(key)
	}
	switch valType {
	case StringVal, CustomQuoted:
		f.line.WriteRune('=')
		f.line.WriteString(strconv.Quote(val))
	default:
		f.line.WriteRune('=')
		f.line.WriteString(val)
	}
	f.line.WriteRune(' ')
}

func (f *formatLogfmt) Finish() []byte {
	f.line.WriteRune('\n')
	return f.line.Bytes()
}

type formatJSON struct {
	line *bytes.Buffer
}

// UseJSON says that a sink uses JSON (RFC-7159) format for records output.
func UseJSON() *formatJSON {
	return &formatJSON{line: bytes.NewBuffer(make([]byte, 256))}
}

func (f *formatJSON) Begin() {
	f.line.Reset()
	f.line.WriteRune('{')
}

func (f *formatJSON) Pair(key, val string, valType int) {
	f.line.WriteString(strconv.Quote(key))
	f.line.WriteRune(':')
	switch valType {
	case StringVal, TimeVal, CustomQuoted:
		f.line.WriteString(strconv.Quote(val))
	default:
		f.line.WriteString(val)
	}
	f.line.WriteString(", ")
}

func (f *formatJSON) Finish() []byte {
	f.line.WriteRune('}')
	f.line.WriteRune('\n')
	return f.line.Bytes()
}
<commit_msg>Rename formatters<commit_after>package kiwi

// This file consists of realizations of default formatters.

/* Copyright (c) 2016-2018, Alexander I.Grafov <grafov@gmail.com>
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

* Redistributions of source code must retain the above copyright notice, this
  list of conditions and the following disclaimer.

* Redistributions in binary form must reproduce the above copyright notice,
  this list of conditions and the following disclaimer in the documentation
  and/or other materials provided with the distribution.

* Neither the name of kvlog nor the names of its
  contributors may be used to endorse or promote products derived from
  this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

ॐ तारे तुत्तारे तुरे स्व */

import (
	"bytes"
	"strconv"
	"strings"
)

// Formatter represents format of the output.
type Formatter interface {
	// Begin function allows to add prefix string for the output
	// or make some preparations before the output.
	Begin()
	// Pair function called for each key-value pair of the record.
	// ValueType is hint that helps the formatter decide how to output the value.
	// For example formatter can format string values in quotes but numbers without them.
	Pair(key, value string, valueType int)
	// Finish function allows to add suffix string for the output.
	// Also it returns result string for the displaying of the single record.
	// It may be multiline if you wish. Result has no restrictions for you imagination :)
	Finish() []byte
}

type formatLogfmt struct {
	line *bytes.Buffer
}

// AsLogfmt says that a sink uses Logfmt format for records output.
func AsLogfmt() *formatLogfmt {
	return &formatLogfmt{line: bytes.NewBuffer(make([]byte, 256))}
}

func (f *formatLogfmt) Begin() {
	f.line.Reset()
}

func (f *formatLogfmt) Pair(key, val string, valType int) {
	// TODO allow multiline values output?
	// TODO extend check for all non printable chars, so it need just check for each byte>space
	if strings.ContainsAny(key, " \n\r\t") {
		f.line.WriteString(strconv.Quote(key))
	} else {
		f.line.WriteString(key)
	}
	switch valType {
	case StringVal, CustomQuoted:
		f.line.WriteRune('=')
		f.line.WriteString(strconv.Quote(val))
	default:
		f.line.WriteRune('=')
		f.line.WriteString(val)
	}
	f.line.WriteRune(' ')
}

func (f *formatLogfmt) Finish() []byte {
	f.line.WriteRune('\n')
	return f.line.Bytes()
}

type formatJSON struct {
	line *bytes.Buffer
}

// AsJSON says that a sink uses JSON (RFC-7159) format for records output.
func AsJSON() *formatJSON {
	return &formatJSON{line: bytes.NewBuffer(make([]byte, 256))}
}

func (f *formatJSON) Begin() {
	f.line.Reset()
	f.line.WriteRune('{')
}

func (f *formatJSON) Pair(key, val string, valType int) {
	f.line.WriteString(strconv.Quote(key))
	f.line.WriteRune(':')
	switch valType {
	case StringVal, TimeVal, CustomQuoted:
		f.line.WriteString(strconv.Quote(val))
	default:
		f.line.WriteString(val)
	}
	f.line.WriteString(", ")
}

func (f *formatJSON) Finish() []byte {
	f.line.WriteRune('}')
	f.line.WriteRune('\n')
	return f.line.Bytes()
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"strings"
	"sync"
	"time"

	"github.com/miekg/dns"
)

//Forwarder Forwarder forward the request to upstreams
type Forwarder struct {
	upstreams map[string][]string
}

//NewForwarder New Forwarder
func NewForwarder() *Forwarder {
	upstreams := parseUpstreams(Conf.Upstreams)
	return &Forwarder{
		upstreams: upstreams,
	}
}

func parseUpstreams(upstreams []string) map[string][]string {
	servers := make(map[string][]string, 0)
	for _, upstream := range upstreams {
		net, server, err := parseUpstream(upstream)
		if err != nil {
			Logger.Error(err)
		}
		servers[net] = append(servers[net], server)
	}
	return servers
}

func parseUpstream(upstream string) (net, server string, err error) {
	upstreamSlice := strings.Split(upstream, "://")
	if len(upstreamSlice) != 2 {
		err = errors.New("A valid server config must contains `://` .")
		return
	}

	net = upstreamSlice[0]
	server = upstreamSlice[1]
	return
}

//Lookup Lookup the given domain with upstreams
func (f *Forwarder) Lookup(req *dns.Msg, net string) (*dns.Msg, error) {
	result := make(chan *dns.Msg, len(f.upstreams[net]))
	err := make(chan error, 1)

	var wg sync.WaitGroup
	lookup := func(net string, req *dns.Msg, server string, result chan *dns.Msg, err chan error) {
		domain := req.Question[0].Name
		Logger.Debugf("Looking up %s with %s.", domain, server)

		client := &dns.Client{
			Net:          net,
			DialTimeout:  time.Duration(Conf.Timeout.Forwarder.Read) * time.Millisecond * 10,
			WriteTimeout: time.Duration(Conf.Timeout.Forwarder.Write) * time.Millisecond * 10,
		}
		resp, _, lookupError := client.Exchange(req, server)
		if lookupError != nil {
			return
		}
		if resp != nil && resp.Rcode == dns.RcodeServerFailure {
			Logger.Warningf("%s failed to get an valid record from upstream %s", domain, server)
			return
		}

		wg.Done()
		result <- resp
	}

	for _, server := range f.upstreams[net] {
		wg.Add(1)
		go lookup(net, req, server, result, err)
	}

	wg.Wait()
	for {
		select {
		case r := <-result:
			return r, nil
		case e := <-err:
			return nil, e
		}
	}
}
<commit_msg>Fix channels in Lookup.<commit_after>package main

import (
	"errors"
	"strings"
	"sync"
	"time"

	"github.com/miekg/dns"
)

//Forwarder Forwarder forward the request to upstreams
type Forwarder struct {
	upstreams map[string][]string
}

//NewForwarder New Forwarder
func NewForwarder() *Forwarder {
	upstreams := parseUpstreams(Conf.Upstreams)
	return &Forwarder{
		upstreams: upstreams,
	}
}

func parseUpstreams(upstreams []string) map[string][]string {
	servers := make(map[string][]string, 0)
	for _, upstream := range upstreams {
		net, server, err := parseUpstream(upstream)
		if err != nil {
			Logger.Error(err)
		}
		servers[net] = append(servers[net], server)
	}
	return servers
}

func parseUpstream(upstream string) (net, server string, err error) {
	upstreamSlice := strings.Split(upstream, "://")
	if len(upstreamSlice) != 2 {
		err = errors.New("A valid server config must contains `://` .")
		return
	}

	net = upstreamSlice[0]
	server = upstreamSlice[1]
	return
}

//Lookup Lookup the given domain with upstreams
func (f *Forwarder) Lookup(req *dns.Msg, net string) (*dns.Msg, error) {
	var wg sync.WaitGroup
	lookup := func(net string, req *dns.Msg, server string, result chan *dns.Msg, err chan error) {
		defer wg.Done()
		domain := req.Question[0].Name

		client := &dns.Client{
			Net:          net,
			DialTimeout:  time.Duration(Conf.Timeout.Forwarder.Read) * time.Millisecond * 10,
			WriteTimeout: time.Duration(Conf.Timeout.Forwarder.Write) * time.Millisecond * 10,
		}

		resp, _, lookupError := client.Exchange(req, server)
		if lookupError != nil {
			err <- lookupError
			return
		}
		if resp != nil && resp.Rcode == dns.RcodeServerFailure {
			Logger.Warningf("%s failed to get an valid record from upstream %s", domain, server)
			err <- errors.New("Upstream server failure.")
			return
		}

		result <- resp
		return
	}

	result := make(chan *dns.Msg, len(f.upstreams[net]))
	err := make(chan error, len(f.upstreams[net]))
	for _, server := range f.upstreams[net] {
		wg.Add(1)
		go lookup(net, req, server, result, err)
	}

	wg.Wait()
	for {
		select {
		case r := <-result:
			return r, nil
		case e := <-err:
			return nil, e
		default:
		}
	}
}
<|endoftext|>


<commit_before>package meritop

import (
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"strconv"

	"github.com/coreos/go-etcd/etcd"
)

type taskRole int

const (
	roleNone taskRole = iota
	roleParent
	roleChild
)

const (
	DataRequestPrefix string = "/datareq"
	DataRequestTaskID string = "taskID"
	DataRequestReq    string = "req"
)

// This interface is used by application during taskgraph configuration phase.
type Bootstrap interface {
	// These allow application developer to set the task configuration so framework
	// implementation knows which task to invoke at each node.
	SetTaskBuilder(taskBuilder TaskBuilder)

	// This allow the application to specify how tasks are connection at each epoch
	SetTopology(topology Topology)

	// After all the configure is done, driver need to call start so that all
	// nodes will get into the event loop to run the application.
	Start()
}

// Note that framework can decide how update can be done, and how to serve the updatelog.
type BackedUpFramework interface {
	// Ask framework to do update on this update on this task, which consists
	// of one primary and some backup copies.
	Update(taskID uint64, log UpdateLog)
}

type Framework interface {
	// These two are useful for task to inform the framework their status change.
	// metaData has to be really small, since it might be stored in etcd.
	// Flags that parent/child's metadata of the current task is ready.
	FlagParentMetaReady(meta string)
	FlagChildMetaReady(meta string)

	// This allow the task implementation query its neighbors.
	GetTopology() Topology

	// Some task can inform all participating tasks to exit.
	Exit()

	// Some task can inform all participating tasks to new epoch
	SetEpoch(epoch uint64)

	GetLogger() log.Logger

	// Request data from parent or children.
	DataRequest(toID uint64, meta string)

	// This is used to figure out taskid for current node
	GetTaskID() uint64
}

func NewBootStrap() Bootstrap {
	return &framework{}
}

type framework struct {
	// These should be passed by outside world
	name     string
	etcdURLs []string

	// user defined interfaces
	builder  TaskBuilder
	topology Topology

	task          Task
	taskID        uint64
	epoch         uint64
	etcdClient    *etcd.Client
	stops         []chan bool
	ln            net.Listener
	addressMap    map[uint64]string // taskId -> node address. Maybe in etcd later.
	dataRespChan  chan *dataResponse
	dataCloseChan chan struct{}
}

type dataResponse struct {
	taskID uint64
	req    string
	data   []byte
}

func (f *framework) SetTaskBuilder(taskBuilder TaskBuilder) {
	f.builder = taskBuilder
}

func (f *framework) SetTopology(topology Topology) {
	f.topology = topology
}

func (f *framework) parentOrChild(taskID uint64) taskRole {
	for _, id := range f.topology.GetParents(f.epoch) {
		if taskID == id {
			return roleParent
		}
	}

	for _, id := range f.topology.GetChildren(f.epoch) {
		if taskID == id {
			return roleChild
		}
	}
	return roleNone
}

func (f *framework) Start() {
	f.etcdClient = etcd.NewClient(f.etcdURLs)
	f.topology.SetTaskID(f.taskID)
	f.epoch = 0
	f.stops = make([]chan bool, 0)
	f.dataRespChan = make(chan *dataResponse, 100)
	f.dataCloseChan = make(chan struct{})

	// Get the task implementation for this node (indentified by taskID)
	// f.task = f.builder.GetTask(f.taskID)

	// setup etcd watches
	// - create self's parent and child meta flag
	// - watch parents' child meta flag
	// - watch children's parent meta flag
	f.etcdClient.Create(MakeParentMetaPath(f.name, f.GetTaskID()), "", 0)
	f.etcdClient.Create(MakeChildMetaPath(f.name, f.GetTaskID()), "", 0)
	parentStops := f.watchAll(roleParent, f.topology.GetParents(f.epoch))
	childStops := f.watchAll(roleChild, f.topology.GetChildren(f.epoch))
	f.stops = append(f.stops, parentStops...)
	f.stops = append(f.stops, childStops...)

	go f.startHttp()
	go f.dataResponseReceiver()

	// After framework init finished, it should init task.
	f.task.Init(f.taskID, f, nil)
	f.SetEpoch(f.epoch)
}

type dataReqHandler struct {
	f *framework
}

func (h *dataReqHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path != DataRequestPrefix {
		http.Error(w, "bad path", http.StatusBadRequest)
		return
	}
	// parse url query
	q := r.URL.Query()
	fromIDStr := q.Get(DataRequestTaskID)
	fromID, err := strconv.ParseUint(fromIDStr, 0, 64)
	if err != nil {
		http.Error(w, "taskID couldn't be parsed", http.StatusBadRequest)
		return
	}
	req := q.Get(DataRequestReq)
	// ask task to serve data
	var b []byte
	switch h.f.parentOrChild(fromID) {
	case roleParent:
		b = h.f.task.ServeAsChild(fromID, req)
	case roleChild:
		b = h.f.task.ServeAsParent(fromID, req)
	default:
		http.Error(w, "taskID isn't a parent or child of this task", http.StatusBadRequest)
		return
	}

	if _, err := w.Write(b); err != nil {
		log.Printf("response write errored: %v", err)
	}
}

// Framework http server for data request.
// Each request will be in the format: "/datareq?taskID=XXX&req=XXX".
// "taskID" indicates the requesting task. "req" is the meta data for this request.
// On success, it should respond with requested data in http body.
func (f *framework) startHttp() {
	log.Printf("framework: serving http on %s", f.ln.Addr())
	if err := http.Serve(f.ln, &dataReqHandler{f}); err != nil {
		log.Fatalf("http.Serve() returns error: %v\n", err)
	}
}

// Framework event loop handles data response for requests sent in DataRequest().
func (f *framework) dataResponseReceiver() {
	for {
		select {
		case dataResp := <-f.dataRespChan:
			switch f.parentOrChild(dataResp.taskID) {
			case roleParent:
				go f.task.ParentDataReady(dataResp.taskID, dataResp.req, dataResp.data)
			case roleChild:
				go f.task.ChildDataReady(dataResp.taskID, dataResp.req, dataResp.data)
			default:
				panic("unimplemented")
			}
		case <-f.dataCloseChan:
			return
		}
	}
}

func (f *framework) stop() {
	close(f.dataCloseChan)
	for _, c := range f.stops {
		close(c)
	}
}

func (f *framework) FlagParentMetaReady(meta string) {
	f.etcdClient.Set(
		MakeParentMetaPath(f.name, f.GetTaskID()),
		meta,
		0)
}

func (f *framework) FlagChildMetaReady(meta string) {
	f.etcdClient.Set(
		MakeChildMetaPath(f.name, f.GetTaskID()),
		meta,
		0)
}

func (f *framework) SetEpoch(epoch uint64) {
	f.epoch = epoch
}

func (f *framework) watchAll(who taskRole, taskIDs []uint64) []chan bool {
	stops := make([]chan bool, len(taskIDs))

	for i, taskID := range taskIDs {
		receiver := make(chan *etcd.Response, 10)
		stop := make(chan bool, 1)
		stops[i] = stop

		var watchPath string
		var taskCallback func(uint64, string)
		switch who {
		case roleParent:
			// Watch parent's child.
			watchPath = MakeChildMetaPath(f.name, taskID)
			taskCallback = f.task.ParentMetaReady
		case roleChild:
			// Watch child's parent.
			watchPath = MakeParentMetaPath(f.name, taskID)
			taskCallback = f.task.ChildMetaReady
		default:
			panic("unimplemented")
		}

		go f.etcdClient.Watch(watchPath, 0, false, receiver, stop)
		go func(receiver <-chan *etcd.Response, taskID uint64) {
			for {
				resp, ok := <-receiver
				if !ok {
					return
				}
				if resp.Action != "set" {
					continue
				}
				taskCallback(taskID, resp.Node.Value)
			}
		}(receiver, taskID)
	}
	return stops
}

func (f *framework) DataRequest(toID uint64, req string) {
	// getAddressFromTaskID
	addr, ok := f.addressMap[toID]
	if !ok {
		log.Fatalf("ID = %d not found", toID)
		return
	}
	u := url.URL{
		Scheme: "http",
		Host:   addr,
		Path:   DataRequestPrefix,
	}
	q := u.Query()
	q.Add(DataRequestTaskID, strconv.FormatUint(f.taskID, 10))
	q.Add(DataRequestReq, req)
	u.RawQuery = q.Encode()
	urlStr := u.String()
	// send request
	// pass the response to the awaiting event loop for data response
	go func(urlStr string) {
		resp, err := http.Get(urlStr)
		if err != nil {
			log.Fatalf("http.Get(%s) returns error: %v", urlStr, err)
		}
		defer resp.Body.Close()
		if resp.StatusCode != 200 {
			log.Fatalf("response code = %d, assume = %d", resp.StatusCode, 200)
		}
		data, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			log.Fatalf("ioutil.ReadAll(%v) returns error: %v", resp.Body, err)
		}
		dataResp := &dataResponse{
			taskID: toID,
			req:    req,
			data:   data,
		}
		f.dataRespChan <- dataResp
	}(urlStr)
}

func (f *framework) GetTopology() Topology {
	panic("unimplemented")
}

func (f *framework) Exit() {
}

func (f *framework) GetLogger() log.Logger {
	panic("unimplemented")
}

func (f *framework) GetTaskID() uint64 {
	return f.taskID
}
<commit_msg>add AbortTask to framework interface<commit_after>package meritop

import (
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"strconv"

	"github.com/coreos/go-etcd/etcd"
)

type taskRole int

const (
	roleNone taskRole = iota
	roleParent
	roleChild
)

const (
	DataRequestPrefix string = "/datareq"
	DataRequestTaskID string = "taskID"
	DataRequestReq    string = "req"
)

// This interface is used by application during taskgraph configuration phase.
type Bootstrap interface {
	// These allow application developer to set the task configuration so framework
	// implementation knows which task to invoke at each node.
	SetTaskBuilder(taskBuilder TaskBuilder)

	// This allow the application to specify how tasks are connection at each epoch
	SetTopology(topology Topology)

	// After all the configure is done, driver need to call start so that all
	// nodes will get into the event loop to run the application.
	Start()
}

// Note that framework can decide how update can be done, and how to serve the updatelog.
type BackedUpFramework interface {
	// Ask framework to do update on this update on this task, which consists
	// of one primary and some backup copies.
	Update(taskID uint64, log UpdateLog)
}

type Framework interface {
	// These two are useful for task to inform the framework their status change.
	// metaData has to be really small, since it might be stored in etcd.
	// Flags that parent/child's metadata of the current task is ready.
	FlagParentMetaReady(meta string)
	FlagChildMetaReady(meta string)

	// This allow the task implementation query its neighbors.
	GetTopology() Topology

	// Some task can inform all participating tasks to exit.
	Exit()

	// This method will result in local node abort, the same task can be
	// retried by some other node. Only useful for panic inside user code.
	AbortTask()

	// Some task can inform all participating tasks to new epoch
	SetEpoch(epoch uint64)

	GetLogger() log.Logger

	// Request data from parent or children.
	DataRequest(toID uint64, meta string)

	// This is used to figure out taskid for current node
	GetTaskID() uint64
}

func NewBootStrap() Bootstrap {
	return &framework{}
}

type framework struct {
	// These should be passed by outside world
	name     string
	etcdURLs []string

	// user defined interfaces
	builder  TaskBuilder
	topology Topology

	task          Task
	taskID        uint64
	epoch         uint64
	etcdClient    *etcd.Client
	stops         []chan bool
	ln            net.Listener
	addressMap    map[uint64]string // taskId -> node address. Maybe in etcd later.
	dataRespChan  chan *dataResponse
	dataCloseChan chan struct{}
}

type dataResponse struct {
	taskID uint64
	req    string
	data   []byte
}

func (f *framework) SetTaskBuilder(taskBuilder TaskBuilder) {
	f.builder = taskBuilder
}

func (f *framework) SetTopology(topology Topology) {
	f.topology = topology
}

func (f *framework) parentOrChild(taskID uint64) taskRole {
	for _, id := range f.topology.GetParents(f.epoch) {
		if taskID == id {
			return roleParent
		}
	}

	for _, id := range f.topology.GetChildren(f.epoch) {
		if taskID == id {
			return roleChild
		}
	}
	return roleNone
}

func (f *framework) Start() {
	f.etcdClient = etcd.NewClient(f.etcdURLs)
	f.topology.SetTaskID(f.taskID)
	f.epoch = 0
	f.stops = make([]chan bool, 0)
	f.dataRespChan = make(chan *dataResponse, 100)
	f.dataCloseChan = make(chan struct{})

	// Get the task implementation for this node (indentified by taskID)
	// f.task = f.builder.GetTask(f.taskID)

	// setup etcd watches
	// - create self's parent and child meta flag
	// - watch parents' child meta flag
	// - watch children's parent meta flag
	f.etcdClient.Create(MakeParentMetaPath(f.name, f.GetTaskID()), "", 0)
	f.etcdClient.Create(MakeChildMetaPath(f.name, f.GetTaskID()), "", 0)
	parentStops := f.watchAll(roleParent, f.topology.GetParents(f.epoch))
	childStops := f.watchAll(roleChild, f.topology.GetChildren(f.epoch))
	f.stops = append(f.stops, parentStops...)
	f.stops = append(f.stops, childStops...)

	go f.startHttp()
	go f.dataResponseReceiver()

	// After framework init finished, it should init task.
	f.task.Init(f.taskID, f, nil)
	f.SetEpoch(f.epoch)
}

type dataReqHandler struct {
	f *framework
}

func (h *dataReqHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path != DataRequestPrefix {
		http.Error(w, "bad path", http.StatusBadRequest)
		return
	}
	// parse url query
	q := r.URL.Query()
	fromIDStr := q.Get(DataRequestTaskID)
	fromID, err := strconv.ParseUint(fromIDStr, 0, 64)
	if err != nil {
		http.Error(w, "taskID couldn't be parsed", http.StatusBadRequest)
		return
	}
	req := q.Get(DataRequestReq)
	// ask task to serve data
	var b []byte
	switch h.f.parentOrChild(fromID) {
	case roleParent:
		b = h.f.task.ServeAsChild(fromID, req)
	case roleChild:
		b = h.f.task.ServeAsParent(fromID, req)
	default:
		http.Error(w, "taskID isn't a parent or child of this task", http.StatusBadRequest)
		return
	}

	if _, err := w.Write(b); err != nil {
		log.Printf("response write errored: %v", err)
	}
}

// Framework http server for data request.
// Each request will be in the format: "/datareq?taskID=XXX&req=XXX".
// "taskID" indicates the requesting task. "req" is the meta data for this request.
// On success, it should respond with requested data in http body.
func (f *framework) startHttp() {
	log.Printf("framework: serving http on %s", f.ln.Addr())
	if err := http.Serve(f.ln, &dataReqHandler{f}); err != nil {
		log.Fatalf("http.Serve() returns error: %v\n", err)
	}
}

// Framework event loop handles data response for requests sent in DataRequest().
func (f *framework) dataResponseReceiver() {
	for {
		select {
		case dataResp := <-f.dataRespChan:
			switch f.parentOrChild(dataResp.taskID) {
			case roleParent:
				go f.task.ParentDataReady(dataResp.taskID, dataResp.req, dataResp.data)
			case roleChild:
				go f.task.ChildDataReady(dataResp.taskID, dataResp.req, dataResp.data)
			default:
				panic("unimplemented")
			}
		case <-f.dataCloseChan:
			return
		}
	}
}

func (f *framework) stop() {
	close(f.dataCloseChan)
	for _, c := range f.stops {
		close(c)
	}
}

func (f *framework) FlagParentMetaReady(meta string) {
	f.etcdClient.Set(
		MakeParentMetaPath(f.name, f.GetTaskID()),
		meta,
		0)
}

func (f *framework) FlagChildMetaReady(meta string) {
	f.etcdClient.Set(
		MakeChildMetaPath(f.name, f.GetTaskID()),
		meta,
		0)
}

func (f *framework) SetEpoch(epoch uint64) {
	f.epoch = epoch
}

func (f *framework) watchAll(who taskRole, taskIDs []uint64) []chan bool {
	stops := make([]chan bool, len(taskIDs))

	for i, taskID := range taskIDs {
		receiver := make(chan *etcd.Response, 10)
		stop := make(chan bool, 1)
		stops[i] = stop

		var watchPath string
		var taskCallback func(uint64, string)
		switch who {
		case roleParent:
			// Watch parent's child.
			watchPath = MakeChildMetaPath(f.name, taskID)
			taskCallback = f.task.ParentMetaReady
		case roleChild:
			// Watch child's parent.
			watchPath = MakeParentMetaPath(f.name, taskID)
			taskCallback = f.task.ChildMetaReady
		default:
			panic("unimplemented")
		}

		go f.etcdClient.Watch(watchPath, 0, false, receiver, stop)
		go func(receiver <-chan *etcd.Response, taskID uint64) {
			for {
				resp, ok := <-receiver
				if !ok {
					return
				}
				if resp.Action != "set" {
					continue
				}
				taskCallback(taskID, resp.Node.Value)
			}
		}(receiver, taskID)
	}
	return stops
}

func (f *framework) DataRequest(toID uint64, req string) {
	// getAddressFromTaskID
	addr, ok := f.addressMap[toID]
	if !ok {
		log.Fatalf("ID = %d not found", toID)
		return
	}
	u := url.URL{
		Scheme: "http",
		Host:   addr,
		Path:   DataRequestPrefix,
	}
	q := u.Query()
	q.Add(DataRequestTaskID, strconv.FormatUint(f.taskID, 10))
	q.Add(DataRequestReq, req)
	u.RawQuery = q.Encode()
	urlStr := u.String()
	// send request
	// pass the response to the awaiting event loop for data response
	go func(urlStr string) {
		resp, err := http.Get(urlStr)
		if err != nil {
			log.Fatalf("http.Get(%s) returns error: %v", urlStr, err)
		}
		defer resp.Body.Close()
		if resp.StatusCode != 200 {
			log.Fatalf("response code = %d, assume = %d", resp.StatusCode, 200)
		}
		data, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			log.Fatalf("ioutil.ReadAll(%v) returns error: %v", resp.Body, err)
		}
		dataResp := &dataResponse{
			taskID: toID,
			req:    req,
			data:   data,
		}
		f.dataRespChan <- dataResp
	}(urlStr)
}

func (f *framework) GetTopology() Topology {
	panic("unimplemented")
}

func (f *framework) Exit() {
}

func (f *framework) AbortTask() {
	panic("unimplemented")
}

func (f *framework) GetLogger() log.Logger {
	panic("unimplemented")
}

func (f *framework) GetTaskID() uint64 {
	return f.taskID
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"path/filepath"

	"code.google.com/p/go.exp/fsnotify"
)

func indexOf(element string, array []string) int {
	for i := 0; i < len(array); i++ {
		if array[i] == element {
			return i
		}
	}
	return -1
}

func collectPaths(paths []string) []string {
	// paths to be returned
	collectedPaths := make([]string, 1, 1)

	for _, thisPath := range paths {
		err := filepath.Walk(thisPath,
			// Function arg for filepath.Walk
			func(path string, info os.FileInfo, err error) error {
				if info == nil {
					fmt.Println("File or directory does not exist.", path)
				} else if info.IsDir() {
					collectedPaths = append(collectedPaths, path)
				}
				return nil
			})

		if err != nil {
			fmt.Println(err)
		}
	}

	return collectedPaths
}

func SetupWatch(paths []string, excludes []string) (int, *fsnotify.Watcher) {
	var watchedCount int

	paths = collectPaths(paths)
	excludes = collectPaths(excludes)

	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		fmt.Println("Error establishing watcher: ", err)
	}

	// establish watches
	for _, path := range paths {
		if indexOf(path, excludes) >= -1 {
			err = watcher.Watch(path)
			if err != nil {
				fmt.Println("Error: ", err, "  establishing watch on: ", path)
			}
			watchedCount++
		}
	}
	return watchedCount, watcher
}
<commit_msg>Fixed equality error when establishing watches.<commit_after>package main

import (
	"fmt"
	"os"
	"path/filepath"

	"code.google.com/p/go.exp/fsnotify"
)

func indexOf(element string, array []string) int {
	for i := 0; i < len(array); i++ {
		if array[i] == element {
			return i
		}
	}
	return -1
}

func collectPaths(paths []string) []string {
	// paths to be returned
	collectedPaths := make([]string, 1, 1)

	for _, thisPath := range paths {
		err := filepath.Walk(thisPath,
			// Function arg for filepath.Walk
			func(path string, info os.FileInfo, err error) error {
				if info == nil {
					fmt.Println("File or directory does not exist.", path)
				} else if info.IsDir() {
					collectedPaths = append(collectedPaths, path)
				}
				return nil
			})

		if err != nil {
			fmt.Println(err)
		}
	}
	return collectedPaths
}

func SetupWatch(paths []string, excludes []string) (int, *fsnotify.Watcher) {
	var watchedCount int

	paths = collectPaths(paths)
	excludes = collectPaths(excludes)

	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		fmt.Println("Error establishing watcher: ", err)
	}

	// establish watches
	for _, path := range paths {
		if indexOf(path, excludes) == -1 {
			err = watcher.Watch(path)
			if err != nil {
				fmt.Println("Error: ", err, "  establishing watch on: ", path)
			}
			watchedCount++
		}
	}
	return watchedCount, watcher
}
<|endoftext|>


<commit_before>package shared

import (
	"bufio"
	"crypto/md5"
	"crypto/rand"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/user"
	"strconv"
	"strings"
)

/*
IsTinzenite checks whether a given path is indeed a valid directory
*/
// TODO detect incomplete dir (no connected peers, etc) or write a validate method
func IsTinzenite(dirpath string) bool {
	value, _ := DirectoryExists(dirpath + "/" + TINZENITEDIR)
	return value
}

/*
IsEncrypted checks whether a given path is indeed a valid directory for an
encrypted peer.
*/
// TODO detect incomplete dir (no connected peers, etc) or write a validate method
func IsEncrypted(dirpath string) bool {
	value, _ := DirectoryExists(dirpath + "/" + LOCALDIR)
	return value
}

/*
MakeTinzeniteDir creates the directory structure for the .tinzenite directory
including the .tinignore file required for it. The given path is the path to the
directory (NOT the .TINZENITEDIR!).
*/
func MakeTinzeniteDir(root string) error {
	root = root + "/" + TINZENITEDIR
	// build directory structure
	err := MakeDirectories(root, ORGDIR+"/"+PEERSDIR, TEMPDIR, REMOVEDIR,
		LOCALDIR, LOCALDIR+"/"+REMOVESTOREDIR, RECEIVINGDIR, SENDINGDIR)
	if err != nil {
		return err
	}
	// write required .tinignore file
	return ioutil.WriteFile(root+"/"+TINIGNORE, []byte(TINDIRIGNORE), FILEPERMISSIONMODE)
}

/*
MakeEncryptedDir builds the directory structure for an encrypted peer. The given
path is the path to the directory.
*/
func MakeEncryptedDir(root string) error {
	peers := ORGDIR + "/" + PEERSDIR
	return MakeDirectories(root, LOCALDIR, ORGDIR, peers, RECEIVINGDIR, SENDINGDIR)
}

/*
RemoveDotTinzenite directory. Specifically leaves all user files but removes all
Tinzenite specific items.
*/
func RemoveDotTinzenite(path string) error {
	if !IsTinzenite(path) {
		return ErrNotTinzenite
	}
	/* TODO remove from directory list*/
	return os.RemoveAll(path + "/" + TINZENITEDIR)
}

/*
WriteDirectoryList adds the given path to the DIRECTORYLIST file. Will try to
avoid writing the same path multiple times.
*/
func WriteDirectoryList(path string) error {
	filePath, err := directoryListPath()
	if err != nil {
		return err
	}
	// make dir in case that it doesn't exist yet (root path here is the directory)
	err = MakeDirectory(filePath.RootPath())
	if err != nil {
		return err
	}
	lines, err := ReadDirectoryList()
	if err != nil {
		return err
	}
	// only add new entry if it doesn't yet exist
	if !Contains(lines, path) {
		lines = append(lines, path)
		newContent := strings.Join(lines, "\n")
		return ioutil.WriteFile(filePath.FullPath(), []byte(newContent), FILEPERMISSIONMODE)
	}
	// if already exists we're done
	return nil
}

/*
ReadDirectoryList reads all registered Tinzenite directories in the system. May
return an empty listing if none found!
*/
func ReadDirectoryList() ([]string, error) {
	filePath, err := directoryListPath()
	if err != nil {
		return nil, err
	}
	exists, err := FileExists(filePath.FullPath())
	// if file doesn't exist / error happened we're done
	if !exists || err != nil {
		return []string{}, nil
	}
	bytes, err := ioutil.ReadFile(filePath.FullPath())
	if err != nil {
		return nil, err
	}
	return strings.Split(string(bytes), "\n"), nil
}

/*
directoryListPath returns the path where the file lies.
*/
func directoryListPath() (*RelativePath, error) {
	user, err := user.Current()
	if err != nil {
		return nil, err
	}
	return CreatePath(user.HomeDir+"/.config/tinzenite/", DIRECTORYLIST), nil
}

/*
RandomHash genereates one long random hash.
*/
func RandomHash() (string, error) {
	data := make([]byte, RANDOMSEEDLENGTH)
	_, err := rand.Read(data)
	if err != nil {
		return "", err
	}
	hash := sha256.New()
	_, err = hash.Write(data)
	if err != nil {
		return "", err
	}
	return hex.EncodeToString(hash.Sum(nil)), nil
}

/*
NewIdentifier creates a new random hash that is intended as identification
strings for all manner of different objects. Length is IDMAXLENGTH.
*/
func NewIdentifier() (string, error) {
	data := make([]byte, RANDOMSEEDLENGTH)
	_, err := rand.Read(data)
	if err != nil {
		return "", err
	}
	hash := md5.New()
	_, err = hash.Write(data)
	if err != nil {
		return "", err
	}
	return hex.EncodeToString(hash.Sum(nil))[:IDMAXLENGTH], nil
}

/*
ContentHash generates the hash of the content of the given file at path.
*/
func ContentHash(path string) (string, error) {
	file, err := os.Open(path)
	if err != nil {
		return "", err
	}
	defer file.Close()
	hash := md5.New()
	buf := make([]byte, CHUNKSIZE)
	// create hash
	for amount := CHUNKSIZE; amount == CHUNKSIZE; {
		amount, _ = file.Read(buf)
		// log.Printf("Read %d bytes", amount)
		hash.Write(buf)
	}
	// return hex representation
	return hex.EncodeToString(hash.Sum(nil)), nil
}

/*
MakeDirectory creates the path.
*/
func MakeDirectory(path string) error {
	err := os.MkdirAll(path, FILEPERMISSIONMODE)
	// TODO this doesn't seem to work... why not?
	if err == os.ErrExist {
		return nil
	}
	// either successful or true error
	return err
}

/*
MakeDirectories creates a number of directories in the given root path. Useful
if a complete directory tree has to be built at once.
*/
func MakeDirectories(root string, subdirs ...string) error {
	for _, path := range subdirs {
		err := MakeDirectory(root + "/" + path)
		if err != nil {
			return err
		}
	}
	return nil
}

/*
RemoveDirContents removes all files within the given path, leaving the directory
as is.
*/
func RemoveDirContents(path string) error {
	allStat, err := ioutil.ReadDir(path)
	if err != nil {
		return err
	}
	for _, stat := range allStat {
		err := os.Remove(path + "/" + stat.Name())
		if err != nil {
			return err
		}
	}
	return nil
}

/*
FileExists checks whether a file at that location exists.
*/
func FileExists(path string) (bool, error) {
	stat, err := os.Lstat(path)
	// sadly any error means it doesn't exist, no way to differentiate easily here
	if err != nil {
		return false, nil
	}
	if stat.IsDir() {
		return false, errors.New("path is not a file")
	}
	return true, nil
}

/*
DirectoryExists checks whether a directory at that location exists.
*/
func DirectoryExists(path string) (bool, error) {
	stat, err := os.Lstat(path)
	// sadly any error means it doesn't exist, no way to differentiate easily here
	if err != nil {
		return false, nil
	}
	if !stat.IsDir() {
		return false, errors.New("path is not a directory")
	}
	return true, nil
}

/*
ObjectExists combines FileExists and DirectoryExists.
*/
func ObjectExists(path string) (bool, error) {
	file, errFile := FileExists(path)
	dir, errDir := DirectoryExists(path)
	exists := dir || file
	// if either exists we can ignore all errors as we satisified the query
	if exists {
		return true, nil
	}
	// false case without errors
	if errFile == nil && errDir == nil {
		return false, nil
	}
	// build unified error
	text := "Errors: "
	if errFile != nil {
		text += "FileExists: " + errFile.Error() + " "
	}
	if errDir != nil {
		text += "DirectoryExists: " + errDir.Error()
	}
	// return errors
	return false, errors.New(text)
}

/*
IsDirectoryEmpty checks whether the given directory is empty.
*/
func IsDirectoryEmpty(path string) (bool, error) {
	isDir, err := DirectoryExists(path)
	if err != nil {
		return false, err
	}
	if !isDir {
		return false, errors.New("directory doesn't exist")
	}
	subFiles, err := ioutil.ReadDir(path)
	if err != nil {
		return false, err
	}
	if len(subFiles) > 0 {
		return false, nil
	}
	return true, nil
}

/*
Contains check whether the string slice contains the given string value.
*/
func Contains(s []string, value string) bool {
	for _, entry := range s {
		if entry == value {
			return true
		}
	}
	return false
}

/*
GetString poses a request to the user and returns his entry as a string.
*/
func GetString(request string) string {
	fmt.Println(request)
	// read input
	reader := bufio.NewReader(os.Stdin)
	input, _ := reader.ReadString('\n')
	input = strings.Trim(input, "\n")
	return input
}

/*
GetInt poses a request to the user and returns his entry as an integer.
*/
func GetInt(request string) int {
	for {
		fmt.Println(request)
		// read input
		reader := bufio.NewReader(os.Stdin)
		input, _ := reader.ReadString('\n')
		input = strings.Trim(input, "\n")
		value, err := strconv.ParseInt(input, 10, 0)
		if err == nil {
			return int(value)
		}
	}
}

/*
Difference takes the starting and target path lists and returns the operations
required for each path to get from the starting to the target maps.
*/
func Difference(start, target map[string]bool) (created, modified, removed []string) {
	for subpath := range start {
		_, exists := target[subpath]
		if exists {
			delete(target, subpath)
			modified = append(modified, subpath)
		} else {
			removed = append(removed, subpath)
		}
	}
	for subpath := range target {
		created = append(created, subpath)
	}
	// it is important to return these sorted: create dirs before their contents for example
	return SortString(created), SortString(modified), SortString(removed)
}
<commit_msg>fixed Difference for call-by-reference of map (removed unwanted side effect), removed unneeded function<commit_after>package shared

import (
	"bufio"
	"crypto/md5"
	"crypto/rand"
	"encoding/hex"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/user"
	"strconv"
	"strings"
)

/*
IsTinzenite checks whether a given path is indeed a valid directory
*/
// TODO detect incomplete dir (no connected peers, etc) or write a validate method
func IsTinzenite(dirpath string) bool {
	value, _ := DirectoryExists(dirpath + "/" + TINZENITEDIR)
	return value
}

/*
IsEncrypted checks whether a given path is indeed a valid directory for an
encrypted peer.
*/
// TODO detect incomplete dir (no connected peers, etc) or write a validate method
func IsEncrypted(dirpath string) bool {
	value, _ := DirectoryExists(dirpath + "/" + LOCALDIR)
	return value
}

/*
MakeTinzeniteDir creates the directory structure for the .tinzenite directory
including the .tinignore file required for it. The given path is the path to the
directory (NOT the .TINZENITEDIR!).
*/
func MakeTinzeniteDir(root string) error {
	root = root + "/" + TINZENITEDIR
	// build directory structure
	err := MakeDirectories(root, ORGDIR+"/"+PEERSDIR, TEMPDIR, REMOVEDIR,
		LOCALDIR, LOCALDIR+"/"+REMOVESTOREDIR, RECEIVINGDIR, SENDINGDIR)
	if err != nil {
		return err
	}
	// write required .tinignore file
	return ioutil.WriteFile(root+"/"+TINIGNORE, []byte(TINDIRIGNORE), FILEPERMISSIONMODE)
}

/*
MakeEncryptedDir builds the directory structure for an encrypted peer. The given
path is the path to the directory.
*/
func MakeEncryptedDir(root string) error {
	peers := ORGDIR + "/" + PEERSDIR
	return MakeDirectories(root, LOCALDIR, ORGDIR, peers, RECEIVINGDIR, SENDINGDIR)
}

/*
RemoveDotTinzenite directory. Specifically leaves all user files but removes all
Tinzenite specific items.
*/
func RemoveDotTinzenite(path string) error {
	if !IsTinzenite(path) {
		return ErrNotTinzenite
	}
	/* TODO remove from directory list*/
	return os.RemoveAll(path + "/" + TINZENITEDIR)
}

/*
WriteDirectoryList adds the given path to the DIRECTORYLIST file. Will try to
avoid writing the same path multiple times.
*/
func WriteDirectoryList(path string) error {
	filePath, err := directoryListPath()
	if err != nil {
		return err
	}
	// make dir in case that it doesn't exist yet (root path here is the directory)
	err = MakeDirectory(filePath.RootPath())
	if err != nil {
		return err
	}
	lines, err := ReadDirectoryList()
	if err != nil {
		return err
	}
	// only add new entry if it doesn't yet exist
	if !Contains(lines, path) {
		lines = append(lines, path)
		newContent := strings.Join(lines, "\n")
		return ioutil.WriteFile(filePath.FullPath(), []byte(newContent), FILEPERMISSIONMODE)
	}
	// if already exists we're done
	return nil
}

/*
ReadDirectoryList reads all registered Tinzenite directories in the system. May
return an empty listing if none found!
*/
func ReadDirectoryList() ([]string, error) {
	filePath, err := directoryListPath()
	if err != nil {
		return nil, err
	}
	exists, err := FileExists(filePath.FullPath())
	// if file doesn't exist / error happened we're done
	if !exists || err != nil {
		return []string{}, nil
	}
	bytes, err := ioutil.ReadFile(filePath.FullPath())
	if err != nil {
		return nil, err
	}
	return strings.Split(string(bytes), "\n"), nil
}

/*
directoryListPath returns the path where the file lies.
*/
func directoryListPath() (*RelativePath, error) {
	user, err := user.Current()
	if err != nil {
		return nil, err
	}
	return CreatePath(user.HomeDir+"/.config/tinzenite/", DIRECTORYLIST), nil
}

/*
NewIdentifier creates a new random hash that is intended as identification
strings for all manner of different objects. Length is IDMAXLENGTH.
*/
func NewIdentifier() (string, error) {
	data := make([]byte, RANDOMSEEDLENGTH)
	_, err := rand.Read(data)
	if err != nil {
		return "", err
	}
	hash := md5.New()
	_, err = hash.Write(data)
	if err != nil {
		return "", err
	}
	return hex.EncodeToString(hash.Sum(nil))[:IDMAXLENGTH], nil
}

/*
ContentHash generates the hash of the content of the given file at path.
*/
func ContentHash(path string) (string, error) {
	file, err := os.Open(path)
	if err != nil {
		return "", err
	}
	defer file.Close()
	hash := md5.New()
	buf := make([]byte, CHUNKSIZE)
	// create hash
	for amount := CHUNKSIZE; amount == CHUNKSIZE; {
		amount, _ = file.Read(buf)
		// log.Printf("Read %d bytes", amount)
		hash.Write(buf)
	}
	// return hex representation
	return hex.EncodeToString(hash.Sum(nil)), nil
}

/*
MakeDirectory creates the path.
*/
func MakeDirectory(path string) error {
	err := os.MkdirAll(path, FILEPERMISSIONMODE)
	// TODO this doesn't seem to work... why not?
	if err == os.ErrExist {
		return nil
	}
	// either successful or true error
	return err
}

/*
MakeDirectories creates a number of directories in the given root path. Useful
if a complete directory tree has to be built at once.
*/
func MakeDirectories(root string, subdirs ...string) error {
	for _, path := range subdirs {
		err := MakeDirectory(root + "/" + path)
		if err != nil {
			return err
		}
	}
	return nil
}

/*
RemoveDirContents removes all files within the given path, leaving the directory
as is.
*/
func RemoveDirContents(path string) error {
	allStat, err := ioutil.ReadDir(path)
	if err != nil {
		return err
	}
	for _, stat := range allStat {
		err := os.Remove(path + "/" + stat.Name())
		if err != nil {
			return err
		}
	}
	return nil
}

/*
FileExists checks whether a file at that location exists.
*/
func FileExists(path string) (bool, error) {
	stat, err := os.Lstat(path)
	// sadly any error means it doesn't exist, no way to differentiate easily here
	if err != nil {
		return false, nil
	}
	if stat.IsDir() {
		return false, errors.New("path is not a file")
	}
	return true, nil
}

/*
DirectoryExists checks whether a directory at that location exists.
*/
func DirectoryExists(path string) (bool, error) {
	stat, err := os.Lstat(path)
	// sadly any error means it doesn't exist, no way to differentiate easily here
	if err != nil {
		return false, nil
	}
	if !stat.IsDir() {
		return false, errors.New("path is not a directory")
	}
	return true, nil
}

/*
ObjectExists combines FileExists and DirectoryExists.
*/
func ObjectExists(path string) (bool, error) {
	file, errFile := FileExists(path)
	dir, errDir := DirectoryExists(path)
	exists := dir || file
	// if either exists we can ignore all errors as we satisified the query
	if exists {
		return true, nil
	}
	// false case without errors
	if errFile == nil && errDir == nil {
		return false, nil
	}
	// build unified error
	text := "Errors: "
	if errFile != nil {
		text += "FileExists: " + errFile.Error() + " "
	}
	if errDir != nil {
		text += "DirectoryExists: " + errDir.Error()
	}
	// return errors
	return false, errors.New(text)
}

/*
IsDirectoryEmpty checks whether the given directory is empty.
*/
func IsDirectoryEmpty(path string) (bool, error) {
	isDir, err := DirectoryExists(path)
	if err != nil {
		return false, err
	}
	if !isDir {
		return false, errors.New("directory doesn't exist")
	}
	subFiles, err := ioutil.ReadDir(path)
	if err != nil {
		return false, err
	}
	if len(subFiles) > 0 {
		return false, nil
	}
	return true, nil
}

/*
Contains check whether the string slice contains the given string value.
*/
func Contains(s []string, value string) bool {
	for _, entry := range s {
		if entry == value {
			return true
		}
	}
	return false
}

/*
GetString poses a request to the user and returns his entry as a string.
*/
func GetString(request string) string {
	fmt.Println(request)
	// read input
	reader := bufio.NewReader(os.Stdin)
	input, _ := reader.ReadString('\n')
	input = strings.Trim(input, "\n")
	return input
}

/*
GetInt poses a request to the user and returns his entry as an integer.
*/
func GetInt(request string) int {
	for {
		fmt.Println(request)
		// read input
		reader := bufio.NewReader(os.Stdin)
		input, _ := reader.ReadString('\n')
		input = strings.Trim(input, "\n")
		value, err := strconv.ParseInt(input, 10, 0)
		if err == nil {
			return int(value)
		}
	}
}

/*
Difference takes the starting and target path lists and returns the operations
required for each path to get from the starting to the target maps.

NOTE: Extra care has been taken that Difference does NOT modify the passed map
references.
*/
func Difference(start, target map[string]bool) (created, modified, removed []string) {
	for subpath := range start {
		_, exists := target[subpath]
		if exists {
			modified = append(modified, subpath)
		} else {
			removed = append(removed, subpath)
		}
	}
	for subpath := range target {
		// if already exists it wasn't created, so skip
		_, exists := start[subpath]
		if exists {
			continue
		}
		created = append(created, subpath)
	}
	// it is important to return these sorted: create dirs before their contents for example
	return SortString(created), SortString(modified), SortString(removed)
}
<|endoftext|>


<commit_before>// Copyright 2015 Pushkin Ivan. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package game

import (
	"encoding/json"
	"fmt"
	"time"

	"bitbucket.org/pushkin_ivan/clever-snake/game/objects"
	"bitbucket.org/pushkin_ivan/clever-snake/game/playground"
	"github.com/golang/glog"
	"golang.org/x/net/context"
)

type Game struct {
	cxt context.Context
	p   GameProcessor
	pg  *playground.Playground
}

type errStartingGame struct {
	err error
}

func (e *errStartingGame) Error() string {
	return "cannot create game: " + e.err.Error()
}

func NewGame(cxt context.Context, pgW, pgH uint8) (*Game, error) {
	if err := cxt.Err(); err != nil {
		return nil, &errStartingGame{err}
	}

	pg, err := playground.NewPlayground(pgW, pgH)
	if err != nil {
		return nil, &errStartingGame{err}
	}

	return &Game{cxt, NewGameProcessor(), pg}, nil
}

func (g *Game) StartGame() <-chan interface{} {
	output := make(chan interface{})

	objects.CreateApple(g.p, g.pg)

	go func() {
		defer close(output)
		defer glog.Infoln("finishing game")

		// all running objects work like this code:
		for {
			select {
			case <-g.cxt.Done():
				return
			case <-time.Tick(time.Second * 3):
				output <- "test"
			}
		}
	}()

	return output
}

type StartPlayerFunc func(cxt context.Context, input <-chan *Command,
) (<-chan interface{}, error)

func (g *Game) StartPlayer(cxt context.Context, input <-chan *Command,
) (<-chan interface{}, error) {
	if err := cxt.Err(); err != nil {
		return nil, fmt.Errorf("cannot start player: %s", err)
	}

	objects.CreateSnake(g.p, g.pg, g.cxt)
	output := make(chan interface{})

	go func() {

		defer close(output)
		defer glog.Infoln("finishing player")

		select {
		case <-cxt.Done():
		case <-time.After(time.Second):
		}

		for {
			select {
			case <-cxt.Done():
				return
			case cmd := <-input:
				if cmd == nil {
					return
				}
				output <- json.RawMessage("received cmd =)")
			}
		}
	}()

	return output, nil
}

/////////////////////////////////////////////////////////////////////

type Object json.Marshaler

type ObjectSet map[uint16]Object

type Command struct {
	Command string          `json:"command"`
	Params  json.RawMessage `json:"params"`
}

type Notice struct {
}

// type ObjectContainer map[uint16]interface{}

// func (c *ObjectContainer) MarshalJSON() ([]byte, error) {
// 	return json.Marshal(struct {
// 		Id     uint16      `json:"id"`
// 		Type   string      `json:"type"`
// 		Object interface{} `json:"object"`
// 	}{})
// }

// func (*Game) API_CMD()

type GameObjectCollection map[uint16]interface{}
<commit_msg>Refactor game object<commit_after>package game

import (
	"github.com/ivan1993spb/snake-server/engine"
	"github.com/ivan1993spb/snake-server/playground"
)

type Game struct {
	world *World
}

type ErrCreateGame struct {
	Err error
}

func (e *ErrCreateGame) Error() string {
	return "cannot create game: " + e.Err.Error()
}

func NewGame(width, height uint8) (*Game, error) {
	area, err := engine.NewArea(width, height)
	if err != nil {
		return nil, &ErrCreateGame{Err: err}
	}

	scene, err := engine.NewScene(area)
	if err != nil {
		return nil, &ErrCreateGame{Err: err}
	}

	pg := playground.NewPlayground(scene)
	world := NewWorld(pg)

	return &Game{
		world: world,
	}, nil
}
<|endoftext|>


<commit_before>package hotmaze

import (
	"net/http"
	"time"

	"cloud.google.com/go/storage"
)

// Server encapsulates the Hot Maze backend.
type Server struct {
	GCPProjectID string

	BackendBaseURL string

	StorageClient *storage.Client

	// Service account email e.g. "ephemeral-storage@hot-maze.iam.gserviceaccount.com"
	StorageAccountID string

	// Secret service account private key (PEM).
	// Don't check it in, prefer using Secret Manager.
	StoragePrivateKey []byte

	// StorageBucket e.g. "hot-maze.appspot.com"
	StorageBucket string

	StorageFileTTL time.Duration

	CloudTasksQueuePath string
}

// RegisterHandlers registers the handlers
func (s Server) RegisterHandlers() {
	// Static assets: HTML, JS, CSS
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, "static/index.html")
	})
	http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("static"))))

	// Backend logic
	http.HandleFunc("/secure-urls", s.HandlerGenerateSignedURLs)
	http.HandleFunc("/get/", s.HandlerUnshortenGetURL)
	http.HandleFunc("/forget", s.HandlerForgetFile)
}
<commit_msg>B1 Terms.<commit_after>package hotmaze

import (
	"net/http"
	"time"

	"cloud.google.com/go/storage"
)

// Server encapsulates the Hot Maze backend.
type Server struct {
	GCPProjectID string

	BackendBaseURL string

	StorageClient *storage.Client

	// Service account email e.g. "ephemeral-storage@hot-maze.iam.gserviceaccount.com"
	StorageAccountID string

	// Secret service account private key (PEM).
	// Don't check it in, prefer using Secret Manager.
	StoragePrivateKey []byte

	// StorageBucket e.g. "hot-maze.appspot.com"
	StorageBucket string

	StorageFileTTL time.Duration

	CloudTasksQueuePath string
}

// RegisterHandlers registers the handlers
func (s Server) RegisterHandlers() {
	// Static assets: HTML, JS, CSS
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, "static/index.html")
	})
	http.HandleFunc("/terms.html", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, "static/terms.html")
	})
	http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("static"))))

	// Backend logic
	http.HandleFunc("/secure-urls", s.HandlerGenerateSignedURLs)
	http.HandleFunc("/get/", s.HandlerUnshortenGetURL)
	http.HandleFunc("/forget", s.HandlerForgetFile)
}
<|endoftext|>


<commit_before>package gate

import (
	"io"

	"github.com/elpinal/coco3/config"
	"github.com/elpinal/coco3/editor"
	"github.com/elpinal/coco3/screen/terminal"
)

type Gate interface {
	Read() ([]rune, error)
	Clear()
}

type gate struct {
	e editor.Editor

	history [][]rune
}

func (g *gate) Read() ([]rune, error) {
	g.e.SetHistory(g.history)
	b, err := g.e.Read()
	if err != nil {
		return nil, err
	}
	if len(g.history) == 0 || string(g.history[len(g.history)-1]) != string(b) {
		g.history = append(g.history, b)
	}
	return b, nil
}

func (g *gate) Clear() {
	g.e.Clear()
}

func New(conf *config.Config, in io.Reader, out, err io.Writer) Gate {
	return &gate{
		e: editor.New(terminal.New(out), conf, in, out, err),
	}
}
<commit_msg>Don't record blank command in history<commit_after>package gate

import (
	"io"

	"github.com/elpinal/coco3/config"
	"github.com/elpinal/coco3/editor"
	"github.com/elpinal/coco3/screen/terminal"
)

type Gate interface {
	Read() ([]rune, error)
	Clear()
}

type gate struct {
	e editor.Editor

	history [][]rune
}

func (g *gate) Read() ([]rune, error) {
	g.e.SetHistory(g.history)
	b, err := g.e.Read()
	if err != nil {
		return nil, err
	}
	if len(b) != 0 && (len(g.history) == 0 || string(g.history[len(g.history)-1]) != string(b)) {
		g.history = append(g.history, b)
	}
	return b, nil
}

func (g *gate) Clear() {
	g.e.Clear()
}

func New(conf *config.Config, in io.Reader, out, err io.Writer) Gate {
	return &gate{
		e: editor.New(terminal.New(out), conf, in, out, err),
	}
}
<|endoftext|>