<commit_before>package etcd

import (
	"context"
	"fmt"
	"sync"

	"github.com/coreos/etcd/clientv3"
	"github.com/coreos/etcd/clientv3/concurrency"
	"github.com/gogo/protobuf/proto"
	"github.com/pkg/errors"

	"github.com/taku-k/polymerase/pkg/polypb"
)

type ClientAPI interface {
	GetBackupMeta(key polypb.BackupMetaKey) (polypb.BackupMetaSlice, error)
	PutBackupMeta(key polypb.BackupMetaKey, meta *polypb.BackupMeta) error
	RemoveBackupMeta(key polypb.BackupMetaKey) error
	UpdateLSN(key polypb.BackupMetaKey, lsn string) error

	GetNodeMeta(key polypb.NodeMetaKey) ([]*polypb.NodeMeta, error)
	PutNodeMeta(key polypb.NodeMetaKey, meta *polypb.NodeMeta) error
	RemoveNodeMeta(key polypb.NodeMetaKey) error
	//Get(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error)
	//Put(ctx context.Context, key, val string, opts ...clientv3.OpOption) (*clientv3.PutResponse, error)
	//Delete(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.DeleteResponse, error)
	//Locker(key string) sync.Locker
	Close()
}

type Client struct {
	cli     *clientv3.Client
	session *concurrency.Session
}

func NewClient(cfg clientv3.Config) (ClientAPI, error) {
	cli, err := clientv3.New(cfg)
	if err != nil {
		return nil, err
	}
	session, err := concurrency.NewSession(cli)
	return &Client{
		cli:     cli,
		session: session,
	}, err
}

func NewTestClient(cli *clientv3.Client) (ClientAPI, error) {
	session, err := concurrency.NewSession(cli)
	return &Client{
		cli:     cli,
		session: session,
	}, err
}

func (c *Client) GetBackupMeta(key polypb.BackupMetaKey) (polypb.BackupMetaSlice, error) {
	res, err := c.cli.KV.Get(context.TODO(), string(key),
		clientv3.WithPrefix(), clientv3.WithIgnoreLease())
	if err != nil {
		return nil, err
	}
	result := make(polypb.BackupMetaSlice, len(res.Kvs))
	for i, kv := range res.Kvs {
		meta := &polypb.BackupMeta{}
		if err := proto.Unmarshal(kv.Value, meta); err != nil {
			return nil, err
		}
		result[i] = meta
	}
	return result, nil
}

func (c *Client) PutBackupMeta(key polypb.BackupMetaKey, meta *polypb.BackupMeta) error {
	data, err := meta.Marshal()
	if err != nil {
		return err
	}
	_, err = c.cli.KV.Put(context.TODO(), string(key), string(data))
	return err
}

func (c *Client) RemoveBackupMeta(key polypb.BackupMetaKey) error {
	_, err := c.cli.KV.Delete(context.TODO(), string(key), clientv3.WithPrefix())
	return err
}

func (c *Client) UpdateLSN(key polypb.BackupMetaKey, lsn string) error {
	locker := c.Locker("lock-" + string(key))
	locker.Lock()
	defer locker.Unlock()

	metas, err := c.GetBackupMeta(key)
	if err != nil {
		return err
	}
	if len(metas) != 1 {
		return errors.New(fmt.Sprintf("fetched wrong metadata: %q", metas))
	}
	m := metas[0]
	if details := m.GetXtrabackup(); details != nil {
		details.ToLsn = lsn
	}
	return c.PutBackupMeta(key, m)
}

func (c *Client) GetNodeMeta(key polypb.NodeMetaKey) ([]*polypb.NodeMeta, error) {
	res, err := c.cli.KV.Get(context.TODO(), string(key), clientv3.WithPrefix())
	if err != nil {
		return nil, err
	}
	result := make([]*polypb.NodeMeta, len(res.Kvs))
	for i, kv := range res.Kvs {
		meta := &polypb.NodeMeta{}
		if err := proto.Unmarshal(kv.Value, meta); err != nil {
			return nil, err
		}
		result[i] = meta
	}
	return result, nil
}

func (c *Client) PutNodeMeta(key polypb.NodeMetaKey, meta *polypb.NodeMeta) error {
	data, err := meta.Marshal()
	if err != nil {
		return err
	}
	_, err = c.cli.KV.Put(context.TODO(), string(key), string(data))
	return err
}

func (c *Client) RemoveNodeMeta(key polypb.NodeMetaKey) error {
	_, err := c.cli.KV.Delete(context.TODO(), string(key), clientv3.WithPrefix())
	return err
}

func (c *Client) Get(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) {
	return c.cli.KV.Get(ctx, key, opts...)
}

func (c *Client) Put(ctx context.Context, key, val string, opts ...clientv3.OpOption) (*clientv3.PutResponse, error) {
	return c.cli.KV.Put(ctx, key, val, opts...)
}

func (c *Client) Delete(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.DeleteResponse, error) {
	return c.cli.KV.Delete(ctx, key, opts...)
}

func (c *Client) Locker(key string) sync.Locker {
	return concurrency.NewLocker(c.session, key)
}

func (c *Client) Close() {
	c.cli.Close()
}
<commit_msg>fix bug<commit_after>package etcd

import (
	"context"
	"fmt"
	"sync"

	"github.com/coreos/etcd/clientv3"
	"github.com/coreos/etcd/clientv3/concurrency"
	"github.com/gogo/protobuf/proto"
	"github.com/pkg/errors"

	"github.com/taku-k/polymerase/pkg/polypb"
)

type ClientAPI interface {
	GetBackupMeta(key polypb.BackupMetaKey) (polypb.BackupMetaSlice, error)
	PutBackupMeta(key polypb.BackupMetaKey, meta *polypb.BackupMeta) error
	RemoveBackupMeta(key polypb.BackupMetaKey) error
	UpdateLSN(key polypb.BackupMetaKey, lsn string) error

	GetNodeMeta(key polypb.NodeMetaKey) ([]*polypb.NodeMeta, error)
	PutNodeMeta(key polypb.NodeMetaKey, meta *polypb.NodeMeta) error
	RemoveNodeMeta(key polypb.NodeMetaKey) error
	//Get(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error)
	//Put(ctx context.Context, key, val string, opts ...clientv3.OpOption) (*clientv3.PutResponse, error)
	//Delete(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.DeleteResponse, error)
	//Locker(key string) sync.Locker
	Close()
}

type Client struct {
	cli     *clientv3.Client
	session *concurrency.Session
}

func NewClient(cfg clientv3.Config) (ClientAPI, error) {
	cli, err := clientv3.New(cfg)
	if err != nil {
		return nil, err
	}
	session, err := concurrency.NewSession(cli)
	return &Client{
		cli:     cli,
		session: session,
	}, err
}

func NewTestClient(cli *clientv3.Client) (ClientAPI, error) {
	session, err := concurrency.NewSession(cli)
	return &Client{
		cli:     cli,
		session: session,
	}, err
}

func (c *Client) GetBackupMeta(key polypb.BackupMetaKey) (polypb.BackupMetaSlice, error) {
	res, err := c.cli.KV.Get(context.TODO(), string(key),
		clientv3.WithPrefix(), clientv3.WithIgnoreLease())
	if err != nil {
		return nil, err
	}
	result := make(polypb.BackupMetaSlice, len(res.Kvs))
	for i, kv := range res.Kvs {
		meta := &polypb.BackupMeta{}
		if err := proto.Unmarshal(kv.Value, meta); err != nil {
			return nil, err
		}
		result[i] = meta
	}
	return result, nil
}

func (c *Client) PutBackupMeta(key polypb.BackupMetaKey, meta *polypb.BackupMeta) error {
	data, err := meta.Marshal()
	if err != nil {
		return err
	}
	_, err = c.cli.KV.Put(context.TODO(), string(key), string(data))
	return err
}

func (c *Client) RemoveBackupMeta(key polypb.BackupMetaKey) error {
	_, err := c.cli.KV.Delete(context.TODO(), string(key), clientv3.WithPrefix())
	return err
}

func (c *Client) UpdateLSN(key polypb.BackupMetaKey, lsn string) error {
	locker := c.Locker("lock-" + string(key))
	locker.Lock()
	defer locker.Unlock()

	metas, err := c.GetBackupMeta(key)
	if err != nil {
		return err
	}
	if len(metas) != 1 {
		return errors.New(fmt.Sprintf("fetched wrong metadata: %q", metas))
	}
	m := metas[0]
	if m.Details == nil {
		m.Details = &polypb.BackupMeta_Xtrabackup{
			Xtrabackup: &polypb.XtrabackupMeta{
				ToLsn: lsn,
			},
		}
	} else if details := m.GetXtrabackup(); details != nil {
		details.ToLsn = lsn
	}
	return c.PutBackupMeta(key, m)
}

func (c *Client) GetNodeMeta(key polypb.NodeMetaKey) ([]*polypb.NodeMeta, error) {
	res, err := c.cli.KV.Get(context.TODO(), string(key), clientv3.WithPrefix())
	if err != nil {
		return nil, err
	}
	result := make([]*polypb.NodeMeta, len(res.Kvs))
	for i, kv := range res.Kvs {
		meta := &polypb.NodeMeta{}
		if err := proto.Unmarshal(kv.Value, meta); err != nil {
			return nil, err
		}
		result[i] = meta
	}
	return result, nil
}

func (c *Client) PutNodeMeta(key polypb.NodeMetaKey, meta *polypb.NodeMeta) error {
	data, err := meta.Marshal()
	if err != nil {
		return err
	}
	_, err = c.cli.KV.Put(context.TODO(), string(key), string(data))
	return err
}

func (c *Client) RemoveNodeMeta(key polypb.NodeMetaKey) error {
	_, err := c.cli.KV.Delete(context.TODO(), string(key), clientv3.WithPrefix())
	return err
}

func (c *Client) Get(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) {
	return c.cli.KV.Get(ctx, key, opts...)
}

func (c *Client) Put(ctx context.Context, key, val string, opts ...clientv3.OpOption) (*clientv3.PutResponse, error) {
	return c.cli.KV.Put(ctx, key, val, opts...)
}

func (c *Client) Delete(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.DeleteResponse, error) {
	return c.cli.KV.Delete(ctx, key, opts...)
}

func (c *Client) Locker(key string) sync.Locker {
	return concurrency.NewLocker(c.session, key)
}

func (c *Client) Close() {
	c.cli.Close()
}
<|endoftext|>


<commit_before>package irc

import (
	"errors"
	"strings"
)

type Mask struct {
	Nick     string
	Username string
	Hostname string
	Mask     string
}
type Message struct {
	Raw     string
	Tags    map[string]string
	Prefix  *Mask
	Command string
	Params  []string
}

func NewMessage() *Message {
	return &Message{
		Tags:   make(map[string]string),
		Prefix: &Mask{},
	}
}

// GetParam - Get a param value, returning a default value if it doesn't exist
func (m *Message) GetParam(idx int, def string) string {
	if idx < 0 || idx > len(m.Params)-1 {
		return def
	}

	return m.Params[idx]
}

// ToLine - Convert the Message struct to its raw IRC line
func (m *Message) ToLine() string {
	line := ""

	if len(m.Tags) > 0 {
		line += "@"

		for tagName, tagVal := range m.Tags {
			line += tagName
			if tagVal != "" {
				line += "=" + tagVal
			}
			line += ";"
		}
	}

	if m.Prefix != nil && (m.Prefix.Nick != "" || m.Prefix.Username != "" || m.Prefix.Hostname != "") {
		prefix := ""

		if m.Prefix.Nick != "" {
			prefix += m.Prefix.Nick
		}

		if m.Prefix.Username != "" && m.Prefix.Nick != "" {
			prefix += "!" + m.Prefix.Username
		} else if m.Prefix.Username != "" {
			prefix += m.Prefix.Username
		}

		if m.Prefix.Hostname != "" && prefix != "" {
			prefix += "@" + m.Prefix.Username
		} else if m.Prefix.Hostname != "" {
			prefix += m.Prefix.Hostname
		}

		if line != "" {
			line += " :" + prefix
		} else {
			line += ":" + prefix
		}
	}

	if line != "" {
		line += " " + m.Command
	} else {
		line += m.Command
	}

	paramLen := len(m.Params)
	for idx, param := range m.Params {
		if idx == paramLen-1 && (strings.Contains(param, " ") || strings.HasPrefix(param, ":")) {
			line += " :" + param
		} else {
			line += " " + param
		}
	}

	return line
}

func createMask(maskStr string) *Mask {
	mask := &Mask{
		Mask: maskStr,
	}

	usernameStart := strings.Index(maskStr, "!")
	hostStart := strings.Index(maskStr, "@")

	if usernameStart == -1 && hostStart == -1 {
		mask.Nick = maskStr
	} else if usernameStart > -1 && hostStart > -1 {
		mask.Nick = maskStr[0:usernameStart]
		mask.Username = maskStr[usernameStart+1 : hostStart]
		mask.Hostname = maskStr[hostStart+1:]
	} else if usernameStart > -1 && hostStart == -1 {
		mask.Nick = maskStr[0:usernameStart]
		mask.Username = maskStr[usernameStart+1:]
	} else if usernameStart == -1 && hostStart > -1 {
		mask.Username = maskStr[0:hostStart]
		mask.Hostname = maskStr[hostStart+1:]
	}

	return mask
}

// ParseLine - Turn a raw IRC line into a message
func ParseLine(input string) (*Message, error) {
	line := strings.Trim(input, "\r\n")

	message := NewMessage()
	message.Raw = line

	token := ""
	rest := ""

	token, rest = nextToken(line, false)
	if token == "" {
		return message, errors.New("Empty line")
	}

	// Tags. Starts with "@"
	if token[0] == 64 {
		tagsRaw := token[1:]
		tags := strings.Split(tagsRaw, ";")
		for _, tag := range tags {
			parts := strings.Split(tag, "=")
			if len(parts) == 1 {
				message.Tags[parts[0]] = ""
			} else {
				message.Tags[parts[0]] = parts[1]
			}
		}

		token, rest = nextToken(rest, false)
	}

	// Prefix. Starts with ":"
	if token != "" && token[0] == 58 {
		message.Prefix = createMask(token[1:])
		token, rest = nextToken(rest, false)
	} else {
		message.Prefix = createMask("")
	}

	// Command
	if token == "" {
		return message, errors.New("Missing command")
	}

	message.Command = token

	// Params
	for {
		token, rest = nextToken(rest, true)
		if token == "" {
			break
		}

		message.Params = append(message.Params, token)
	}

	return message, nil
}

func nextToken(s string, allowTrailing bool) (string, string) {
	s = strings.TrimLeft(s, " ")

	if len(s) == 0 {
		return "", ""
	}

	// The last token (trailing) start with :
	if allowTrailing && s[0] == 58 {
		return s[1:], ""
	}

	token := ""
	spaceIdx := strings.Index(s, " ")
	if spaceIdx > -1 {
		token = s[:spaceIdx]
		s = s[spaceIdx+1:]
	} else {
		token = s
		s = ""
	}

	return token, s
}
<commit_msg>Skip IRC tags on messages without a name<commit_after>package irc

import (
	"errors"
	"strings"
)

type Mask struct {
	Nick     string
	Username string
	Hostname string
	Mask     string
}
type Message struct {
	Raw     string
	Tags    map[string]string
	Prefix  *Mask
	Command string
	Params  []string
}

func NewMessage() *Message {
	return &Message{
		Tags:   make(map[string]string),
		Prefix: &Mask{},
	}
}

// GetParam - Get a param value, returning a default value if it doesn't exist
func (m *Message) GetParam(idx int, def string) string {
	if idx < 0 || idx > len(m.Params)-1 {
		return def
	}

	return m.Params[idx]
}

// ToLine - Convert the Message struct to its raw IRC line
func (m *Message) ToLine() string {
	line := ""

	if len(m.Tags) > 0 {
		line += "@"

		for tagName, tagVal := range m.Tags {
			line += tagName
			if tagVal != "" {
				line += "=" + tagVal
			}
			line += ";"
		}
	}

	if m.Prefix != nil && (m.Prefix.Nick != "" || m.Prefix.Username != "" || m.Prefix.Hostname != "") {
		prefix := ""

		if m.Prefix.Nick != "" {
			prefix += m.Prefix.Nick
		}

		if m.Prefix.Username != "" && m.Prefix.Nick != "" {
			prefix += "!" + m.Prefix.Username
		} else if m.Prefix.Username != "" {
			prefix += m.Prefix.Username
		}

		if m.Prefix.Hostname != "" && prefix != "" {
			prefix += "@" + m.Prefix.Username
		} else if m.Prefix.Hostname != "" {
			prefix += m.Prefix.Hostname
		}

		if line != "" {
			line += " :" + prefix
		} else {
			line += ":" + prefix
		}
	}

	if line != "" {
		line += " " + m.Command
	} else {
		line += m.Command
	}

	paramLen := len(m.Params)
	for idx, param := range m.Params {
		if idx == paramLen-1 && (strings.Contains(param, " ") || strings.HasPrefix(param, ":")) {
			line += " :" + param
		} else {
			line += " " + param
		}
	}

	return line
}

func createMask(maskStr string) *Mask {
	mask := &Mask{
		Mask: maskStr,
	}

	usernameStart := strings.Index(maskStr, "!")
	hostStart := strings.Index(maskStr, "@")

	if usernameStart == -1 && hostStart == -1 {
		mask.Nick = maskStr
	} else if usernameStart > -1 && hostStart > -1 {
		mask.Nick = maskStr[0:usernameStart]
		mask.Username = maskStr[usernameStart+1 : hostStart]
		mask.Hostname = maskStr[hostStart+1:]
	} else if usernameStart > -1 && hostStart == -1 {
		mask.Nick = maskStr[0:usernameStart]
		mask.Username = maskStr[usernameStart+1:]
	} else if usernameStart == -1 && hostStart > -1 {
		mask.Username = maskStr[0:hostStart]
		mask.Hostname = maskStr[hostStart+1:]
	}

	return mask
}

// ParseLine - Turn a raw IRC line into a message
func ParseLine(input string) (*Message, error) {
	line := strings.Trim(input, "\r\n")

	message := NewMessage()
	message.Raw = line

	token := ""
	rest := ""

	token, rest = nextToken(line, false)
	if token == "" {
		return message, errors.New("Empty line")
	}

	// Tags. Starts with "@"
	if token[0] == 64 {
		tagsRaw := token[1:]
		tags := strings.Split(tagsRaw, ";")
		for _, tag := range tags {
			parts := strings.Split(tag, "=")
			if len(parts) > 0 && parts[0] == "" {
				continue
			}

			if len(parts) == 1 {
				message.Tags[parts[0]] = ""
			} else {
				message.Tags[parts[0]] = parts[1]
			}
		}

		token, rest = nextToken(rest, false)
	}

	// Prefix. Starts with ":"
	if token != "" && token[0] == 58 {
		message.Prefix = createMask(token[1:])
		token, rest = nextToken(rest, false)
	} else {
		message.Prefix = createMask("")
	}

	// Command
	if token == "" {
		return message, errors.New("Missing command")
	}

	message.Command = token

	// Params
	for {
		token, rest = nextToken(rest, true)
		if token == "" {
			break
		}

		message.Params = append(message.Params, token)
	}

	return message, nil
}

func nextToken(s string, allowTrailing bool) (string, string) {
	s = strings.TrimLeft(s, " ")

	if len(s) == 0 {
		return "", ""
	}

	// The last token (trailing) start with :
	if allowTrailing && s[0] == 58 {
		return s[1:], ""
	}

	token := ""
	spaceIdx := strings.Index(s, " ")
	if spaceIdx > -1 {
		token = s[:spaceIdx]
		s = s[spaceIdx+1:]
	} else {
		token = s
		s = ""
	}

	return token, s
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"golang.org/x/net/html"
	"net/http"
	"net/url"
	"os"
	"strings"
)

// Helper function to pull the href attribute from a Token
func getHref(t html.Token) (ok bool, href string) {
	// Iterate over all of the Token's attributes until we find an "href"
	for _, a := range t.Attr {
		if a.Key == "href" {
			href = a.Val
			ok = true
		}
	}

	// "bare" return will return the variables (ok, href) as defined in
	// the function definition
	return
}

type UrlResponse struct {
	from string
	url  string
	code int
	err  error
}

type NewUrl struct {
	from string
	url  string
}

func crawl(chWork chan NewUrl, ch chan NewUrl, chFinished chan UrlResponse) {
	for true {
		new := <-chWork
		crawlOne(new, ch, chFinished)
	}
}

// Extract all http** links from a given webpage
func crawlOne(req NewUrl, ch chan NewUrl, chFinished chan UrlResponse) {
	base, err := url.Parse(req.url)
	reply := UrlResponse{
		url:  req.url,
		from: req.from,
		code: 999,
	}
	if err != nil {
		fmt.Println("ERROR: failed to Parse \"" + req.url + "\"")
		reply.err = err
		chFinished <- reply
		return
	}
	switch base.Scheme {
	case "mailto", "irc":
		reply.err = fmt.Errorf("%s on page %s", base.Scheme, req.from)
		reply.code = 900
		chFinished <- reply
		return
	}
	resp, err := http.Get(req.url)
	if err != nil {
		fmt.Println("ERROR: Failed to crawl \"" + req.url + "\"  " + err.Error())
		reply.err = err
		chFinished <- reply
		return
	}
	defer func() {
		// Notify that we're done after this function
		reply.code = resp.StatusCode
		chFinished <- reply
	}()

	loc, err := resp.Location()
	if err == nil && req.url != loc.String() {
		fmt.Printf("\t crawled \"%s\"", req.url)
		fmt.Printf("\t\t to \"%s\"", loc)
	}

	b := resp.Body
	defer b.Close() // close Body when the function returns

	// only parse if this page is on the original site
	// if we moved this check back to the main loop, we'd parse more sites
	if !strings.HasPrefix(req.url, seedUrl) {
		return
	}

	z := html.NewTokenizer(b)

	for {
		tt := z.Next()

		switch {
		case tt == html.ErrorToken:
			// End of the document, we're done
			return
		case tt == html.StartTagToken:
			t := z.Token()

			// Check if the token is an <a> tag
			isAnchor := t.Data == "a"
			if !isAnchor {
				continue
			}

			// Extract the href value, if there is one
			ok, newUrl := getHref(t)
			if !ok {
				continue
			}
			u, e := url.Parse(newUrl)
			if e != nil {
				fmt.Println("ERROR: failed to Parse \"" + newUrl + "\"")
				continue
			}
			new := NewUrl{
				from: req.url,
				url:  base.ResolveReference(u).String(),
			}
			ch <- new
		}
	}
}

var seedUrl = os.Args[1]

type FoundUrls struct {
	response   int
	usageCount int
	err        error
	from       map[string]int
}

func main() {
	seedUrls := os.Args[1:]

	// Channels
	chUrls := make(chan NewUrl, 1000)
	chWork := make(chan NewUrl, 1000)
	chFinished := make(chan UrlResponse)

	var foundUrls = make(map[string]FoundUrls)

	for w := 1; w <= 10; w++ {
		go crawl(chWork, chUrls, chFinished)
	}

	for _, url := range seedUrls {
		new := NewUrl{
			from: "",
			url:  url,
		}
		chUrls <- new
	}

	// Subscribe to both channels
	count := 0
	for len(chUrls) > 0 || count > 0 {
		select {
		case foundUrl := <-chUrls:
			// don't need to check err - its already been checked before its put in the chUrls que
			u, _ := url.Parse(foundUrl.url)
			// TODO: need a different pipeline for ensuring anchor fragments exist
			// TODO: consider only removing the query/fragment for docs urls
			u.RawQuery = ""
			u.Fragment = ""
			resourceUrl := u.String()

			f, ok := foundUrls[resourceUrl]
			if !ok {
				count++
				f.usageCount = 0
				f.response = 0
				f.from = make(map[string]int)
				f.from[foundUrl.from] = 1
				chWork <- NewUrl{
					from: foundUrl.from,
					url:  resourceUrl,
				}
			}
			f.usageCount++
			f.from[foundUrl.from]++
			foundUrls[resourceUrl] = f

		case ret := <-chFinished:
			count--
			info := foundUrls[ret.url]
			//info.from[ret.from]++
			info.response = ret.code
			info.err = ret.err
			foundUrls[ret.url] = info
		}
		// fmt.Printf("(w%d, u%d, c%d)", len(chWork), len(chUrls), count)
	}

	// We're done! Print the results...
	fmt.Println("\nDone.")
	summary := make(map[int]int)
	for url, info := range foundUrls {
		summary[info.response]++
		if info.response != 200 && info.response != 900 {
			fmt.Printf(" - %d (%d): %s\n", info.response, info.usageCount, url)
			if info.err != nil {
				fmt.Printf("\t%s\n", info.err)
			}
			for from, count := range info.from {
				fmt.Printf("\t\t%d times from %s\n", count, from)
			}
		}
	}
	fmt.Println("\nFound", len(foundUrls), "unique urls\n")
	for code, count := range summary {
		fmt.Printf("\t\tStatus %d : %d\n", code, count)
	}

	close(chUrls)

	// return the number of 404's to show that there are things to be fixed
	os.Exit(summary[404])
}
<commit_msg>Give the user some idea what to do<commit_after>package main

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"strings"

	"golang.org/x/net/html"
)

// Helper function to pull the href attribute from a Token
func getHref(t html.Token) (ok bool, href string) {
	// Iterate over all of the Token's attributes until we find an "href"
	for _, a := range t.Attr {
		if a.Key == "href" {
			href = a.Val
			ok = true
		}
	}

	// "bare" return will return the variables (ok, href) as defined in
	// the function definition
	return
}

type UrlResponse struct {
	from string
	url  string
	code int
	err  error
}

type NewUrl struct {
	from string
	url  string
}

func crawl(chWork chan NewUrl, ch chan NewUrl, chFinished chan UrlResponse) {
	for true {
		new := <-chWork
		crawlOne(new, ch, chFinished)
	}
}

// Extract all http** links from a given webpage
func crawlOne(req NewUrl, ch chan NewUrl, chFinished chan UrlResponse) {
	base, err := url.Parse(req.url)
	reply := UrlResponse{
		url:  req.url,
		from: req.from,
		code: 999,
	}
	fmt.Printf("Crawling: %s\n", req.url)
	if err != nil {
		fmt.Println("ERROR: failed to Parse \"" + req.url + "\"")
		reply.err = err
		chFinished <- reply
		return
	}
	switch base.Scheme {
	case "mailto", "irc":
		reply.err = fmt.Errorf("%s on page %s", base.Scheme, req.from)
		reply.code = 900
		chFinished <- reply
		return
	}
	resp, err := http.Get(req.url)
	if err != nil {
		fmt.Println("ERROR: Failed to crawl \"" + req.url + "\"  " + err.Error())
		reply.err = err
		chFinished <- reply
		return
	}
	defer func() {
		// Notify that we're done after this function
		reply.code = resp.StatusCode
		chFinished <- reply
	}()

	loc, err := resp.Location()
	if err == nil && req.url != loc.String() {
		fmt.Printf("\t crawled \"%s\"", req.url)
		fmt.Printf("\t\t to \"%s\"", loc)
	}

	b := resp.Body
	defer b.Close() // close Body when the function returns

	// only parse if this page is on the original site
	// if we moved this check back to the main loop, we'd parse more sites
	if !strings.HasPrefix(req.url, seedUrl) {
		return
	}

	z := html.NewTokenizer(b)

	for {
		tt := z.Next()

		switch {
		case tt == html.ErrorToken:
			// End of the document, we're done
			return
		case tt == html.StartTagToken:
			t := z.Token()

			// Check if the token is an <a> tag
			isAnchor := t.Data == "a"
			if !isAnchor {
				continue
			}

			// Extract the href value, if there is one
			ok, newUrl := getHref(t)
			if !ok {
				continue
			}
			u, e := url.Parse(newUrl)
			if e != nil {
				fmt.Println("ERROR: failed to Parse \"" + newUrl + "\"")
				continue
			}
			new := NewUrl{
				from: req.url,
				url:  base.ResolveReference(u).String(),
			}
			ch <- new
		}
	}
}

var seedUrl string

type FoundUrls struct {
	response   int
	usageCount int
	err        error
	from       map[string]int
}

func main() {
	if len(os.Args[1:]) == 0 {
		fmt.Println("Please specify a URL to check")
		os.Exit(-1)
	}
	seedUrl := os.Args[1]

	// Channels
	chUrls := make(chan NewUrl, 1000)
	chWork := make(chan NewUrl, 1000)
	chFinished := make(chan UrlResponse)

	var foundUrls = make(map[string]FoundUrls)

	for w := 1; w <= 10; w++ {
		go crawl(chWork, chUrls, chFinished)
	}

	new := NewUrl{
		from: "",
		url:  seedUrl,
	}
	chUrls <- new

	// Subscribe to both channels
	count := 0
	for len(chUrls) > 0 || count > 0 {
		select {
		case foundUrl := <-chUrls:
			// don't need to check err - its already been checked before its put in the chUrls que
			u, _ := url.Parse(foundUrl.url)
			// TODO: need a different pipeline for ensuring anchor fragments exist
			// TODO: consider only removing the query/fragment for docs urls
			u.RawQuery = ""
			u.Fragment = ""
			resourceUrl := u.String()

			f, ok := foundUrls[resourceUrl]
			if !ok {
				count++
				f.usageCount = 0
				f.response = 0
				f.from = make(map[string]int)
				f.from[foundUrl.from] = 1
				chWork <- NewUrl{
					from: foundUrl.from,
					url:  resourceUrl,
				}
			}
			f.usageCount++
			f.from[foundUrl.from]++
			foundUrls[resourceUrl] = f

		case ret := <-chFinished:
			count--
			info := foundUrls[ret.url]
			//info.from[ret.from]++
			info.response = ret.code
			info.err = ret.err
			foundUrls[ret.url] = info
		}
		// fmt.Printf("(w%d, u%d, c%d)", len(chWork), len(chUrls), count)
	}

	// We're done! Print the results...
	fmt.Println("\nDone.")
	summary := make(map[int]int)
	for url, info := range foundUrls {
		summary[info.response]++
		if info.response != 200 && info.response != 900 {
			fmt.Printf(" - %d (%d): %s\n", info.response, info.usageCount, url)
			if info.err != nil {
				fmt.Printf("\t%s\n", info.err)
			}
			for from, count := range info.from {
				fmt.Printf("\t\t%d times from %s\n", count, from)
			}
		}
	}
	fmt.Println("\nFound", len(foundUrls), "unique urls\n")
	for code, count := range summary {
		fmt.Printf("\t\tStatus %d : %d\n", code, count)
	}

	close(chUrls)

	// return the number of 404's to show that there are things to be fixed
	os.Exit(summary[404])
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package routes

import (
	"net/http"
	"path"

	"github.com/emicklei/go-restful"
)

// Logs adds handlers for the /logs path serving log files from /var/log.
type Logs struct{}

// Install func registers the logs handler.
func (l Logs) Install(c *restful.Container) {
	// use restful: ws.Route(ws.GET("/logs/{logpath:*}").To(fileHandler))
	// See github.com/emicklei/go-restful/blob/master/examples/restful-serve-static.go
	ws := new(restful.WebService)
	ws.Path("/logs")
	ws.Doc("get log files")
	ws.Route(ws.GET("/{logpath:*}").To(logFileHandler).Param(ws.PathParameter("logpath", "path to the log").DataType("string")))
	ws.Route(ws.GET("/").To(logFileListHandler))

	c.Add(ws)
}

func logFileHandler(req *restful.Request, resp *restful.Response) {
	logdir := "/var/log"
	actual := path.Join(logdir, req.PathParameter("logpath"))
	http.ServeFile(resp.ResponseWriter, req.Request, actual)
}

func logFileListHandler(req *restful.Request, resp *restful.Response) {
	logdir := "/var/log"
	http.ServeFile(resp.ResponseWriter, req.Request, logdir)
}
<commit_msg>issues-98409 fix the address of restful-serve-static.go in the comment of pkg/routes/logs.go<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package routes

import (
	"net/http"
	"path"

	"github.com/emicklei/go-restful"
)

// Logs adds handlers for the /logs path serving log files from /var/log.
type Logs struct{}

// Install func registers the logs handler.
func (l Logs) Install(c *restful.Container) {
	// use restful: ws.Route(ws.GET("/logs/{logpath:*}").To(fileHandler))
	// See github.com/emicklei/go-restful/blob/master/examples/static/restful-serve-static.go
	ws := new(restful.WebService)
	ws.Path("/logs")
	ws.Doc("get log files")
	ws.Route(ws.GET("/{logpath:*}").To(logFileHandler).Param(ws.PathParameter("logpath", "path to the log").DataType("string")))
	ws.Route(ws.GET("/").To(logFileListHandler))

	c.Add(ws)
}

func logFileHandler(req *restful.Request, resp *restful.Response) {
	logdir := "/var/log"
	actual := path.Join(logdir, req.PathParameter("logpath"))
	http.ServeFile(resp.ResponseWriter, req.Request, actual)
}

func logFileListHandler(req *restful.Request, resp *restful.Response) {
	logdir := "/var/log"
	http.ServeFile(resp.ResponseWriter, req.Request, logdir)
}
<|endoftext|>


<commit_before>// Package blackhole is an outbound handler that blocks all connections.
package blackhole

import (
	"v2ray.com/core/app"
	v2net "v2ray.com/core/common/net"
	"v2ray.com/core/proxy"
	"v2ray.com/core/transport/ray"
)

// Handler is an outbound connection that sliently swallow the entire payload.
type Handler struct {
	meta     *proxy.OutboundHandlerMeta
	response ResponseConfig
}

// New creates a new blackhole handler.
func New(space app.Space, config *Config, meta *proxy.OutboundHandlerMeta) (proxy.OutboundHandler, error) {
	response, err := config.GetInternalResponse()
	if err != nil {
		return nil, err
	}
	return &Handler{
		meta:     meta,
		response: response,
	}, nil
}

// Dispatch implements OutboundHandler.Dispatch().
func (v *Handler) Dispatch(destination v2net.Destination, ray ray.OutboundRay) {
	v.response.WriteTo(ray.OutboundOutput())
	ray.OutboundOutput().Close()

	ray.OutboundInput().CloseError()
}

// Factory is an utility for creating blackhole handlers.
type Factory struct{}

// StreamCapability implements OutboundHandlerFactory.StreamCapability().
func (v *Factory) StreamCapability() v2net.NetworkList {
	return v2net.NetworkList{
		Network: []v2net.Network{v2net.Network_TCP},
	}
}

// Create implements OutboundHandlerFactory.Create().
func (v *Factory) Create(space app.Space, config interface{}, meta *proxy.OutboundHandlerMeta) (proxy.OutboundHandler, error) {
	return New(space, config.(*Config), meta)
}
<commit_msg>delay close error on blackhole<commit_after>// Package blackhole is an outbound handler that blocks all connections.
package blackhole

import (
	"time"

	"v2ray.com/core/app"
	v2net "v2ray.com/core/common/net"
	"v2ray.com/core/proxy"
	"v2ray.com/core/transport/ray"
)

// Handler is an outbound connection that sliently swallow the entire payload.
type Handler struct {
	meta     *proxy.OutboundHandlerMeta
	response ResponseConfig
}

// New creates a new blackhole handler.
func New(space app.Space, config *Config, meta *proxy.OutboundHandlerMeta) (proxy.OutboundHandler, error) {
	response, err := config.GetInternalResponse()
	if err != nil {
		return nil, err
	}
	return &Handler{
		meta:     meta,
		response: response,
	}, nil
}

// Dispatch implements OutboundHandler.Dispatch().
func (v *Handler) Dispatch(destination v2net.Destination, ray ray.OutboundRay) {
	v.response.WriteTo(ray.OutboundOutput())
	ray.OutboundOutput().Close()

	// CloseError() will immediately close the connection.
	// Sleep a little here to make sure the response is sent to client.
	time.Sleep(time.Millisecond * 500)
	ray.OutboundInput().CloseError()
}

// Factory is an utility for creating blackhole handlers.
type Factory struct{}

// StreamCapability implements OutboundHandlerFactory.StreamCapability().
func (v *Factory) StreamCapability() v2net.NetworkList {
	return v2net.NetworkList{
		Network: []v2net.Network{v2net.Network_TCP},
	}
}

// Create implements OutboundHandlerFactory.Create().
func (v *Factory) Create(space app.Space, config interface{}, meta *proxy.OutboundHandlerMeta) (proxy.OutboundHandler, error) {
	return New(space, config.(*Config), meta)
}
<|endoftext|>


<commit_before>package ssh

import (
	"os"
	"os/user"
	"strconv"
	"strings"

	"github.com/tmrts/flamingo/pkg/context"
	"github.com/tmrts/flamingo/pkg/file"
	"github.com/tmrts/flamingo/pkg/sys"
)

const (
	SSHDirPath         = ".ssh"
	AuthorizedKeysPath = ".ssh/authorized_keys"
)

// Verify uses ssh-keygen utility to verify an SSH key.
// It returns an error if a problem occurs or the key is invalid.
// The caller should diagnose the error for more information.
func Verify(key []byte) error {
	tmpFile := &context.TempFile{
		Content: string(key),
	}

	var checkSSHValidity = func(f *os.File) error {
		_, err := sys.DefaultExecutor.Execute("ssh-keygen", "-l", "-f", f.Name())
		return err
	}

	errch := context.Using(tmpFile, checkSSHValidity)

	return <-errch
}

func InitializeFor(owner *user.User) error {
	userSSHDirPath := owner.HomeDir + "/" + SSHDirPath
	userAuthorizedKeysPath := owner.HomeDir + "/" + AuthorizedKeysPath

	userID, err := strconv.Atoi(owner.Uid)
	if err != nil {
		return err
	}

	groupID, err := strconv.Atoi(owner.Gid)
	if err != nil {
		return err
	}

	err = file.EnsureDirectoryExists(userSSHDirPath, 0700, userID, groupID)
	if err != nil {
		return err
	}

	return file.EnsureExists(userAuthorizedKeysPath, 0600, userID, groupID)
}

func AuthorizeKeys(authorizedKeysFile *os.File, publicKeys ...[]byte) error {
	var keys []string
	for _, key := range publicKeys {
		keys = append(keys, string(key))
	}

	_, err := authorizedKeysFile.WriteString(strings.Join(keys, "\n") + "\n")
	return err
}
<commit_msg>Extract SSH keys to its own type from []byte<commit_after>package ssh

import (
	"os"
	"os/user"
	"strconv"
	"strings"

	"github.com/tmrts/flamingo/pkg/context"
	"github.com/tmrts/flamingo/pkg/file"
	"github.com/tmrts/flamingo/pkg/sys"
)

const (
	SSHDirPath         = ".ssh"
	AuthorizedKeysPath = ".ssh/authorized_keys"
)

type Key []byte

// Verify uses ssh-keygen utility to verify an SSH key.
// It returns an error if a problem occurs or the key is invalid.
// The caller should diagnose the error for more information.
func Verify(key Key) error {
	tmpFile := &context.TempFile{
		Content: string(key),
	}

	var checkSSHValidity = func(f *os.File) error {
		_, err := sys.DefaultExecutor.Execute("ssh-keygen", "-l", "-f", f.Name())
		return err
	}

	errch := context.Using(tmpFile, checkSSHValidity)

	return <-errch
}

func InitializeFor(owner *user.User) error {
	userSSHDirPath := owner.HomeDir + "/" + SSHDirPath
	userAuthorizedKeysPath := owner.HomeDir + "/" + AuthorizedKeysPath

	userID, err := strconv.Atoi(owner.Uid)
	if err != nil {
		return err
	}

	groupID, err := strconv.Atoi(owner.Gid)
	if err != nil {
		return err
	}

	err = file.EnsureDirectoryExists(userSSHDirPath, 0700, userID, groupID)
	if err != nil {
		return err
	}

	return file.EnsureExists(userAuthorizedKeysPath, 0600, userID, groupID)
}

func AuthorizeKeys(authorizedKeysFile *os.File, publicKeys ...Key) error {
	var keys []string
	for _, key := range publicKeys {
		keys = append(keys, string(key))
	}

	_, err := authorizedKeysFile.WriteString(strings.Join(keys, "\n") + "\n")
	return err
}
<|endoftext|>


<commit_before>// Copyright 2016 CodisLabs. All Rights Reserved.
// Licensed under the MIT (MIT-LICENSE.txt) license.

package topom

import (
	"container/list"
	"net"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/CodisLabs/codis/pkg/models"
	"github.com/CodisLabs/codis/pkg/utils"
	"github.com/CodisLabs/codis/pkg/utils/errors"
	"github.com/CodisLabs/codis/pkg/utils/log"
	"github.com/CodisLabs/codis/pkg/utils/math2"
	"github.com/CodisLabs/codis/pkg/utils/redis"
	"github.com/CodisLabs/codis/pkg/utils/rpc"
	"github.com/CodisLabs/codis/pkg/utils/sync2/atomic2"
)

type Topom struct {
	mu sync.Mutex

	xauth string
	model *models.Topom
	store *models.Store
	cache struct {
		hooks list.List
		slots []*models.SlotMapping
		group map[int]*models.Group
		proxy map[string]*models.Proxy

		sentinel *models.Sentinel
	}

	exit struct {
		C chan struct{}
	}

	config *Config
	online bool
	closed bool

	ladmin net.Listener

	action struct {
		redisp *redis.Pool

		interval atomic2.Int64
		disabled atomic2.Bool

		progress struct {
			status atomic.Value
		}
		executor atomic2.Int64
	}

	stats struct {
		redisp *redis.Pool

		servers map[string]*RedisStats
		proxies map[string]*ProxyStats
	}

	ha struct {
		redisp *redis.Pool

		monitor *redis.Sentinel
		masters map[int]string
	}
}

var ErrClosedTopom = errors.New("use of closed topom")

func New(client models.Client, config *Config) (*Topom, error) {
	if err := config.Validate(); err != nil {
		return nil, errors.Trace(err)
	}
	if err := models.ValidateProduct(config.ProductName); err != nil {
		return nil, errors.Trace(err)
	}
	s := &Topom{}
	s.config = config
	s.exit.C = make(chan struct{})
	s.action.redisp = redis.NewPool(config.ProductAuth, config.MigrationTimeout.Duration())
	s.action.progress.status.Store("")

	s.ha.redisp = redis.NewPool("", time.Second*5)

	s.model = &models.Topom{
		StartTime: time.Now().String(),
	}
	s.model.ProductName = config.ProductName
	s.model.Pid = os.Getpid()
	s.model.Pwd, _ = os.Getwd()
	if b, err := exec.Command("uname", "-a").Output(); err != nil {
		log.WarnErrorf(err, "run command uname failed")
	} else {
		s.model.Sys = strings.TrimSpace(string(b))
	}
	s.store = models.NewStore(client, config.ProductName)

	s.stats.redisp = redis.NewPool(config.ProductAuth, time.Second*5)
	s.stats.servers = make(map[string]*RedisStats)
	s.stats.proxies = make(map[string]*ProxyStats)

	if err := s.setup(config); err != nil {
		s.Close()
		return nil, err
	}

	log.Warnf("create new topom:\n%s", s.model.Encode())

	go s.serveAdmin()

	return s, nil
}

func (s *Topom) setup(config *Config) error {
	if l, err := net.Listen("tcp", config.AdminAddr); err != nil {
		return errors.Trace(err)
	} else {
		s.ladmin = l

		x, err := utils.ReplaceUnspecifiedIP("tcp", l.Addr().String(), s.config.HostAdmin)
		if err != nil {
			return err
		}
		s.model.AdminAddr = x
	}

	s.model.Token = rpc.NewToken(
		config.ProductName,
		s.ladmin.Addr().String(),
	)
	s.xauth = rpc.NewXAuth(config.ProductName)

	return nil
}

func (s *Topom) Close() error {
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.closed {
		return nil
	}
	s.closed = true
	close(s.exit.C)

	if s.ladmin != nil {
		s.ladmin.Close()
	}
	for _, p := range []*redis.Pool{
		s.action.redisp, s.stats.redisp, s.ha.redisp,
	} {
		if p != nil {
			p.Close()
		}
	}

	defer s.store.Close()

	if s.online {
		if err := s.store.Release(); err != nil {
			log.ErrorErrorf(err, "store: release lock of %s failed", s.config.ProductName)
			return errors.Errorf("store: release lock of %s failed", s.config.ProductName)
		}
	}
	return nil
}

func (s *Topom) Start(routines bool) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.closed {
		return ErrClosedTopom
	}
	if s.online {
		return nil
	} else {
		if err := s.store.Acquire(s.model); err != nil {
			log.ErrorErrorf(err, "store: acquire lock of %s failed", s.config.ProductName)
			return errors.Errorf("store: acquire lock of %s failed", s.config.ProductName)
		}
		s.online = true
	}

	if !routines {
		return nil
	}

	go func() {
		for !s.IsClosed() {
			if s.IsOnline() {
				w, _ := s.RefreshRedisStats(time.Second)
				if w != nil {
					w.Wait()
				}
			}
			time.Sleep(time.Second)
		}
	}()

	go func() {
		for !s.IsClosed() {
			if s.IsOnline() {
				w, _ := s.RefreshProxyStats(time.Second)
				if w != nil {
					w.Wait()
				}
			}
			time.Sleep(time.Second)
		}
	}()

	go func() {
		for !s.IsClosed() {
			if s.IsOnline() {
				if err := s.ProcessSlotAction(); err != nil {
					log.WarnErrorf(err, "process slot action failed")
					time.Sleep(time.Second * 5)
				}
			}
			time.Sleep(time.Second)
		}
	}()

	go func() {
		for !s.IsClosed() {
			if s.IsOnline() {
				if err := s.ProcessSyncAction(); err != nil {
					log.WarnErrorf(err, "process sync action failed")
					time.Sleep(time.Second * 5)
				}
			}
			time.Sleep(time.Second)
		}
	}()

	return nil
}

func (s *Topom) XAuth() string {
	return s.xauth
}

func (s *Topom) Model() *models.Topom {
	return s.model
}

var ErrNotOnline = errors.New("topom is not online")

func (s *Topom) newContext() (*context, error) {
	if s.closed {
		return nil, ErrClosedTopom
	}
	if s.online {
		if err := s.refillCache(); err != nil {
			return nil, err
		} else {
			ctx := &context{}
			ctx.slots = s.cache.slots
			ctx.group = s.cache.group
			ctx.proxy = s.cache.proxy
			ctx.sentinel = s.cache.sentinel
			ctx.hosts.m = make(map[string]net.IP)
			ctx.method, _ = models.ParseForwardMethod(s.config.MigrationMethod)
			return ctx, nil
		}
	} else {
		return nil, ErrNotOnline
	}
}

func (s *Topom) Stats() (*Stats, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	ctx, err := s.newContext()
	if err != nil {
		return nil, err
	}

	stats := &Stats{}
	stats.Closed = s.closed

	stats.Slots = ctx.slots

	stats.Group.Models = models.SortGroup(ctx.group)
	stats.Group.Stats = map[string]*RedisStats{}
	for _, g := range ctx.group {
		for _, x := range g.Servers {
			if v := s.stats.servers[x.Addr]; v != nil {
				stats.Group.Stats[x.Addr] = v
			}
		}
	}

	stats.Proxy.Models = models.SortProxy(ctx.proxy)
	stats.Proxy.Stats = s.stats.proxies

	stats.SlotAction.Interval = s.action.interval.Int64()
	stats.SlotAction.Disabled = s.action.disabled.Bool()
	stats.SlotAction.Progress.Status = s.action.progress.status.Load().(string)
	stats.SlotAction.Executor = s.action.executor.Int64()

	stats.HA.Model = ctx.sentinel
	stats.HA.Stats = map[string]*RedisStats{}
	for _, server := range ctx.sentinel.Servers {
		if v := s.stats.servers[server]; v != nil {
			stats.HA.Stats[server] = v
		}
	}
	stats.HA.Masters = make(map[string]string)
	if s.ha.masters != nil {
		for gid, addr := range s.ha.masters {
			stats.HA.Masters[strconv.Itoa(gid)] = addr
		}
	}
	return stats, nil
}

type Stats struct {
	Closed bool `json:"closed"`

	Slots []*models.SlotMapping `json:"slots"`

	Group struct {
		Models []*models.Group        `json:"models"`
		Stats  map[string]*RedisStats `json:"stats"`
	} `json:"group"`

	Proxy struct {
		Models []*models.Proxy        `json:"models"`
		Stats  map[string]*ProxyStats `json:"stats"`
	} `json:"proxy"`

	SlotAction struct {
		Interval int64 `json:"interval"`
		Disabled bool  `json:"disabled"`

		Progress struct {
			Status string `json:"status"`
		} `json:"progress"`

		Executor int64 `json:"executor"`
	} `json:"slot_action"`

	HA struct {
		Model   *models.Sentinel       `json:"model"`
		Stats   map[string]*RedisStats `json:"stats"`
		Masters map[string]string      `json:"masters"`
	} `json:"sentinels"`
}

func (s *Topom) Config() *Config {
	return s.config
}

func (s *Topom) IsOnline() bool {
	s.mu.Lock()
	defer s.mu.Unlock()
	return s.online && !s.closed
}

func (s *Topom) IsClosed() bool {
	s.mu.Lock()
	defer s.mu.Unlock()
	return s.closed
}

func (s *Topom) GetSlotActionInterval() int {
	return s.action.interval.AsInt()
}

func (s *Topom) SetSlotActionInterval(us int) {
	us = math2.MinMaxInt(us, 0, 1000*1000)
	s.action.interval.Set(int64(us))
	log.Warnf("set action interval = %d", us)
}

func (s *Topom) GetSlotActionDisabled() bool {
	return s.action.disabled.Bool()
}

func (s *Topom) SetSlotActionDisabled(value bool) {
	s.action.disabled.Set(value)
	log.Warnf("set action disabled = %t", value)
}

func (s *Topom) Slots() ([]*models.Slot, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	ctx, err := s.newContext()
	if err != nil {
		return nil, err
	}
	return ctx.toSlotSlice(ctx.slots, nil), nil
}

func (s *Topom) Reload() error {
	s.mu.Lock()
	defer s.mu.Unlock()
	_, err := s.newContext()
	if err != nil {
		return err
	}
	defer s.dirtyCacheAll()
	return nil
}

func (s *Topom) serveAdmin() {
	if s.IsClosed() {
		return
	}
	defer s.Close()

	log.Warnf("admin start service on %s", s.ladmin.Addr())

	eh := make(chan error, 1)
	go func(l net.Listener) {
		h := http.NewServeMux()
		h.Handle("/", newApiServer(s))
		hs := &http.Server{Handler: h}
		eh <- hs.Serve(l)
	}(s.ladmin)

	select {
	case <-s.exit.C:
		log.Warnf("admin shutdown")
	case err := <-eh:
		log.ErrorErrorf(err, "admin exit on error")
	}
}

type Overview struct {
	Version string        `json:"version"`
	Compile string        `json:"compile"`
	Config  *Config       `json:"config,omitempty"`
	Model   *models.Topom `json:"model,omitempty"`
	Stats   *Stats        `json:"stats,omitempty"`
}

func (s *Topom) Overview() (*Overview, error) {
	if stats, err := s.Stats(); err != nil {
		return nil, err
	} else {
		return &Overview{
			Version: utils.Version,
			Compile: utils.Compile,
			Config:  s.Config(),
			Model:   s.Model(),
			Stats:   stats,
		}, nil
	}
}
<commit_msg>topom: start sentinel watcher automatically, fix #1291<commit_after>// Copyright 2016 CodisLabs. All Rights Reserved.
// Licensed under the MIT (MIT-LICENSE.txt) license.

package topom

import (
	"container/list"
	"net"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/CodisLabs/codis/pkg/models"
	"github.com/CodisLabs/codis/pkg/utils"
	"github.com/CodisLabs/codis/pkg/utils/errors"
	"github.com/CodisLabs/codis/pkg/utils/log"
	"github.com/CodisLabs/codis/pkg/utils/math2"
	"github.com/CodisLabs/codis/pkg/utils/redis"
	"github.com/CodisLabs/codis/pkg/utils/rpc"
	"github.com/CodisLabs/codis/pkg/utils/sync2/atomic2"
)

type Topom struct {
	mu sync.Mutex

	xauth string
	model *models.Topom
	store *models.Store
	cache struct {
		hooks list.List
		slots []*models.SlotMapping
		group map[int]*models.Group
		proxy map[string]*models.Proxy

		sentinel *models.Sentinel
	}

	exit struct {
		C chan struct{}
	}

	config *Config
	online bool
	closed bool

	ladmin net.Listener

	action struct {
		redisp *redis.Pool

		interval atomic2.Int64
		disabled atomic2.Bool

		progress struct {
			status atomic.Value
		}
		executor atomic2.Int64
	}

	stats struct {
		redisp *redis.Pool

		servers map[string]*RedisStats
		proxies map[string]*ProxyStats
	}

	ha struct {
		redisp *redis.Pool

		monitor *redis.Sentinel
		masters map[int]string
	}
}

var ErrClosedTopom = errors.New("use of closed topom")

func New(client models.Client, config *Config) (*Topom, error) {
	if err := config.Validate(); err != nil {
		return nil, errors.Trace(err)
	}
	if err := models.ValidateProduct(config.ProductName); err != nil {
		return nil, errors.Trace(err)
	}
	s := &Topom{}
	s.config = config
	s.exit.C = make(chan struct{})
	s.action.redisp = redis.NewPool(config.ProductAuth, config.MigrationTimeout.Duration())
	s.action.progress.status.Store("")

	s.ha.redisp = redis.NewPool("", time.Second*5)

	s.model = &models.Topom{
		StartTime: time.Now().String(),
	}
	s.model.ProductName = config.ProductName
	s.model.Pid = os.Getpid()
	s.model.Pwd, _ = os.Getwd()
	if b, err := exec.Command("uname", "-a").Output(); err != nil {
		log.WarnErrorf(err, "run command uname failed")
	} else {
		s.model.Sys = strings.TrimSpace(string(b))
	}
	s.store = models.NewStore(client, config.ProductName)

	s.stats.redisp = redis.NewPool(config.ProductAuth, time.Second*5)
	s.stats.servers = make(map[string]*RedisStats)
	s.stats.proxies = make(map[string]*ProxyStats)

	if err := s.setup(config); err != nil {
		s.Close()
		return nil, err
	}

	log.Warnf("create new topom:\n%s", s.model.Encode())

	go s.serveAdmin()

	return s, nil
}

func (s *Topom) setup(config *Config) error {
	if l, err := net.Listen("tcp", config.AdminAddr); err != nil {
		return errors.Trace(err)
	} else {
		s.ladmin = l

		x, err := utils.ReplaceUnspecifiedIP("tcp", l.Addr().String(), s.config.HostAdmin)
		if err != nil {
			return err
		}
		s.model.AdminAddr = x
	}

	s.model.Token = rpc.NewToken(
		config.ProductName,
		s.ladmin.Addr().String(),
	)
	s.xauth = rpc.NewXAuth(config.ProductName)

	return nil
}

func (s *Topom) Close() error {
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.closed {
		return nil
	}
	s.closed = true
	close(s.exit.C)

	if s.ladmin != nil {
		s.ladmin.Close()
	}
	for _, p := range []*redis.Pool{
		s.action.redisp, s.stats.redisp, s.ha.redisp,
	} {
		if p != nil {
			p.Close()
		}
	}

	defer s.store.Close()

	if s.online {
		if err := s.store.Release(); err != nil {
			log.ErrorErrorf(err, "store: release lock of %s failed", s.config.ProductName)
			return errors.Errorf("store: release lock of %s failed", s.config.ProductName)
		}
	}
	return nil
}

func (s *Topom) Start(routines bool) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.closed {
		return ErrClosedTopom
	}
	if s.online {
		return nil
	} else {
		if err := s.store.Acquire(s.model); err != nil {
			log.ErrorErrorf(err, "store: acquire lock of %s failed", s.config.ProductName)
			return errors.Errorf("store: acquire lock of %s failed", s.config.ProductName)
		}
		s.online = true
	}

	if !routines {
		return nil
	}
	ctx, err := s.newContext()
	if err != nil {
		return err
	}
	s.rewatchSentinels(ctx.sentinel.Servers)

	go func() {
		for !s.IsClosed() {
			if s.IsOnline() {
				w, _ := s.RefreshRedisStats(time.Second)
				if w != nil {
					w.Wait()
				}
			}
			time.Sleep(time.Second)
		}
	}()

	go func() {
		for !s.IsClosed() {
			if s.IsOnline() {
				w, _ := s.RefreshProxyStats(time.Second)
				if w != nil {
					w.Wait()
				}
			}
			time.Sleep(time.Second)
		}
	}()

	go func() {
		for !s.IsClosed() {
			if s.IsOnline() {
				if err := s.ProcessSlotAction(); err != nil {
					log.WarnErrorf(err, "process slot action failed")
					time.Sleep(time.Second * 5)
				}
			}
			time.Sleep(time.Second)
		}
	}()

	go func() {
		for !s.IsClosed() {
			if s.IsOnline() {
				if err := s.ProcessSyncAction(); err != nil {
					log.WarnErrorf(err, "process sync action failed")
					time.Sleep(time.Second * 5)
				}
			}
			time.Sleep(time.Second)
		}
	}()

	return nil
}

func (s *Topom) XAuth() string {
	return s.xauth
}

func (s *Topom) Model() *models.Topom {
	return s.model
}

var ErrNotOnline = errors.New("topom is not online")

func (s *Topom) newContext() (*context, error) {
	if s.closed {
		return nil, ErrClosedTopom
	}
	if s.online {
		if err := s.refillCache(); err != nil {
			return nil, err
		} else {
			ctx := &context{}
			ctx.slots = s.cache.slots
			ctx.group = s.cache.group
			ctx.proxy = s.cache.proxy
			ctx.sentinel = s.cache.sentinel
			ctx.hosts.m = make(map[string]net.IP)
			ctx.method, _ = models.ParseForwardMethod(s.config.MigrationMethod)
			return ctx, nil
		}
	} else {
		return nil, ErrNotOnline
	}
}

func (s *Topom) Stats() (*Stats, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	ctx, err := s.newContext()
	if err != nil {
		return nil, err
	}

	stats := &Stats{}
	stats.Closed = s.closed

	stats.Slots = ctx.slots

	stats.Group.Models = models.SortGroup(ctx.group)
	stats.Group.Stats = map[string]*RedisStats{}
	for _, g := range ctx.group {
		for _, x := range g.Servers {
			if v := s.stats.servers[x.Addr]; v != nil {
				stats.Group.Stats[x.Addr] = v
			}
		}
	}

	stats.Proxy.Models = models.SortProxy(ctx.proxy)
	stats.Proxy.Stats = s.stats.proxies

	stats.SlotAction.Interval = s.action.interval.Int64()
	stats.SlotAction.Disabled = s.action.disabled.Bool()
	stats.SlotAction.Progress.Status = s.action.progress.status.Load().(string)
	stats.SlotAction.Executor = s.action.executor.Int64()

	stats.HA.Model = ctx.sentinel
	stats.HA.Stats = map[string]*RedisStats{}
	for _, server := range ctx.sentinel.Servers {
		if v := s.stats.servers[server]; v != nil {
			stats.HA.Stats[server] = v
		}
	}
	stats.HA.Masters = make(map[string]string)
	if s.ha.masters != nil {
		for gid, addr := range s.ha.masters {
			stats.HA.Masters[strconv.Itoa(gid)] = addr
		}
	}
	return stats, nil
}

type Stats struct {
	Closed bool `json:"closed"`

	Slots []*models.SlotMapping `json:"slots"`

	Group struct {
		Models []*models.Group        `json:"models"`
		Stats  map[string]*RedisStats `json:"stats"`
	} `json:"group"`

	Proxy struct {
		Models []*models.Proxy        `json:"models"`
		Stats  map[string]*ProxyStats `json:"stats"`
	} `json:"proxy"`

	SlotAction struct {
		Interval int64 `json:"interval"`
		Disabled bool  `json:"disabled"`

		Progress struct {
			Status string `json:"status"`
		} `json:"progress"`

		Executor int64 `json:"executor"`
	} `json:"slot_action"`

	HA struct {
		Model   *models.Sentinel       `json:"model"`
		Stats   map[string]*RedisStats `json:"stats"`
		Masters map[string]string      `json:"masters"`
	} `json:"sentinels"`
}

func (s *Topom) Config() *Config {
	return s.config
}

func (s *Topom) IsOnline() bool {
	s.mu.Lock()
	defer s.mu.Unlock()
	return s.online && !s.closed
}

func (s *Topom) IsClosed() bool {
	s.mu.Lock()
	defer s.mu.Unlock()
	return s.closed
}

func (s *Topom) GetSlotActionInterval() int {
	return s.action.interval.AsInt()
}

func (s *Topom) SetSlotActionInterval(us int) {
	us = math2.MinMaxInt(us, 0, 1000*1000)
	s.action.interval.Set(int64(us))
	log.Warnf("set action interval = %d", us)
}

func (s *Topom) GetSlotActionDisabled() bool {
	return s.action.disabled.Bool()
}

func (s *Topom) SetSlotActionDisabled(value bool) {
	s.action.disabled.Set(value)
	log.Warnf("set action disabled = %t", value)
}

func (s *Topom) Slots() ([]*models.Slot, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	ctx, err := s.newContext()
	if err != nil {
		return nil, err
	}
	return ctx.toSlotSlice(ctx.slots, nil), nil
}

func (s *Topom) Reload() error {
	s.mu.Lock()
	defer s.mu.Unlock()
	_, err := s.newContext()
	if err != nil {
		return err
	}
	defer s.dirtyCacheAll()
	return nil
}

func (s *Topom) serveAdmin() {
	if s.IsClosed() {
		return
	}
	defer s.Close()

	log.Warnf("admin start service on %s", s.ladmin.Addr())

	eh := make(chan error, 1)
	go func(l net.Listener) {
		h := http.NewServeMux()
		h.Handle("/", newApiServer(s))
		hs := &http.Server{Handler: h}
		eh <- hs.Serve(l)
	}(s.ladmin)

	select {
	case <-s.exit.C:
		log.Warnf("admin shutdown")
	case err := <-eh:
		log.ErrorErrorf(err, "admin exit on error")
	}
}

type Overview struct {
	Version string        `json:"version"`
	Compile string        `json:"compile"`
	Config  *Config       `json:"config,omitempty"`
	Model   *models.Topom `json:"model,omitempty"`
	Stats   *Stats        `json:"stats,omitempty"`
}

func (s *Topom) Overview() (*Overview, error) {
	if stats, err := s.Stats(); err != nil {
		return nil, err
	} else {
		return &Overview{
			Version: utils.Version,
			Compile: utils.Compile,
			Config:  s.Config(),
			Model:   s.Model(),
			Stats:   stats,
		}, nil
	}
}
<|endoftext|>


<commit_before>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/*
"VCAP_SERVICES": {
  "user-provided": [
   {
    "credentials": {
     "host": "172.32.125.109:9200"
    },
    "label": "user-provided",
    "name": "pz-elasticsearch",
    "syslog_drain_url": "",
    "tags": []
   }
  ]
 }
}
*/

package piazza

import (
	"encoding/json"
	"log"
	"os"
)

type VcapCredentials struct {
	Host string `json:"host"`
}

type VcapServiceEntry struct {
	Credentials    VcapCredentials `json:"credentials"`
	Label          string          `json:"label"`
	Name           string          `json:"name"`
	SyslogDrainUrl string          `json:"syslog_drain_url"`
	Tags           []string        `json:"tags"`
}

type VcapServices struct {
	UserProvided []VcapServiceEntry `json:"user-provided"`

	Services map[ServiceName]string
}

var localVcapServices = &VcapServices{
	UserProvided: []VcapServiceEntry{
		VcapServiceEntry{
			Name: "pz-elasticsearch",
			Credentials: VcapCredentials{
				Host: "localhost:9200",
			},
		},
	},
}

func NewVcapServices() (*VcapServices, error) {

	var err error
	var vcap *VcapServices

	str := os.Getenv("VCAP_SERVICES")
	if str != "" {

		log.Printf("VCAP_SERVICES:\n%s", str)
		vcap = &VcapServices{}

		err = json.Unmarshal([]byte(str), vcap)
		if err != nil {
			return nil, err
		}

	} else {
		vcap = localVcapServices
	}

	vcap.Services = make(ServicesMap)

	for _, serviceEntry := range vcap.UserProvided {
		name := ServiceName(serviceEntry.Name)
		addr := serviceEntry.Credentials.Host
		vcap.Services[name] = addr
		log.Printf("VcapServices: added %s for %s", name, addr)
	}

	return vcap, nil
}
<commit_msg>remove extra log<commit_after>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/*
"VCAP_SERVICES": {
  "user-provided": [
   {
    "credentials": {
     "host": "172.32.125.109:9200"
    },
    "label": "user-provided",
    "name": "pz-elasticsearch",
    "syslog_drain_url": "",
    "tags": []
   }
  ]
 }
}
*/

package piazza

import (
	"encoding/json"
	"log"
	"os"
)

type VcapCredentials struct {
	Host string `json:"host"`
}

type VcapServiceEntry struct {
	Credentials    VcapCredentials `json:"credentials"`
	Label          string          `json:"label"`
	Name           string          `json:"name"`
	SyslogDrainUrl string          `json:"syslog_drain_url"`
	Tags           []string        `json:"tags"`
}

type VcapServices struct {
	UserProvided []VcapServiceEntry `json:"user-provided"`

	Services map[ServiceName]string
}

var localVcapServices = &VcapServices{
	UserProvided: []VcapServiceEntry{
		VcapServiceEntry{
			Name: "pz-elasticsearch",
			Credentials: VcapCredentials{
				Host: "localhost:9200",
			},
		},
	},
}

func NewVcapServices() (*VcapServices, error) {

	var err error
	var vcap *VcapServices

	str := os.Getenv("VCAP_SERVICES")
	if str != "" {

		log.Printf("VCAP_SERVICES:\n%s", str)
		vcap = &VcapServices{}

		err = json.Unmarshal([]byte(str), vcap)
		if err != nil {
			return nil, err
		}

	} else {
		vcap = localVcapServices
	}

	vcap.Services = make(ServicesMap)

	for _, serviceEntry := range vcap.UserProvided {
		name := ServiceName(serviceEntry.Name)
		addr := serviceEntry.Credentials.Host
		vcap.Services[name] = addr
		//log.Printf("VcapServices: added %s for %s", name, addr)
	}

	return vcap, nil
}
<|endoftext|>


<commit_before>package cert

import (
	"crypto/tls"
	"log"
	"crypto/x509"
	"io/ioutil"
	"../config"
)

var TlsConfig *tls.Config;

func init() {
	certpool := x509.NewCertPool()
	for _, crFile := range config.CfgIni.OtherCertificates {
		pem, err := ioutil.ReadFile(crFile)
		if err != nil {
			log.Fatalf("Failed to read client certificate authority: %v", err)
		}
		if !certpool.AppendCertsFromPEM(pem) {
			log.Fatalf("Can't parse client certificate authority")
		}
	}

	TlsConfig = &tls.Config{
		ClientAuth:   tls.RequireAndVerifyClientCert,
		ClientCAs:    certpool,
	}
	TlsConfig.BuildNameToCertificate()
}
<commit_msg>Removed Client SSL Certificate requirement.<commit_after>package cert

import (
	"crypto/tls"
	"log"
	"crypto/x509"
	"io/ioutil"
	"../config"
)

var TlsConfig *tls.Config;

func init() {
	certpool := x509.NewCertPool()
	for _, crFile := range config.CfgIni.OtherCertificates {
		pem, err := ioutil.ReadFile(crFile)
		if err != nil {
			log.Fatalf("Failed to read client certificate authority: %v", err)
		}
		if !certpool.AppendCertsFromPEM(pem) {
			log.Fatalf("Can't parse client certificate authority")
		}
	}

	TlsConfig = &tls.Config{
		ClientCAs:    certpool,
	}
	TlsConfig.BuildNameToCertificate()
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strings"
)

/* This function takes a string and returns
   a (potentially nil) error object */
func TTWS(filename string) error {
	/* Open the input file */
	inf, err := os.Open(filename);
	/* In case this function generates a "panic", be sure to close this file */
	defer inf.Close();
	/* Did we open it successfully?  If not, close and return. */
	if (err!=nil) { return err; }

	/* Open the output file in system temp dir*/
	outf, err := ioutil.TempFile("","");
	/* In case this function generates a "panic", be sure to close this file */
	defer outf.Close();
	/* Did we open it succesfully?  If not, close all and return. */
	if (err!=nil) { inf.Close(); return err; }

	/* Create a scanner object to break this in to lines */
	scanner := bufio.NewScanner(inf);
	/* Declare a variable for the line */
	var line string;
	/* Loop over lines */
	for scanner.Scan() {
		/* Trim right space and then add the \n back on the end before writing */
		line = strings.TrimRight(scanner.Text(), " \t")+"\n"
		outf.Write([]byte(line));
	}
	/* Close all open files */
	inf.Close();
	outf.Close();

	/* Replace the source file by the trimmed file */
	os.Rename(outf.Name(), filename);

	/* No errors, so we return nil */
	return nil;
}

func WalkFunc(path string, fi os.FileInfo, err error) error {
	/* list of directories to ignore */
	blacklist := []string{".bzr", ".cvs", ".git", ".hg", ".svn"}
	if contains(path, blacklist){
		fmt.Printf("Skipping version control dir: %s\n", path)
		return filepath.SkipDir
	} else {
		inf, err := os.Open(path)
		defer inf.Close();
		if (err!=nil) { return err; }
		readStart := io.LimitReader(inf, 512);
		data, err := ioutil.ReadAll(readStart);
		/* Close all open files */
		inf.Close();

		if (err!=nil) { return err; }

		/* Determine file type */
		fileType := http.DetectContentType(data);

		if (fi.IsDir()) return nil; // Now you don't need ot check this

		/* only act on text files */
		if (strings.Contains(fileType, "text/plain")) {
			fmt.Printf("Trimming: %v\n", path);
			TTWS(path);
		} else {
			fmt.Printf("Skipping file of type '%v': %v\n", fileType, path)
		}
	}
	return nil
}

func contains(x string, a []string) bool {
	for _, e := range(a) {
		if (x==e) { return true; }
	}
	return false;
}

func main() {
	flag.Parse()
	root := flag.Arg(0)
	err := filepath.Walk(root, WalkFunc)
	fmt.Printf("filepath.Walk() returned %v\n", err)
}
<commit_msg>Trying an approach with a manual walk<commit_after>package main

import (
	"bufio"
	"flag"
	//"fmt"
	"path"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
)

/* This function takes a string and returns
   a (potentially nil) error object */
func TTWS(filename string) error {
	/* Open the input file */
	inf, err := os.Open(filename);
	/* In case this function generates a "panic", be sure to close this file */
	defer inf.Close();
	/* Did we open it successfully?  If not, close and return. */
	if (err!=nil) { inf.Close(); return err; }

	/* Open the output file in system temp dir*/
	outf, err := ioutil.TempFile("","");
	/* In case this function generates a "panic", be sure to close this file */
	defer outf.Close();
	/* Did we open it succesfully?  If not, close all and return. */
	if (err!=nil) { inf.Close(); outf.Close(); return err; }
	/* Create a scanner object to break this in to lines */
	scanner := bufio.NewScanner(inf);
	/* Declare a variable for the line */
	var line string;
	/* Loop over lines */
	for scanner.Scan() {
		/* Trim right space and then add the \n back on the end before writing */
		line = strings.TrimRight(scanner.Text(), " \t")+"\n"
		outf.Write([]byte(line));
	}
	/* Close all open files */
	inf.Close();
	outf.Close();
	/* Replace the source file by the trimmed file */
	os.Rename(outf.Name(), filename);

	/* No errors, so we return nil */
	return nil;
}

var blacklist = []string{".bzr", ".cvs", ".git", ".hg", ".svn"}

func processFile(filename string) error {
	inf, err := os.Open(filename)
	defer inf.Close();
	if (err!=nil) { inf.Close(); return err; }

	readStart := io.LimitReader(inf, 512);

	data, err := ioutil.ReadAll(readStart);

	/* Close all open files */
	inf.Close();

	/* Determine file type */
	fileType := http.DetectContentType(data);

	/* only act on text files */
	if (strings.Contains(fileType, "text/plain")){
		//fmt.Printf("Trimming: %v\n", filename);
		return TTWS(filename);
	} else {
		//fmt.Printf("Skipping file of type '%v': %v\n", fileType, filename)
		return nil;
	}
}

func processNode(node string) error {
	fi, err := os.Lstat(node)
	if (err!=nil) { return err; }

	if (fi.IsDir()) {
		if contains(fi.Name(), blacklist) { return nil; }
		contents, err := ioutil.ReadDir(node);
		if (err!=nil) { return err; }
		for _, n := range(contents) {
			serr := processNode(path.Join(node, n.Name()));
			if (serr!=nil) { return serr; }
		}
		return nil;
	} else {
		return processFile(node);
	}
}

func contains(x string, a []string) bool {
	for _, e := range(a) {
		if (x==e) { return true; }
	}
	return false;
}

func main() {
	flag.Parse()
	root := flag.Arg(0)
	//err := filepath.Walk(root, WalkFunc)
	//filepath.Walk(root, WalkFunc)
	processNode(root);
	//fmt.Printf("filepath.Walk() returned %v\n", err)
}
<|endoftext|>


<commit_before>package plugins

import (
	"ircclient"
)

type ChannelsPlugin struct {
	ic *ircclient.IRCClient
}

func (q *ChannelsPlugin) Register(cl *ircclient.IRCClient) {
	q.ic = cl
}

func (q *ChannelsPlugin) String() string {
	return "channel"
}

func (q *ChannelsPlugin) Info() string {
	return "Manages channel auto-join and possibly options"
}

func (q *ChannelsPlugin) ProcessLine(msg *ircclient.IRCMessage) {
	if msg.Command != "001" {
		return
	}
	/* When registering, join channels */
	options := q.ic.GetOptions("Channels")
	for _, key := range options {
		q.ic.SendLine("JOIN #" + key)
	}
}

func (q *ChannelsPlugin) ProcessCommand(cmd *ircclient.IRCCommand) {
	// TODO: Delchannel
	if cmd.Command != "addchannel" && cmd.Command != "join" && cmd.Command != "part" {
		return
	}
	if q.ic.GetAccessLevel(cmd.Source) < 200 {
		q.ic.Reply(cmd, "You are not authorized to do that")
		return
	}
	if len(cmd.Args) < 1 {
		q.ic.Reply(cmd, "Too few parameters. Please specify a channel name.")
		return
	}
	if cmd.Command == "join" {
		q.ic.SendLine("JOIN #" + cmd.Args[0])
		return
	}
	if cmd.Command == "part" {
		q.ic.SendLine("PART #" + cmd.Args[0])
		return
	}
	// TODO: Quick'n'dirty. Check whether channel already exists and strip #, if
	// existent.
	q.ic.SetStringOption("Channels", cmd.Args[0], "42")
	q.ic.SendLine("JOIN #" + cmd.Args[0])
}

func (q *ChannelsPlugin) Unregister() {
	return
}
<commit_msg>Update channel.go to use new plugin API<commit_after>package plugins

import (
	"ircclient"
)

type ChannelsPlugin struct {
	ic *ircclient.IRCClient
}

func (q *ChannelsPlugin) Register(cl *ircclient.IRCClient) {
	q.ic = cl
	cl.RegisterCommandHandler("join", 1, 200, q)
	cl.RegisterCommandHandler("part", 1, 200, q)
	cl.RegisterCommandHandler("addchannel", 1, 400, q)
}

func (q *ChannelsPlugin) String() string {
	return "channel"
}

func (q *ChannelsPlugin) Info() string {
	return "Manages channel auto-join and possibly options"
}

func (q *ChannelsPlugin) ProcessLine(msg *ircclient.IRCMessage) {
	if msg.Command != "001" {
		return
	}
	/* When registering, join channels */
	options := q.ic.GetOptions("Channels")
	for _, key := range options {
		q.ic.SendLine("JOIN #" + key)
	}
}

func (q *ChannelsPlugin) ProcessCommand(cmd *ircclient.IRCCommand) {
	switch cmd.Command {
	case "join":
		q.ic.SendLine("JOIN #" + cmd.Args[0])
	case "part":
		q.ic.SendLine("PART #" + cmd.Args[0])
	case "addchannel":
		// TODO: Quick'n'dirty. Check whether channel already exists and strip #, if
		// existent.
		q.ic.SetStringOption("Channels", cmd.Args[0], "42")
		q.ic.SendLine("JOIN #" + cmd.Args[0])
	}
}

func (q *ChannelsPlugin) Unregister() {
	return
}
<|endoftext|>


<commit_before>package crd

import (
	"fmt"
	"reflect"
	"time"

	"github.com/golang/glog"
	"github.com/liyinan926/spark-operator/pkg/apis/v1alpha1"

	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/errors"
	"k8s.io/apimachinery/pkg/util/wait"
)

// CRD metadata.
const (
	CRDPlural   = "sparkapplications"
	CRDSingular = "sparkapplication"
	CRDKind     = "SparkApplication"
	CRDGroup    = v1alpha1.GroupName
	CRDVersion  = "v1alpha1"
	CRDFullName = CRDPlural + "." + CRDGroup
)

// CreateCRD creates a Kubernetes CustomResourceDefinition (CRD) for SparkApplication.
// An error is returned if it fails to create the CustomResourceDefinition before it times out.
func CreateCRD(clientset apiextensionsclient.Interface) error {
	// The CustomResourceDefinition is not found, create it now.
	crd := &apiextensionsv1beta1.CustomResourceDefinition{
		ObjectMeta: metav1.ObjectMeta{
			Name: CRDFullName,
		},
		Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
			Group:   CRDGroup,
			Version: CRDVersion,
			Scope:   apiextensionsv1beta1.NamespaceScoped,
			Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
				Plural: CRDPlural,
				Kind:   reflect.TypeOf(v1alpha1.SparkApplication{}).Name(),
			},
		},
	}
	_, err := clientset.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd)
	if err != nil {
		if apierrors.IsAlreadyExists(err) {
			glog.Warningf("CustomResourceDefinition %s already exists", CRDFullName)
			return nil
		}
		return err
	}

	// Wait for the CustomResourceDefinition to become registered.
	err = waitForCRDEstablishment(clientset)
	// Try deleting the CustomResourceDefinition if it fails to be registered on time.
	if err != nil {
		deleteErr := clientset.ApiextensionsV1beta1().CustomResourceDefinitions().Delete(CRDFullName, &metav1.DeleteOptions{})
		if deleteErr != nil {
			return errors.NewAggregate([]error{err, deleteErr})
		}
		return err
	}

	return nil
}

func getCRD(clientset apiextensionsclient.Interface) (*apiextensionsv1beta1.CustomResourceDefinition, error) {
	return clientset.ApiextensionsV1beta1().CustomResourceDefinitions().Get(CRDFullName, metav1.GetOptions{})
}

// waitForCRDEstablishment waits for the CRD to be registered and established until it times out.
func waitForCRDEstablishment(clientset apiextensionsclient.Interface) error {
	return wait.Poll(500*time.Millisecond, 60*time.Second, func() (bool, error) {
		crd, err := getCRD(clientset)
		for _, cond := range crd.Status.Conditions {
			switch cond.Type {
			case apiextensionsv1beta1.Established:
				if cond.Status == apiextensionsv1beta1.ConditionTrue {
					return true, err
				}
			case apiextensionsv1beta1.NamesAccepted:
				if cond.Status == apiextensionsv1beta1.ConditionFalse {
					fmt.Printf("Name conflict: %v\n", cond.Reason)
				}
			}
		}
		return false, err
	})
}
<commit_msg>Support singular and short names for the CRD<commit_after>package crd

import (
	"fmt"
	"reflect"
	"time"

	"github.com/golang/glog"
	"github.com/liyinan926/spark-operator/pkg/apis/v1alpha1"

	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/errors"
	"k8s.io/apimachinery/pkg/util/wait"
)

// CRD metadata.
const (
	CRDPlural    = "sparkapplications"
	CRDSingular  = "sparkapplication"
	CRDShortName = "sparkapp"
	CRDGroup     = v1alpha1.GroupName
	CRDVersion   = "v1alpha1"
	CRDFullName  = CRDPlural + "." + CRDGroup
)

// CreateCRD creates a Kubernetes CustomResourceDefinition (CRD) for SparkApplication.
// An error is returned if it fails to create the CustomResourceDefinition before it times out.
func CreateCRD(clientset apiextensionsclient.Interface) error {
	// The CustomResourceDefinition is not found, create it now.
	crd := &apiextensionsv1beta1.CustomResourceDefinition{
		ObjectMeta: metav1.ObjectMeta{
			Name: CRDFullName,
		},
		Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
			Group:   CRDGroup,
			Version: CRDVersion,
			Scope:   apiextensionsv1beta1.NamespaceScoped,
			Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
				Plural:     CRDPlural,
				Singular:   CRDSingular,
				ShortNames: []string{CRDShortName},
				Kind:       reflect.TypeOf(v1alpha1.SparkApplication{}).Name(),
			},
		},
	}
	_, err := clientset.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd)
	if err != nil {
		if apierrors.IsAlreadyExists(err) {
			glog.Warningf("CustomResourceDefinition %s already exists", CRDFullName)
			return nil
		}
		return err
	}

	// Wait for the CustomResourceDefinition to become registered.
	err = waitForCRDEstablishment(clientset)
	// Try deleting the CustomResourceDefinition if it fails to be registered on time.
	if err != nil {
		deleteErr := clientset.ApiextensionsV1beta1().CustomResourceDefinitions().Delete(CRDFullName, &metav1.DeleteOptions{})
		if deleteErr != nil {
			return errors.NewAggregate([]error{err, deleteErr})
		}
		return err
	}

	return nil
}

func getCRD(clientset apiextensionsclient.Interface) (*apiextensionsv1beta1.CustomResourceDefinition, error) {
	return clientset.ApiextensionsV1beta1().CustomResourceDefinitions().Get(CRDFullName, metav1.GetOptions{})
}

// waitForCRDEstablishment waits for the CRD to be registered and established until it times out.
func waitForCRDEstablishment(clientset apiextensionsclient.Interface) error {
	return wait.Poll(500*time.Millisecond, 60*time.Second, func() (bool, error) {
		crd, err := getCRD(clientset)
		for _, cond := range crd.Status.Conditions {
			switch cond.Type {
			case apiextensionsv1beta1.Established:
				if cond.Status == apiextensionsv1beta1.ConditionTrue {
					return true, err
				}
			case apiextensionsv1beta1.NamesAccepted:
				if cond.Status == apiextensionsv1beta1.ConditionFalse {
					fmt.Printf("Name conflict: %v\n", cond.Reason)
				}
			}
		}
		return false, err
	})
}
<|endoftext|>


<commit_before>package net

import (
	"crypto/ed25519"
	"crypto/tls"
	"expvar"
	"io"
	"math"
	"net"
	"strings"
	"time"

	"github.com/patrickmn/go-cache"
	"github.com/zserge/metric"

	"nimona.io/pkg/context"
	"nimona.io/pkg/crypto"
	"nimona.io/pkg/errors"
	"nimona.io/pkg/eventbus"
	"nimona.io/pkg/keychain"
	"nimona.io/pkg/log"
	"nimona.io/pkg/peer"
)

var (
	DefaultNetwork = New(
		WithEventBus(eventbus.DefaultEventbus),
		WithKeychain(keychain.DefaultKeychain),
	)
)

// TODO remove UseUPNP and replace with option
// nolint: gochecknoinits
func init() {
	connConnOutCounter := metric.NewCounter("2m1s", "15m30s", "1h1m")
	expvar.Publish("nm:net.conn.out", connConnOutCounter)

	connConnIncCounter := metric.NewCounter("2m1s", "15m30s", "1h1m")
	expvar.Publish("nm:net.conn.in", connConnIncCounter)

	connDialCounter := metric.NewCounter("2m1s", "15m30s", "1h1m")
	expvar.Publish("nm:net.dial", connDialCounter)

	connBlocklistCounter := metric.NewCounter("2m1s", "15m30s", "1h1m")
	expvar.Publish("nm:net.conn.dial.blocked", connBlocklistCounter)
}

type (
	// Network interface
	Network interface {
		Dial(
			ctx context.Context,
			peer *peer.Peer,
		) (*Connection, error)
		Listen(
			ctx context.Context,
			bindAddress string,
		) (Listener, error)
		Accept() (*Connection, error)
		Addresses() []string
	}
	// Option for customizing a new network
	Option func(*network)
)

// New creates a new p2p network
func New(opts ...Option) Network {
	n := &network{
		keychain:    keychain.DefaultKeychain,
		eventbus:    eventbus.DefaultEventbus,
		transports:  map[string]Transport{},
		middleware:  []MiddlewareHandler{},
		listeners:   []*listener{},
		connections: make(chan *Connection),
		blocklist:   cache.New(time.Second*5, time.Second*60),
	}
	for _, opt := range opts {
		opt(n)
	}
	n.transports["tcps"] = &tcpTransport{
		keychain: n.keychain,
	}
	return n
}

func Dial(
	ctx context.Context,
	p *peer.Peer,
) (*Connection, error) {
	return DefaultNetwork.Dial(ctx, p)
}

func Listen(
	ctx context.Context,
	bindAddress string,
) (Listener, error) {
	return DefaultNetwork.Listen(ctx, bindAddress)
}

func Accept() (*Connection, error) {
	return DefaultNetwork.Accept()
}

func Addresses() []string {
	return DefaultNetwork.Addresses()
}

// network allows dialing and listening for p2p connections
type network struct {
	eventbus    eventbus.Eventbus
	keychain    keychain.Keychain
	transports  map[string]Transport
	middleware  []MiddlewareHandler
	listeners   []*listener
	connections chan *Connection
	attempts    attemptsMap
	blocklist   *cache.Cache
}

// Dial to a peer and return a net.Conn or error
func (n *network) Dial(
	ctx context.Context,
	p *peer.Peer,
) (*Connection, error) {
	logger := log.FromContext(ctx).With(
		log.String("peer", p.PublicKey().String()),
		log.Strings("addresses", p.Addresses),
	)

	if len(p.Addresses) == 0 {
		return nil, ErrNoAddresses
	}

	logger.Debug("dialing")
	expvar.Get("nm:net.dial").(metric.Metric).Add(1)

	// keep a flag on whether all addresses where blocked so we can return
	// an ErrAllAddressesBlocked error
	allBlocked := true

	// go through all addresses and try to dial them
	for _, address := range p.Addresses {
		// check if address is currently blocklisted
		if _, blocklisted := n.blocklist.Get(address); blocklisted {
			logger.Debug("address is blocklisted, skipping")
			continue
		}
		// get protocol from address
		addressType := strings.Split(address, ":")[0]
		trsp, ok := n.transports[addressType]
		if !ok {
			logger.Debug("not sure how to dial",
				log.String("type", addressType),
			)
			continue
		}

		// reset blocked flag
		allBlocked = false

		// dial address
		conn, err := trsp.Dial(ctx, address)
		if err != nil {
			// blocking address
			expvar.Get("nm:net.conn.dial.blocked").(metric.Metric).Add(1)
			attempts, backoff := n.exponentialyBlockAddress(address)
			logger.Error("could not dial address, blocking",
				log.Int("failedAttempts", attempts),
				log.String("backoff", backoff.String()),
				log.String("type", addressType),
				log.Error(err),
			)
			continue
		}

		// pass connection to all middleware
		conn, err = n.handleMiddleware(ctx, conn)
		if err != nil {
			continue
		}

		// check negotiated key against dialed
		if conn.RemotePeerKey != p.PublicKey() {
			n.exponentialyBlockAddress(address)
			logger.Error("remote didn't match expect key, blocking",
				log.String("expected", p.PublicKey().String()),
				log.String("received", conn.RemotePeerKey.String()),
			)
			continue
		}

		// at this point we consider the connection successful, so we can
		// reset the failed attempts
		n.attempts.Put(address, 0)
		n.attempts.Put(p.PublicKey().String(), 0)

		expvar.Get("nm:net.conn.out").(metric.Metric).Add(1)

		return conn, nil
	}

	err := ErrAllAddressesFailed
	if allBlocked {
		err = ErrAllAddressesBlocked
	}

	logger.Error("could not dial peer", log.Error(err))
	return nil, err
}

func (n *network) exponentialyBlockAddress(k string) (int, time.Duration) {
	baseBackoff := float64(time.Second * 1)
	maxBackoff := float64(time.Minute * 10)
	attempts, _ := n.attempts.Get(k)
	attempts++
	backoff := baseBackoff * math.Pow(1.5, float64(attempts))
	if backoff > maxBackoff {
		backoff = maxBackoff
	}
	n.attempts.Put(k, attempts)
	n.blocklist.Set(k, attempts, time.Duration(backoff))
	return attempts, time.Duration(backoff)
}

func (n *network) Accept() (*Connection, error) {
	conn := <-n.connections
	return conn, nil
}

// Listen
// TODO do we need to return a listener?
func (n *network) Listen(
	ctx context.Context,
	bindAddress string,
) (Listener, error) {
	mlst := &listener{
		addresses: []string{},
		listeners: []net.Listener{},
	}
	k := n.keychain.GetPrimaryPeerKey()
	for pt, tsp := range n.transports {
		lst, err := tsp.Listen(ctx, bindAddress, k)
		if err != nil {
			return nil, err
		}

		mlst.listeners = append(mlst.listeners, lst)
		mlst.addresses = append(mlst.addresses, GetAddresses(pt, lst)...)

		n.listeners = append(n.listeners, mlst)

		for _, addr := range mlst.addresses {
			n.eventbus.Publish(eventbus.NetworkAddressAdded{
				Address: addr,
			})
		}

		// TODO goroutine never ends
		go func() {
			for {
				rawConn, err := lst.Accept()
				if err != nil {
					// we need to check whether the error is temporary,
					// a non-temporary error would be for example a closed
					// listener
					errIsTemp := true
					if opErr, ok := err.(*net.OpError); ok {
						errIsTemp = opErr.Temporary()
					}
					// if the error is not temporary stop trying to accept
					// connections from this lsitener
					if !errIsTemp {
						return
					}
					// else, just move on
					continue
				}

				conn := newConnection(rawConn, true)
				conn.remoteAddress = rawConn.RemoteAddr().String()
				conn.localAddress = rawConn.LocalAddr().String()

				if tlsConn, ok := rawConn.(*tls.Conn); ok {
					if err := tlsConn.Handshake(); err != nil {
						// not currently supported
						// TODO find a way to surface this error
						conn.Close() // nolint: errcheck
						continue
					}
					state := tlsConn.ConnectionState()
					certs := state.PeerCertificates
					if len(certs) != 1 {
						// not currently supported
						// TODO find a way to surface this error
						conn.Close() // nolint: errcheck
						continue
					}
					pubKey, ok := certs[0].PublicKey.(ed25519.PublicKey)
					if !ok {
						// not currently supported
						// TODO find a way to surface this error
						conn.Close() // nolint: errcheck
						continue
					}
					conn.RemotePeerKey = crypto.NewPublicKey(pubKey)
				} else {
					// not currently supported
					// TODO find a way to surface this error
					conn.Close() // nolint: errcheck
					continue
				}

				n.eventbus.Publish(eventbus.PeerConnectionEstablished{
					PublicKey: conn.RemotePeerKey,
				})

				expvar.Get("nm:net.conn.in").(metric.Metric).Add(1)
				n.connections <- conn
			}
		}()
	}
	// block our own addresses, just in case anyone tries to dial them
	for _, addr := range mlst.addresses {
		n.blocklist.Set(addr, 0, cache.NoExpiration)
	}
	return mlst, nil
}

func (n *network) Addresses() []string {
	addrs := []string{}
	for _, l := range n.listeners {
		addrs = append(addrs, l.Addresses()...)
	}
	return addrs
}

func (n *network) handleMiddleware(
	ctx context.Context,
	conn *Connection,
) (*Connection, error) {
	var err error
	for _, mh := range n.middleware {
		conn, err = mh(ctx, conn)
		if err != nil {
			if errors.CausedBy(err, io.EOF) {
				break
			}
			if conn != nil {
				conn.conn.Close() // nolint: errcheck
			}
			return nil, err
		}
	}
	return conn, nil
}
<commit_msg>refactor(net): replace expvar with prometheus metrics<commit_after>package net

import (
	"crypto/ed25519"
	"crypto/tls"
	"io"
	"math"
	"net"
	"strings"
	"time"

	"github.com/patrickmn/go-cache"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"

	"nimona.io/pkg/context"
	"nimona.io/pkg/crypto"
	"nimona.io/pkg/errors"
	"nimona.io/pkg/eventbus"
	"nimona.io/pkg/keychain"
	"nimona.io/pkg/log"
	"nimona.io/pkg/peer"
)

var (
	DefaultNetwork = New(
		WithEventBus(eventbus.DefaultEventbus),
		WithKeychain(keychain.DefaultKeychain),
	)
	connConnOutCounter = promauto.NewCounter(
		prometheus.CounterOpts{
			Name: "nimona_net_conn_out_total",
			Help: "Total number of outgoing connections",
		},
	)
	connConnIncCounter = promauto.NewCounter(
		prometheus.CounterOpts{
			Name: "nimona_net_conn_in_total",
			Help: "Total number of incoming connections",
		},
	)
	connDialAttemptCounter = promauto.NewCounter(
		prometheus.CounterOpts{
			Name: "nimona_net_dial_attempt_total",
			Help: "Total number of dial attempts",
		},
	)
	connDialSuccessCounter = promauto.NewCounter(
		prometheus.CounterOpts{
			Name: "nimona_net_dial_success_total",
			Help: "Total number of successful dials",
		},
	)
	connDialErrorCounter = promauto.NewCounter(
		prometheus.CounterOpts{
			Name: "nimona_net_dial_failed_total",
			Help: "Total number of failed dials",
		},
	)
	connDialBlockedCounter = promauto.NewCounter(
		prometheus.CounterOpts{
			Name: "nimona_net_dial_blocked_total",
			Help: "Total number of failed dials due to all addresses blocked",
		},
	)
)

type (
	// Network interface
	Network interface {
		Dial(
			ctx context.Context,
			peer *peer.Peer,
		) (*Connection, error)
		Listen(
			ctx context.Context,
			bindAddress string,
		) (Listener, error)
		Accept() (*Connection, error)
		Addresses() []string
	}
	// Option for customizing a new network
	Option func(*network)
)

// New creates a new p2p network
func New(opts ...Option) Network {
	n := &network{
		keychain:    keychain.DefaultKeychain,
		eventbus:    eventbus.DefaultEventbus,
		transports:  map[string]Transport{},
		middleware:  []MiddlewareHandler{},
		listeners:   []*listener{},
		connections: make(chan *Connection),
		blocklist:   cache.New(time.Second*5, time.Second*60),
	}
	for _, opt := range opts {
		opt(n)
	}
	n.transports["tcps"] = &tcpTransport{
		keychain: n.keychain,
	}
	return n
}

func Dial(
	ctx context.Context,
	p *peer.Peer,
) (*Connection, error) {
	return DefaultNetwork.Dial(ctx, p)
}

func Listen(
	ctx context.Context,
	bindAddress string,
) (Listener, error) {
	return DefaultNetwork.Listen(ctx, bindAddress)
}

func Accept() (*Connection, error) {
	return DefaultNetwork.Accept()
}

func Addresses() []string {
	return DefaultNetwork.Addresses()
}

// network allows dialing and listening for p2p connections
type network struct {
	eventbus    eventbus.Eventbus
	keychain    keychain.Keychain
	transports  map[string]Transport
	middleware  []MiddlewareHandler
	listeners   []*listener
	connections chan *Connection
	attempts    attemptsMap
	blocklist   *cache.Cache
}

// Dial to a peer and return a net.Conn or error
func (n *network) Dial(
	ctx context.Context,
	p *peer.Peer,
) (*Connection, error) {
	logger := log.FromContext(ctx).With(
		log.String("peer", p.PublicKey().String()),
		log.Strings("addresses", p.Addresses),
	)

	if len(p.Addresses) == 0 {
		return nil, ErrNoAddresses
	}

	logger.Debug("dialing")
	connDialAttemptCounter.Inc()

	// keep a flag on whether all addresses where blocked so we can return
	// an ErrAllAddressesBlocked error
	allBlocked := true

	// go through all addresses and try to dial them
	for _, address := range p.Addresses {
		// check if address is currently blocklisted
		if _, blocklisted := n.blocklist.Get(address); blocklisted {
			logger.Debug("address is blocklisted, skipping")
			continue
		}
		// get protocol from address
		addressType := strings.Split(address, ":")[0]
		trsp, ok := n.transports[addressType]
		if !ok {
			logger.Debug("not sure how to dial",
				log.String("type", addressType),
			)
			continue
		}

		// reset blocked flag
		allBlocked = false

		// dial address
		conn, err := trsp.Dial(ctx, address)
		if err != nil {
			// blocking address
			attempts, backoff := n.exponentialyBlockAddress(address)
			logger.Error("could not dial address, blocking",
				log.Int("failedAttempts", attempts),
				log.String("backoff", backoff.String()),
				log.String("type", addressType),
				log.Error(err),
			)
			continue
		}

		// pass connection to all middleware
		conn, err = n.handleMiddleware(ctx, conn)
		if err != nil {
			continue
		}

		// check negotiated key against dialed
		if conn.RemotePeerKey != p.PublicKey() {
			n.exponentialyBlockAddress(address)
			logger.Error("remote didn't match expect key, blocking",
				log.String("expected", p.PublicKey().String()),
				log.String("received", conn.RemotePeerKey.String()),
			)
			continue
		}

		// at this point we consider the connection successful, so we can
		// reset the failed attempts
		n.attempts.Put(address, 0)
		n.attempts.Put(p.PublicKey().String(), 0)

		connDialSuccessCounter.Inc()
		connConnOutCounter.Inc()

		return conn, nil
	}

	err := ErrAllAddressesFailed
	if allBlocked {
		err = ErrAllAddressesBlocked
		connDialBlockedCounter.Inc()
	} else {
		connDialErrorCounter.Inc()
	}

	logger.Error("could not dial peer", log.Error(err))
	return nil, err
}

func (n *network) exponentialyBlockAddress(k string) (int, time.Duration) {
	baseBackoff := float64(time.Second * 1)
	maxBackoff := float64(time.Minute * 10)
	attempts, _ := n.attempts.Get(k)
	attempts++
	backoff := baseBackoff * math.Pow(1.5, float64(attempts))
	if backoff > maxBackoff {
		backoff = maxBackoff
	}
	n.attempts.Put(k, attempts)
	n.blocklist.Set(k, attempts, time.Duration(backoff))
	return attempts, time.Duration(backoff)
}

func (n *network) Accept() (*Connection, error) {
	conn := <-n.connections
	return conn, nil
}

// Listen
// TODO do we need to return a listener?
func (n *network) Listen(
	ctx context.Context,
	bindAddress string,
) (Listener, error) {
	mlst := &listener{
		addresses: []string{},
		listeners: []net.Listener{},
	}
	k := n.keychain.GetPrimaryPeerKey()
	for pt, tsp := range n.transports {
		lst, err := tsp.Listen(ctx, bindAddress, k)
		if err != nil {
			return nil, err
		}

		mlst.listeners = append(mlst.listeners, lst)
		mlst.addresses = append(mlst.addresses, GetAddresses(pt, lst)...)

		n.listeners = append(n.listeners, mlst)

		for _, addr := range mlst.addresses {
			n.eventbus.Publish(eventbus.NetworkAddressAdded{
				Address: addr,
			})
		}

		// TODO goroutine never ends
		go func() {
			for {
				rawConn, err := lst.Accept()
				if err != nil {
					// we need to check whether the error is temporary,
					// a non-temporary error would be for example a closed
					// listener
					errIsTemp := true
					if opErr, ok := err.(*net.OpError); ok {
						errIsTemp = opErr.Temporary()
					}
					// if the error is not temporary stop trying to accept
					// connections from this lsitener
					if !errIsTemp {
						return
					}
					// else, just move on
					continue
				}

				conn := newConnection(rawConn, true)
				conn.remoteAddress = rawConn.RemoteAddr().String()
				conn.localAddress = rawConn.LocalAddr().String()

				if tlsConn, ok := rawConn.(*tls.Conn); ok {
					if err := tlsConn.Handshake(); err != nil {
						// not currently supported
						// TODO find a way to surface this error
						conn.Close() // nolint: errcheck
						continue
					}
					state := tlsConn.ConnectionState()
					certs := state.PeerCertificates
					if len(certs) != 1 {
						// not currently supported
						// TODO find a way to surface this error
						conn.Close() // nolint: errcheck
						continue
					}
					pubKey, ok := certs[0].PublicKey.(ed25519.PublicKey)
					if !ok {
						// not currently supported
						// TODO find a way to surface this error
						conn.Close() // nolint: errcheck
						continue
					}
					conn.RemotePeerKey = crypto.NewPublicKey(pubKey)
				} else {
					// not currently supported
					// TODO find a way to surface this error
					conn.Close() // nolint: errcheck
					continue
				}

				n.eventbus.Publish(eventbus.PeerConnectionEstablished{
					PublicKey: conn.RemotePeerKey,
				})

				connConnIncCounter.Inc()
				n.connections <- conn
			}
		}()
	}
	// block our own addresses, just in case anyone tries to dial them
	for _, addr := range mlst.addresses {
		n.blocklist.Set(addr, 0, cache.NoExpiration)
	}
	return mlst, nil
}

func (n *network) Addresses() []string {
	addrs := []string{}
	for _, l := range n.listeners {
		addrs = append(addrs, l.Addresses()...)
	}
	return addrs
}

func (n *network) handleMiddleware(
	ctx context.Context,
	conn *Connection,
) (*Connection, error) {
	var err error
	for _, mh := range n.middleware {
		conn, err = mh(ctx, conn)
		if err != nil {
			if errors.CausedBy(err, io.EOF) {
				break
			}
			if conn != nil {
				conn.conn.Close() // nolint: errcheck
			}
			return nil, err
		}
	}
	return conn, nil
}
<|endoftext|>