<commit_before>package garvis

import (
	"bytes"
	"fmt"
	"regexp"
	"strconv"
	"strings"

	"golang.org/x/net/context"
	"google.golang.org/appengine/datastore"
	glog "google.golang.org/appengine/log"

	"github.com/go-telegram-bot-api/telegram-bot-api"
)

type TextFilter struct {
	bot  *tgbotapi.BotAPI
	ctx  context.Context
	done chan bool
}

type TextFilterRule struct {
	ChatID    int64
	RxText    string
	TextReply string
	Count     int
	Limit     int
	UserID    int
	Hidden    bool
	CreatorID int
}

type User struct {
	UserID   int
	UserName string
}

func (filter TextFilter) Run(update tgbotapi.Update) error {
	ctx := filter.ctx

	// It is not possible to match usernames to user ids from the API
	// for the case of mentions so we need to store username-ids in the database
	// for lookup
	updateUsers(ctx, update)

	query := datastore.NewQuery("Rule").Filter("ChatID = ", update.Message.Chat.ID)
	queryUserRules := query.Filter("UserID = ", update.Message.From.ID)
	queryStaticRules := query.Filter("UserID = ", 0)

	var userRules []TextFilterRule
	var staticRules []TextFilterRule
	userKeys, err := queryUserRules.GetAll(ctx, &userRules)
	staticKeys, err := queryStaticRules.GetAll(ctx, &staticRules)
	if err != nil {
		glog.Errorf(ctx, "client.GetAll: %v", err)
	}
	rules := append(userRules, staticRules...)
	keys := append(userKeys, staticKeys...)

	for i, rule := range rules {
		k := keys[i]

		rxRule := regexp.MustCompile(rule.RxText)

		if rxRule.MatchString(update.Message.Text) {
			rule.Count = rule.Count + 1
			if rule.Count >= rule.Limit {
				rule.Count = 0
				_, err = datastore.Put(ctx, k, &rule)
				if err != nil {
					glog.Errorf(ctx, "client.Put(reset): %v", err)
				}
				filter.Trigger(rule)
			}
			_, err = datastore.Put(ctx, k, &rule)
			if err != nil {
				glog.Errorf(ctx, "client.Put: %v", err)
			}
		}
	}

	filter.done <- true
	return nil
}

func (filter TextFilter) GetCommands(commands map[string]Filter) {
	commands["addrule"] = filter
	commands["deleterule"] = filter
	commands["listrules"] = filter
	commands["addhiddenrule"] = filter
}

func (filter TextFilter) RunCommand(cmd string, cmdarg CommandArguments) {
	update := cmdarg.update
	var err error
	switch cmd {
	case "addrule":
		err = filter.addRule(update, false)
	case "deleterule":
		err = filter.deleteRule(update)
	case "listrules":
		err = filter.listRules(update)
	case "addhiddenrule":
		err = filter.addRule(update, true)
	}
	if err != nil {
		glog.Errorf(filter.ctx, err.Error())
	}
}

func (filter TextFilter) Trigger(rule TextFilterRule) {
	msg := tgbotapi.NewMessage(rule.ChatID, rule.TextReply)
	filter.bot.Send(msg)
}

func (filter TextFilter) addRule(update tgbotapi.Update, hidden bool) (err error) {
	ctx := filter.ctx

	command := strings.SplitN(update.Message.Text, " ", 2)
	if len(command) < 2 {
		usage := "Usage: /addrule {regex matcher}~{reply}{#count (optional default: 1)}~{user (optional)}"
		msg := tgbotapi.NewMessage(update.Message.Chat.ID, usage)
		filter.bot.Send(msg)
		return nil
	}
	argstr := command[1]
	args := strings.SplitN(argstr, "~", 3)
	var userID int
	switch len(args) {
	case 2:
		userID = 0
	case 3:
		ents := update.Message.Entities
		for _, ent := range *ents {
			switch ent.Type {
			case "text_mention":
				userID = ent.User.ID
			case "mention":
				if userID, err = getUserID(ctx, args[2][1:]); err != nil {
					return err
				}
			}
		}
	default:
		return nil
	}
	arg1 := strings.SplitN(args[0], "#", 2)
	text := fmt.Sprintf("(?i)%v(?-i)", arg1[0])
	_, err = regexp.Compile(text)
	if err != nil {
		usage := "Invalid Regex"
		msg := tgbotapi.NewMessage(update.Message.Chat.ID, usage)
		filter.bot.Send(msg)
		return nil
	}
	var limit int
	if len(arg1) < 2 {
		limit = 1
	} else {
		limit, _ = strconv.Atoi(arg1[1])
	}
	textreply := args[1]

	keyl, _, _ := datastore.AllocateIDs(ctx, "Rule", nil, 1)
	ruleKey := datastore.NewKey(ctx, "Rule", "", keyl, nil)
	var creatorID int
	if creatorID, err = getUserID(ctx, update.Message.From.UserName); err != nil {
		return err
	}
	rule := TextFilterRule{
		ChatID:    update.Message.Chat.ID,
		RxText:    text,
		TextReply: textreply,
		Count:     0,
		Limit:     int(limit),
		UserID:    userID,
		CreatorID: creatorID,
		Hidden:    hidden,
	}

	_, err = datastore.Put(ctx, ruleKey, &rule)

	return err
}

func (filter TextFilter) deleteRule(update tgbotapi.Update) (err error) {
	ctx := filter.ctx

	command := strings.SplitN(update.Message.Text, " ", 2)
	if len(command) < 2 {
		usage := "Usage: /deleterule {ruleID}"
		msg := tgbotapi.NewMessage(update.Message.Chat.ID, usage)
		filter.bot.Send(msg)
		return nil
	}
	key, _ := strconv.ParseInt(command[1], 10, 64)
	ruleKey := datastore.NewKey(ctx, "Rule", "", key, nil)

	var userID int
	if userID, err = getUserID(ctx, update.Message.From.UserName); err != nil {
		return err
	}

	err = datastore.RunInTransaction(ctx, func(ctx context.Context) error {
		var rule TextFilterRule
		err := datastore.Get(ctx, ruleKey, &rule)
		if err != nil {
			return err
		}

		if rule.ChatID == update.Message.Chat.ID || rule.CreatorID == userID {
			err = datastore.Delete(ctx, ruleKey)
		}

		return err
	}, nil)

	return err
}

func (filter TextFilter) listRules(update tgbotapi.Update) (err error) {
	ctx := filter.ctx

	var buffer bytes.Buffer
	var query *datastore.Query
	var header string

	if update.Message.Chat.Type == "private" {
		var userID int
		if userID, err = getUserID(ctx, update.Message.From.UserName); err != nil {
			return err
		}
		query = datastore.NewQuery("Rule").Filter("CreatorID = ", userID)
		header = fmt.Sprintf("|%s|%s|%s|%s|%s|%s|\n", "Chat name", "ID", "Regex", "Reply", "Count", "User(0 for all)")
	} else {
		query = datastore.NewQuery("Rule").Filter("ChatID = ", update.Message.Chat.ID)
		header = fmt.Sprintf("|%s|%s|%s|%s|%s|\n", "ID", "Regex", "Reply", "Count", "User(0 for all)")
	}
	buffer.WriteString(header)

	for t := query.Run(ctx); ; {
		var rule TextFilterRule
		var ruleText string

		k, err := t.Next(&rule)
		if err == datastore.Done {
			break
		}

		if update.Message.Chat.Type == "private" {
			chat, _ := filter.bot.GetChat(tgbotapi.ChatConfig{ChatID: rule.ChatID})
			ruleText = fmt.Sprintf("|%v|%v|%v|%v|%v|%v|\n", chat.Title, k.IntID(), rule.RxText, rule.TextReply, rule.Limit, rule.UserID)
		} else if !rule.Hidden {
			ruleText = fmt.Sprintf("|%v|%v|%v|%v|%v|\n", k.IntID(), rule.RxText, rule.TextReply, rule.Limit, rule.UserID)
		}

		buffer.WriteString(ruleText)
	}

	msg := tgbotapi.NewMessage(update.Message.Chat.ID, buffer.String())
	filter.bot.Send(msg)

	return nil
}

func updateUsers(ctx context.Context, update tgbotapi.Update) (err error) {
	userKey := datastore.NewKey(ctx, "User", update.Message.From.UserName, 0, nil)
	user := User{
		UserID:   update.Message.From.ID,
		UserName: update.Message.From.UserName,
	}
	err = datastore.RunInTransaction(ctx, func(ctx context.Context) error {
		var empty User
		if err := datastore.Get(ctx, userKey, &empty); err != datastore.ErrNoSuchEntity {
			return err
		}
		_, err := datastore.Put(ctx, userKey, &user)
		return err
	}, nil)

	return err
}

func getUserID(ctx context.Context, mention string) (int, error) {
	var user User
	key := datastore.NewKey(ctx, "User", mention, 0, nil)
	err := datastore.Get(ctx, key, &user)
	if err != nil {
		glog.Errorf(ctx, "Error fetching user: %v", err)
		return -1, err
	}

	return user.UserID, nil
}
<commit_msg>Fix bug: users without username can add rules<commit_after>package garvis

import (
	"bytes"
	"fmt"
	"regexp"
	"strconv"
	"strings"

	"golang.org/x/net/context"
	"google.golang.org/appengine/datastore"
	glog "google.golang.org/appengine/log"

	"github.com/go-telegram-bot-api/telegram-bot-api"
)

type TextFilter struct {
	bot  *tgbotapi.BotAPI
	ctx  context.Context
	done chan bool
}

type TextFilterRule struct {
	ChatID    int64
	RxText    string
	TextReply string
	Count     int
	Limit     int
	UserID    int
	Hidden    bool
	CreatorID int
}

type User struct {
	UserID   int
	UserName string
}

func (filter TextFilter) Run(update tgbotapi.Update) error {
	ctx := filter.ctx

	// It is not possible to match usernames to user ids from the API
	// for the case of mentions so we need to store username-ids in the database
	// for lookup
	updateUsers(ctx, update)

	query := datastore.NewQuery("Rule").Filter("ChatID = ", update.Message.Chat.ID)
	queryUserRules := query.Filter("UserID = ", update.Message.From.ID)
	queryStaticRules := query.Filter("UserID = ", 0)

	var userRules []TextFilterRule
	var staticRules []TextFilterRule
	userKeys, err := queryUserRules.GetAll(ctx, &userRules)
	staticKeys, err := queryStaticRules.GetAll(ctx, &staticRules)
	if err != nil {
		glog.Errorf(ctx, "client.GetAll: %v", err)
	}
	rules := append(userRules, staticRules...)
	keys := append(userKeys, staticKeys...)

	for i, rule := range rules {
		k := keys[i]

		rxRule := regexp.MustCompile(rule.RxText)

		if rxRule.MatchString(update.Message.Text) {
			rule.Count = rule.Count + 1
			if rule.Count >= rule.Limit {
				rule.Count = 0
				_, err = datastore.Put(ctx, k, &rule)
				if err != nil {
					glog.Errorf(ctx, "client.Put(reset): %v", err)
				}
				filter.Trigger(rule)
			}
			_, err = datastore.Put(ctx, k, &rule)
			if err != nil {
				glog.Errorf(ctx, "client.Put: %v", err)
			}
		}
	}

	filter.done <- true
	return nil
}

func (filter TextFilter) GetCommands(commands map[string]Filter) {
	commands["addrule"] = filter
	commands["deleterule"] = filter
	commands["listrules"] = filter
	commands["addhiddenrule"] = filter
}

func (filter TextFilter) RunCommand(cmd string, cmdarg CommandArguments) {
	update := cmdarg.update
	var err error
	switch cmd {
	case "addrule":
		err = filter.addRule(update, false)
	case "deleterule":
		err = filter.deleteRule(update)
	case "listrules":
		err = filter.listRules(update)
	case "addhiddenrule":
		err = filter.addRule(update, true)
	}
	if err != nil {
		glog.Errorf(filter.ctx, err.Error())
	}
}

func (filter TextFilter) Trigger(rule TextFilterRule) {
	msg := tgbotapi.NewMessage(rule.ChatID, rule.TextReply)
	filter.bot.Send(msg)
}

func (filter TextFilter) addRule(update tgbotapi.Update, hidden bool) (err error) {
	ctx := filter.ctx

	command := strings.SplitN(update.Message.Text, " ", 2)
	if len(command) < 2 {
		usage := "Usage: /addrule {regex matcher}~{reply}{#count (optional default: 1)}~{user (optional)}"
		msg := tgbotapi.NewMessage(update.Message.Chat.ID, usage)
		filter.bot.Send(msg)
		return nil
	}
	argstr := command[1]
	args := strings.SplitN(argstr, "~", 3)
	var userID int
	switch len(args) {
	case 2:
		userID = 0
	case 3:
		ents := update.Message.Entities
		for _, ent := range *ents {
			switch ent.Type {
			case "text_mention":
				userID = ent.User.ID
			case "mention":
				if userID, err = getUserID(ctx, args[2][1:]); err != nil {
					return err
				}
			}
		}
	default:
		return nil
	}
	arg1 := strings.SplitN(args[0], "#", 2)
	text := fmt.Sprintf("(?i)%v(?-i)", arg1[0])
	_, err = regexp.Compile(text)
	if err != nil {
		usage := "Invalid Regex"
		msg := tgbotapi.NewMessage(update.Message.Chat.ID, usage)
		filter.bot.Send(msg)
		return nil
	}
	var limit int
	if len(arg1) < 2 {
		limit = 1
	} else {
		limit, _ = strconv.Atoi(arg1[1])
	}
	textreply := args[1]

	keyl, _, _ := datastore.AllocateIDs(ctx, "Rule", nil, 1)
	ruleKey := datastore.NewKey(ctx, "Rule", "", keyl, nil)
	rule := TextFilterRule{
		ChatID:    update.Message.Chat.ID,
		RxText:    text,
		TextReply: textreply,
		Count:     0,
		Limit:     int(limit),
		UserID:    userID,
		CreatorID: update.Message.From.ID,
		Hidden:    hidden,
	}

	_, err = datastore.Put(ctx, ruleKey, &rule)

	return err
}

func (filter TextFilter) deleteRule(update tgbotapi.Update) (err error) {
	ctx := filter.ctx

	command := strings.SplitN(update.Message.Text, " ", 2)
	if len(command) < 2 {
		usage := "Usage: /deleterule {ruleID}"
		msg := tgbotapi.NewMessage(update.Message.Chat.ID, usage)
		filter.bot.Send(msg)
		return nil
	}
	key, _ := strconv.ParseInt(command[1], 10, 64)
	ruleKey := datastore.NewKey(ctx, "Rule", "", key, nil)

	userID := update.Message.From.ID

	err = datastore.RunInTransaction(ctx, func(ctx context.Context) error {
		var rule TextFilterRule
		err := datastore.Get(ctx, ruleKey, &rule)
		if err != nil {
			return err
		}

		if rule.ChatID == update.Message.Chat.ID || rule.CreatorID == userID {
			err = datastore.Delete(ctx, ruleKey)
		}

		return err
	}, nil)

	return err
}

func (filter TextFilter) listRules(update tgbotapi.Update) (err error) {
	ctx := filter.ctx

	var buffer bytes.Buffer
	var query *datastore.Query
	var header string

	if update.Message.Chat.Type == "private" {
		userID := update.Message.From.ID
		query = datastore.NewQuery("Rule").Filter("CreatorID = ", userID)
		header = fmt.Sprintf("|%s|%s|%s|%s|%s|%s|\n", "Chat name", "ID", "Regex", "Reply", "Count", "User(0 for all)")
	} else {
		query = datastore.NewQuery("Rule").Filter("ChatID = ", update.Message.Chat.ID)
		header = fmt.Sprintf("|%s|%s|%s|%s|%s|\n", "ID", "Regex", "Reply", "Count", "User(0 for all)")
	}
	buffer.WriteString(header)

	for t := query.Run(ctx); ; {
		var rule TextFilterRule
		var ruleText string

		k, err := t.Next(&rule)
		if err == datastore.Done {
			break
		}

		if update.Message.Chat.Type == "private" {
			chat, _ := filter.bot.GetChat(tgbotapi.ChatConfig{ChatID: rule.ChatID})
			ruleText = fmt.Sprintf("|%v|%v|%v|%v|%v|%v|\n", chat.Title, k.IntID(), rule.RxText, rule.TextReply, rule.Limit, rule.UserID)
		} else if !rule.Hidden {
			ruleText = fmt.Sprintf("|%v|%v|%v|%v|%v|\n", k.IntID(), rule.RxText, rule.TextReply, rule.Limit, rule.UserID)
		}

		buffer.WriteString(ruleText)
	}

	msg := tgbotapi.NewMessage(update.Message.Chat.ID, buffer.String())
	filter.bot.Send(msg)

	return nil
}

func updateUsers(ctx context.Context, update tgbotapi.Update) (err error) {
	userKey := datastore.NewKey(ctx, "User", update.Message.From.UserName, 0, nil)
	user := User{
		UserID:   update.Message.From.ID,
		UserName: update.Message.From.UserName,
	}
	err = datastore.RunInTransaction(ctx, func(ctx context.Context) error {
		var empty User
		if err := datastore.Get(ctx, userKey, &empty); err != datastore.ErrNoSuchEntity {
			return err
		}
		_, err := datastore.Put(ctx, userKey, &user)
		return err
	}, nil)

	return err
}

func getUserID(ctx context.Context, mention string) (int, error) {
	var user User
	key := datastore.NewKey(ctx, "User", mention, 0, nil)
	err := datastore.Get(ctx, key, &user)
	if err != nil {
		glog.Errorf(ctx, "Error fetching user: %v", err)
		return -1, err
	}

	return user.UserID, nil
}
<|endoftext|>


<commit_before>// package forwarder implements http handler that forwards requests to remote server
// and serves back the response
package forward

import (
	"io"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"time"

	"github.com/mailgun/oxy/utils"
)

// ReqRewriter can alter request headers and body
type ReqRewriter interface {
	Rewrite(r *http.Request)
}

type optSetter func(f *Forwarder) error

func PassHostHeader(bool) optSetter {
	return func(f *Forwarder) error {
		f.passHost = true
		return nil
	}
}

func RoundTripper(r http.RoundTripper) optSetter {
	return func(f *Forwarder) error {
		f.roundTripper = r
		return nil
	}
}

func Rewriter(r ReqRewriter) optSetter {
	return func(f *Forwarder) error {
		f.rewriter = r
		return nil
	}
}

// ErrorHandler is a functional argument that sets error handler of the server
func ErrorHandler(h utils.ErrorHandler) optSetter {
	return func(f *Forwarder) error {
		f.errHandler = h
		return nil
	}
}

func Logger(l utils.Logger) optSetter {
	return func(f *Forwarder) error {
		f.log = l
		return nil
	}
}

type Forwarder struct {
	errHandler   utils.ErrorHandler
	roundTripper http.RoundTripper
	rewriter     ReqRewriter
	log          utils.Logger
	passHost     bool
}

func New(setters ...optSetter) (*Forwarder, error) {
	f := &Forwarder{}
	for _, s := range setters {
		if err := s(f); err != nil {
			return nil, err
		}
	}
	if f.roundTripper == nil {
		f.roundTripper = http.DefaultTransport
	}
	if f.rewriter == nil {
		h, err := os.Hostname()
		if err != nil {
			h = "localhost"
		}
		f.rewriter = &HeaderRewriter{TrustForwardHeader: true, Hostname: h}
	}
	if f.log == nil {
		f.log = utils.NullLogger
	}
	if f.errHandler == nil {
		f.errHandler = utils.DefaultHandler
	}
	return f, nil
}

func (f *Forwarder) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	start := time.Now().UTC()
	response, err := f.roundTripper.RoundTrip(f.copyRequest(req, req.URL))
	if err != nil {
		f.log.Errorf("Error forwarding to %v, err: %v", req.URL, err)
		f.errHandler.ServeHTTP(w, req, err)
		return
	}

	if req.TLS != nil {
		f.log.Infof("Round trip: %v, code: %v, duration: %v tls:version: %x, tls:resume:%t, tls:csuite:%x, tls:server:%v",
			req.URL, response.StatusCode, time.Now().UTC().Sub(start),
			req.TLS.Version,
			req.TLS.DidResume,
			req.TLS.CipherSuite,
			req.TLS.ServerName)
	} else {
		f.log.Infof("Round trip: %v, code: %v, duration: %v",
			req.URL, response.StatusCode, time.Now().UTC().Sub(start))
	}

	utils.CopyHeaders(w.Header(), response.Header)
	w.WriteHeader(response.StatusCode)
	written, _ := io.Copy(w, response.Body)
	if written != 0 {
		w.Header().Set(ContentLength, strconv.FormatInt(written, 10))
	}
	response.Body.Close()
}

func (f *Forwarder) copyRequest(req *http.Request, u *url.URL) *http.Request {
	outReq := new(http.Request)
	*outReq = *req // includes shallow copies of maps, but we handle this below

	outReq.URL = utils.CopyURL(req.URL)
	outReq.URL.Scheme = u.Scheme
	outReq.URL.Host = u.Host
	outReq.URL.Opaque = req.RequestURI
	// raw query is already included in RequestURI, so ignore it to avoid dupes
	outReq.URL.RawQuery = ""
	// Do not pass client Host header unless optsetter PassHostHeader is set.
	if f.passHost != true {
		outReq.Host = u.Host
	}
	outReq.Proto = "HTTP/1.1"
	outReq.ProtoMajor = 1
	outReq.ProtoMinor = 1

	// Overwrite close flag so we can keep persistent connection for the backend servers
	outReq.Close = false

	outReq.Header = make(http.Header)
	utils.CopyHeaders(outReq.Header, req.Header)

	if f.rewriter != nil {
		f.rewriter.Rewrite(outReq)
	}
	return outReq
}
<commit_msg>Fix fwd requests starting with //<commit_after>// package forwarder implements http handler that forwards requests to remote server
// and serves back the response
package forward

import (
	"io"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"time"

	"github.com/mailgun/oxy/utils"
)

// ReqRewriter can alter request headers and body
type ReqRewriter interface {
	Rewrite(r *http.Request)
}

type optSetter func(f *Forwarder) error

func PassHostHeader(bool) optSetter {
	return func(f *Forwarder) error {
		f.passHost = true
		return nil
	}
}

func RoundTripper(r http.RoundTripper) optSetter {
	return func(f *Forwarder) error {
		f.roundTripper = r
		return nil
	}
}

func Rewriter(r ReqRewriter) optSetter {
	return func(f *Forwarder) error {
		f.rewriter = r
		return nil
	}
}

// ErrorHandler is a functional argument that sets error handler of the server
func ErrorHandler(h utils.ErrorHandler) optSetter {
	return func(f *Forwarder) error {
		f.errHandler = h
		return nil
	}
}

func Logger(l utils.Logger) optSetter {
	return func(f *Forwarder) error {
		f.log = l
		return nil
	}
}

type Forwarder struct {
	errHandler   utils.ErrorHandler
	roundTripper http.RoundTripper
	rewriter     ReqRewriter
	log          utils.Logger
	passHost     bool
}

func New(setters ...optSetter) (*Forwarder, error) {
	f := &Forwarder{}
	for _, s := range setters {
		if err := s(f); err != nil {
			return nil, err
		}
	}
	if f.roundTripper == nil {
		f.roundTripper = http.DefaultTransport
	}
	if f.rewriter == nil {
		h, err := os.Hostname()
		if err != nil {
			h = "localhost"
		}
		f.rewriter = &HeaderRewriter{TrustForwardHeader: true, Hostname: h}
	}
	if f.log == nil {
		f.log = utils.NullLogger
	}
	if f.errHandler == nil {
		f.errHandler = utils.DefaultHandler
	}
	return f, nil
}

func (f *Forwarder) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	start := time.Now().UTC()
	response, err := f.roundTripper.RoundTrip(f.copyRequest(req, req.URL))
	if err != nil {
		f.log.Errorf("Error forwarding to %v, err: %v", req.URL, err)
		f.errHandler.ServeHTTP(w, req, err)
		return
	}

	if req.TLS != nil {
		f.log.Infof("Round trip: %v, code: %v, duration: %v tls:version: %x, tls:resume:%t, tls:csuite:%x, tls:server:%v",
			req.URL, response.StatusCode, time.Now().UTC().Sub(start),
			req.TLS.Version,
			req.TLS.DidResume,
			req.TLS.CipherSuite,
			req.TLS.ServerName)
	} else {
		f.log.Infof("Round trip: %v, code: %v, duration: %v",
			req.URL, response.StatusCode, time.Now().UTC().Sub(start))
	}

	utils.CopyHeaders(w.Header(), response.Header)
	w.WriteHeader(response.StatusCode)
	written, _ := io.Copy(w, response.Body)
	if written != 0 {
		w.Header().Set(ContentLength, strconv.FormatInt(written, 10))
	}
	response.Body.Close()
}

func (f *Forwarder) copyRequest(req *http.Request, u *url.URL) *http.Request {
	outReq := new(http.Request)
	*outReq = *req // includes shallow copies of maps, but we handle this below

	outReq.URL = utils.CopyURL(req.URL)
	outReq.URL.Scheme = u.Scheme
	outReq.URL.Host = u.Host
	// workaround for https://github.com/golang/go/issues/10433
	outReq.URL.Opaque = mergeStartingSlashes(req.RequestURI)
	// raw query is already included in RequestURI, so ignore it to avoid dupes
	outReq.URL.RawQuery = ""
	// Do not pass client Host header unless optsetter PassHostHeader is set.
	if f.passHost != true {
		outReq.Host = u.Host
	}
	outReq.Proto = "HTTP/1.1"
	outReq.ProtoMajor = 1
	outReq.ProtoMinor = 1

	// Overwrite close flag so we can keep persistent connection for the backend servers
	outReq.Close = false

	outReq.Header = make(http.Header)
	utils.CopyHeaders(outReq.Header, req.Header)

	if f.rewriter != nil {
		f.rewriter.Rewrite(outReq)
	}
	return outReq
}

// faster than using strings or regexp packages
func mergeStartingSlashes(uri string) string {
	for len(uri) > 1 && uri[0] == '/' && uri[1] == '/' {
		uri = uri[1:]
	}
	return uri
}
<|endoftext|>


<commit_before>package github

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"strings"
	"time"

	"github.com/github/hub/ui"
	"github.com/github/hub/utils"
)

type verboseTransport struct {
	Transport   *http.Transport
	Verbose     bool
	OverrideURL *url.URL
	Out         io.Writer
	Colorized   bool
}

func (t *verboseTransport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
	if t.Verbose {
		t.dumpRequest(req)
	}

	if t.OverrideURL != nil {
		port := "80"
		if s := strings.Split(req.URL.Host, ":"); len(s) > 1 {
			port = s[1]
		}

		req = cloneRequest(req)
		req.Header.Set("X-Original-Scheme", req.URL.Scheme)
		req.Header.Set("X-Original-Port", port)
		req.URL.Scheme = t.OverrideURL.Scheme
		req.URL.Host = t.OverrideURL.Host
	}

	resp, err = t.Transport.RoundTrip(req)

	if err == nil && t.Verbose {
		t.dumpResponse(resp)
	}

	return
}

func (t *verboseTransport) dumpRequest(req *http.Request) {
	info := fmt.Sprintf("> %s %s://%s%s", req.Method, req.URL.Scheme, req.Host, req.URL.RequestURI())
	t.verbosePrintln(info)
	t.dumpHeaders(req.Header, ">")
	body := t.dumpBody(req.Body)
	if body != nil {
		// reset body since it's been read
		req.Body = body
	}
}

func (t *verboseTransport) dumpResponse(resp *http.Response) {
	info := fmt.Sprintf("< HTTP %d", resp.StatusCode)
	t.verbosePrintln(info)
	t.dumpHeaders(resp.Header, "<")
	body := t.dumpBody(resp.Body)
	if body != nil {
		// reset body since it's been read
		resp.Body = body
	}
}

func (t *verboseTransport) dumpHeaders(header http.Header, indent string) {
	dumpHeaders := []string{"Authorization", "X-GitHub-OTP", "Location"}
	for _, h := range dumpHeaders {
		v := header.Get(h)
		if v != "" {
			r := regexp.MustCompile("(?i)^(basic|token) (.+)")
			if r.MatchString(v) {
				v = r.ReplaceAllString(v, "$1 [REDACTED]")
			}

			info := fmt.Sprintf("%s %s: %s", indent, h, v)
			t.verbosePrintln(info)
		}
	}
}

func (t *verboseTransport) dumpBody(body io.ReadCloser) io.ReadCloser {
	if body == nil {
		return nil
	}

	defer body.Close()
	buf := new(bytes.Buffer)
	_, err := io.Copy(buf, body)
	utils.Check(err)

	if buf.Len() > 0 {
		t.verbosePrintln(buf.String())
	}

	return ioutil.NopCloser(buf)
}

func (t *verboseTransport) verbosePrintln(msg string) {
	if t.Colorized {
		msg = fmt.Sprintf("\033[36m%s\033[0m", msg)
	}

	fmt.Fprintln(t.Out, msg)
}

func newHttpClient(testHost string, verbose bool) *http.Client {
	var testURL *url.URL
	if testHost != "" {
		testURL, _ = url.Parse(testHost)
	}
	tr := &verboseTransport{
		Transport: &http.Transport{
			Proxy: proxyFromEnvironment,
			Dial: (&net.Dialer{
				Timeout:   30 * time.Second,
				KeepAlive: 30 * time.Second,
			}).Dial,
			TLSHandshakeTimeout: 10 * time.Second,
		},
		Verbose:     verbose,
		OverrideURL: testURL,
		Out:         ui.Stderr,
		Colorized:   ui.IsTerminal(os.Stderr),
	}
	return &http.Client{Transport: tr}
}

func cloneRequest(req *http.Request) *http.Request {
	dup := new(http.Request)
	*dup = *req
	dup.URL, _ = url.Parse(req.URL.String())
	dup.Header = make(http.Header)
	for k, s := range req.Header {
		dup.Header[k] = s
	}
	return dup
}

// An implementation of http.ProxyFromEnvironment that isn't broken
func proxyFromEnvironment(req *http.Request) (*url.URL, error) {
	proxy := os.Getenv("http_proxy")
	if proxy == "" {
		proxy = os.Getenv("HTTP_PROXY")
	}
	if proxy == "" {
		return nil, nil
	}

	proxyURL, err := url.Parse(proxy)
	if err != nil || !strings.HasPrefix(proxyURL.Scheme, "http") {
		if proxyURL, err := url.Parse("http://" + proxy); err == nil {
			return proxyURL, nil
		}
	}

	if err != nil {
		return nil, fmt.Errorf("invalid proxy address %q: %v", proxy, err)
	}

	return proxyURL, nil
}

type simpleClient struct {
	httpClient  *http.Client
	rootUrl     *url.URL
	accessToken string
}

func (c *simpleClient) performRequest(method, path string, body io.Reader, configure func(*http.Request)) (res *simpleResponse, err error) {
	url, err := url.Parse(path)
	if err != nil {
		return
	}

	url = c.rootUrl.ResolveReference(url)
	req, err := http.NewRequest(method, url.String(), body)
	if err != nil {
		return
	}
	req.Header.Set("Authorization", "token "+c.accessToken)
	req.Header.Set("User-Agent", UserAgent)
	if configure != nil {
		configure(req)
	}

	httpResponse, err := c.httpClient.Do(req)
	if err == nil {
		res = &simpleResponse{httpResponse}
	}

	return
}

func (c *simpleClient) jsonRequest(method, path string, body interface{}) (*simpleResponse, error) {
	json, err := json.Marshal(body)
	if err != nil {
		return nil, err
	}
	buf := bytes.NewBuffer(json)

	return c.performRequest(method, path, buf, func(req *http.Request) {
		req.Header.Set("Content-Type", "application/json")
	})
}

func (c *simpleClient) Get(path string) (*simpleResponse, error) {
	return c.performRequest("GET", path, nil, nil)
}

func (c *simpleClient) Delete(path string) (*simpleResponse, error) {
	return c.performRequest("DELETE", path, nil, nil)
}

func (c *simpleClient) PostJSON(path string, payload interface{}) (*simpleResponse, error) {
	return c.jsonRequest("POST", path, payload)
}

func (c *simpleClient) PatchJSON(path string, payload interface{}) (*simpleResponse, error) {
	return c.jsonRequest("PATCH", path, payload)
}

func (c *simpleClient) PostFile(path, filename string) (*simpleResponse, error) {
	stat, err := os.Stat(filename)
	if err != nil {
		return nil, err
	}

	file, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	return c.performRequest("POST", path, file, func(req *http.Request) {
		req.ContentLength = stat.Size()
		req.Header.Set("Content-Type", "application/octet-stream")
	})
}

type simpleResponse struct {
	*http.Response
}

type errorInfo struct {
	Message  string       `json:"message"`
	Errors   []fieldError `json:"errors"`
	Response *http.Response
}
type fieldError struct {
	Resource string `json:"resource"`
	Message  string `json:"message"`
	Code     string `json:"code"`
	Field    string `json:"field"`
}

func (e *errorInfo) Error() string {
	return e.Message
}

func (res *simpleResponse) Unmarshal(dest interface{}) (err error) {
	defer res.Body.Close()

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return
	}

	return json.Unmarshal(body, dest)
}

func (res *simpleResponse) ErrorInfo() (msg *errorInfo, err error) {
	defer res.Body.Close()

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return
	}

	msg = &errorInfo{}
	err = json.Unmarshal(body, msg)
	if err == nil {
		msg.Response = res.Response
	}

	return
}
<commit_msg>Set `charset` declaration with JSON payloads<commit_after>package github

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"strings"
	"time"

	"github.com/github/hub/ui"
	"github.com/github/hub/utils"
)

type verboseTransport struct {
	Transport   *http.Transport
	Verbose     bool
	OverrideURL *url.URL
	Out         io.Writer
	Colorized   bool
}

func (t *verboseTransport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
	if t.Verbose {
		t.dumpRequest(req)
	}

	if t.OverrideURL != nil {
		port := "80"
		if s := strings.Split(req.URL.Host, ":"); len(s) > 1 {
			port = s[1]
		}

		req = cloneRequest(req)
		req.Header.Set("X-Original-Scheme", req.URL.Scheme)
		req.Header.Set("X-Original-Port", port)
		req.URL.Scheme = t.OverrideURL.Scheme
		req.URL.Host = t.OverrideURL.Host
	}

	resp, err = t.Transport.RoundTrip(req)

	if err == nil && t.Verbose {
		t.dumpResponse(resp)
	}

	return
}

func (t *verboseTransport) dumpRequest(req *http.Request) {
	info := fmt.Sprintf("> %s %s://%s%s", req.Method, req.URL.Scheme, req.Host, req.URL.RequestURI())
	t.verbosePrintln(info)
	t.dumpHeaders(req.Header, ">")
	body := t.dumpBody(req.Body)
	if body != nil {
		// reset body since it's been read
		req.Body = body
	}
}

func (t *verboseTransport) dumpResponse(resp *http.Response) {
	info := fmt.Sprintf("< HTTP %d", resp.StatusCode)
	t.verbosePrintln(info)
	t.dumpHeaders(resp.Header, "<")
	body := t.dumpBody(resp.Body)
	if body != nil {
		// reset body since it's been read
		resp.Body = body
	}
}

func (t *verboseTransport) dumpHeaders(header http.Header, indent string) {
	dumpHeaders := []string{"Authorization", "X-GitHub-OTP", "Location"}
	for _, h := range dumpHeaders {
		v := header.Get(h)
		if v != "" {
			r := regexp.MustCompile("(?i)^(basic|token) (.+)")
			if r.MatchString(v) {
				v = r.ReplaceAllString(v, "$1 [REDACTED]")
			}

			info := fmt.Sprintf("%s %s: %s", indent, h, v)
			t.verbosePrintln(info)
		}
	}
}

func (t *verboseTransport) dumpBody(body io.ReadCloser) io.ReadCloser {
	if body == nil {
		return nil
	}

	defer body.Close()
	buf := new(bytes.Buffer)
	_, err := io.Copy(buf, body)
	utils.Check(err)

	if buf.Len() > 0 {
		t.verbosePrintln(buf.String())
	}

	return ioutil.NopCloser(buf)
}

func (t *verboseTransport) verbosePrintln(msg string) {
	if t.Colorized {
		msg = fmt.Sprintf("\033[36m%s\033[0m", msg)
	}

	fmt.Fprintln(t.Out, msg)
}

func newHttpClient(testHost string, verbose bool) *http.Client {
	var testURL *url.URL
	if testHost != "" {
		testURL, _ = url.Parse(testHost)
	}
	tr := &verboseTransport{
		Transport: &http.Transport{
			Proxy: proxyFromEnvironment,
			Dial: (&net.Dialer{
				Timeout:   30 * time.Second,
				KeepAlive: 30 * time.Second,
			}).Dial,
			TLSHandshakeTimeout: 10 * time.Second,
		},
		Verbose:     verbose,
		OverrideURL: testURL,
		Out:         ui.Stderr,
		Colorized:   ui.IsTerminal(os.Stderr),
	}
	return &http.Client{Transport: tr}
}

func cloneRequest(req *http.Request) *http.Request {
	dup := new(http.Request)
	*dup = *req
	dup.URL, _ = url.Parse(req.URL.String())
	dup.Header = make(http.Header)
	for k, s := range req.Header {
		dup.Header[k] = s
	}
	return dup
}

// An implementation of http.ProxyFromEnvironment that isn't broken
func proxyFromEnvironment(req *http.Request) (*url.URL, error) {
	proxy := os.Getenv("http_proxy")
	if proxy == "" {
		proxy = os.Getenv("HTTP_PROXY")
	}
	if proxy == "" {
		return nil, nil
	}

	proxyURL, err := url.Parse(proxy)
	if err != nil || !strings.HasPrefix(proxyURL.Scheme, "http") {
		if proxyURL, err := url.Parse("http://" + proxy); err == nil {
			return proxyURL, nil
		}
	}

	if err != nil {
		return nil, fmt.Errorf("invalid proxy address %q: %v", proxy, err)
	}

	return proxyURL, nil
}

type simpleClient struct {
	httpClient  *http.Client
	rootUrl     *url.URL
	accessToken string
}

func (c *simpleClient) performRequest(method, path string, body io.Reader, configure func(*http.Request)) (res *simpleResponse, err error) {
	url, err := url.Parse(path)
	if err != nil {
		return
	}

	url = c.rootUrl.ResolveReference(url)
	req, err := http.NewRequest(method, url.String(), body)
	if err != nil {
		return
	}
	req.Header.Set("Authorization", "token "+c.accessToken)
	req.Header.Set("User-Agent", UserAgent)
	if configure != nil {
		configure(req)
	}

	httpResponse, err := c.httpClient.Do(req)
	if err == nil {
		res = &simpleResponse{httpResponse}
	}

	return
}

func (c *simpleClient) jsonRequest(method, path string, body interface{}) (*simpleResponse, error) {
	json, err := json.Marshal(body)
	if err != nil {
		return nil, err
	}
	buf := bytes.NewBuffer(json)

	return c.performRequest(method, path, buf, func(req *http.Request) {
		req.Header.Set("Content-Type", "application/json; charset=utf-8")
	})
}

func (c *simpleClient) Get(path string) (*simpleResponse, error) {
	return c.performRequest("GET", path, nil, nil)
}

func (c *simpleClient) Delete(path string) (*simpleResponse, error) {
	return c.performRequest("DELETE", path, nil, nil)
}

func (c *simpleClient) PostJSON(path string, payload interface{}) (*simpleResponse, error) {
	return c.jsonRequest("POST", path, payload)
}

func (c *simpleClient) PatchJSON(path string, payload interface{}) (*simpleResponse, error) {
	return c.jsonRequest("PATCH", path, payload)
}

func (c *simpleClient) PostFile(path, filename string) (*simpleResponse, error) {
	stat, err := os.Stat(filename)
	if err != nil {
		return nil, err
	}

	file, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	return c.performRequest("POST", path, file, func(req *http.Request) {
		req.ContentLength = stat.Size()
		req.Header.Set("Content-Type", "application/octet-stream")
	})
}

type simpleResponse struct {
	*http.Response
}

type errorInfo struct {
	Message  string       `json:"message"`
	Errors   []fieldError `json:"errors"`
	Response *http.Response
}
type fieldError struct {
	Resource string `json:"resource"`
	Message  string `json:"message"`
	Code     string `json:"code"`
	Field    string `json:"field"`
}

func (e *errorInfo) Error() string {
	return e.Message
}

func (res *simpleResponse) Unmarshal(dest interface{}) (err error) {
	defer res.Body.Close()

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return
	}

	return json.Unmarshal(body, dest)
}

func (res *simpleResponse) ErrorInfo() (msg *errorInfo, err error) {
	defer res.Body.Close()

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return
	}

	msg = &errorInfo{}
	err = json.Unmarshal(body, msg)
	if err == nil {
		msg.Response = res.Response
	}

	return
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"log"
	"sort"
	"strings"

	"github.com/couchbaselabs/go-couchbase"
)

var ddocs = map[string]string{
	"jenkins": `{
		"views": {
			"by_build": {
				"map": "function (doc, meta) {emit([doc.build, doc.priority, doc.os], [doc.failCount, doc.totalCount]);}"
			}
		}
	}`,
}

type DataSource struct {
	CouchbaseAddress string
}

func (ds *DataSource) GetBucket(bucket string) *couchbase.Bucket {
	client, _ := couchbase.Connect(ds.CouchbaseAddress)
	pool, _ := client.GetPool("default")

	b, err := pool.GetBucket(bucket)
	if err != nil {
		log.Fatalf("Error reading bucket:  %v", err)
	}
	return b
}

func (ds *DataSource) QueryView(b *couchbase.Bucket, ddoc, view string,
	params map[string]interface{}) []couchbase.ViewRow {
	params["stale"] = "false"
	vr, err := b.View(ddoc, view, params)
	if err != nil {
		ds.installDDoc(ddoc)
	}
	return vr.Rows
}

func (ds *DataSource) installDDoc(ddoc string) {
	b := ds.GetBucket(ddoc) // bucket name == ddoc name
	err := b.PutDDoc(ddoc, ddocs[ddoc])
	if err != nil {
		log.Fatalf("%v", err)
	}
}

func (ds *DataSource) GetTimeline(abs bool) []byte {
	b := ds.GetBucket("jenkins")
	rows := ds.QueryView(b, "jenkins", "by_build", map[string]interface{}{})

	failed := map[string]float64{}
	total := map[string]float64{}
	builds := []string{}
	for i := range rows {
		build := rows[i].Key.([]interface{})[0].(string)
		failCount := rows[i].Value.([]interface{})[0].(float64)
		totalCount := rows[i].Value.([]interface{})[1].(float64)
		if _, ok := failed[build]; ok {
			failed[build] += failCount
		} else {
			failed[build] = failCount
		}
		if _, ok := total[build]; ok {
			total[build] += totalCount
		} else {
			total[build] = totalCount
		}
		builds = appendIfUnique(builds, build)
	}

	timeline := []map[string]interface{}{}
	passedValues := []interface{}{}
	failedValues := []interface{}{}
	sort.Strings(builds)
	if abs {
		for _, build := range builds {
			passedValues = append(passedValues, []interface{}{
				build,
				total[build] - failed[build],
			})
			failedValues = append(failedValues, []interface{}{
				build,
				-failed[build],
			})
		}
		timeline = append(timeline,
			map[string]interface{}{"key": "Passed", "values": passedValues})
		timeline = append(timeline,
			map[string]interface{}{"key": "Failed", "values": failedValues})
	} else {
		for _, build := range builds {
			passedValues = append(passedValues, []interface{}{
				build,
				100.0 * (total[build] - failed[build]) / total[build],
			})
			failedValues = append(failedValues, []interface{}{
				build,
				100.0 * failed[build] / total[build],
			})
		}
		timeline = append(timeline,
			map[string]interface{}{"key": "Passed, %", "values": passedValues})
		timeline = append(timeline,
			map[string]interface{}{"key": "Failed, %", "values": failedValues})
	}

	j, _ := json.Marshal(timeline)
	return j
}

func appendIfUnique(slice []string, s string) []string {
	for i := range slice {
		if slice[i] == s {
			return slice
		}
	}
	return append(slice, s)
}

func (ds *DataSource) GetBreakdown(build string, by_platform bool) []byte {
	b := ds.GetBucket("jenkins")
	params := map[string]interface{}{"startkey": []string{build}}
	rows := ds.QueryView(b, "jenkins", "by_build", params)

	keys := []string{}
	failed := map[string]float64{}
	total := map[string]float64{}
	for i := range rows {
		var key string
		if by_platform {
			key = rows[i].Key.([]interface{})[2].(string)
		} else {
			key = rows[i].Key.([]interface{})[1].(string)
		}
		if key == "N/A" {
			continue
		}

		failCount := rows[i].Value.([]interface{})[0].(float64)
		totalCount := rows[i].Value.([]interface{})[1].(float64)
		if _, ok := failed[key]; ok {
			failed[key] += failCount
		} else {
			failed[key] = failCount
		}
		if _, ok := total[key]; ok {
			total[key] += totalCount
		} else {
			total[key] = totalCount
		}
		keys = appendIfUnique(keys, key)
	}

	sort.Strings(keys)
	data := map[string]interface{}{}
	for _, key := range keys {
		title := strings.Title(strings.ToLower(key))
		data[title] = []map[string]interface{}{
			{"key": "Passed", "value": total[key] - failed[key]},
			{"key": "Failed", "value": failed[key]},
		}
	}

	j, _ := json.Marshal(data)
	return j
}
<commit_msg>safe type assertion<commit_after>package main

import (
	"encoding/json"
	"log"
	"sort"
	"strings"

	"github.com/couchbaselabs/go-couchbase"
)

var ddocs = map[string]string{
	"jenkins": `{
		"views": {
			"by_build": {
				"map": "function (doc, meta) {emit([doc.build, doc.priority, doc.os], [doc.failCount, doc.totalCount]);}"
			}
		}
	}`,
}

type DataSource struct {
	CouchbaseAddress string
}

func (ds *DataSource) GetBucket(bucket string) *couchbase.Bucket {
	client, _ := couchbase.Connect(ds.CouchbaseAddress)
	pool, _ := client.GetPool("default")

	b, err := pool.GetBucket(bucket)
	if err != nil {
		log.Fatalf("Error reading bucket:  %v", err)
	}
	return b
}

func (ds *DataSource) QueryView(b *couchbase.Bucket, ddoc, view string,
	params map[string]interface{}) []couchbase.ViewRow {
	params["stale"] = "false"
	vr, err := b.View(ddoc, view, params)
	if err != nil {
		ds.installDDoc(ddoc)
	}
	return vr.Rows
}

func (ds *DataSource) installDDoc(ddoc string) {
	b := ds.GetBucket(ddoc) // bucket name == ddoc name
	err := b.PutDDoc(ddoc, ddocs[ddoc])
	if err != nil {
		log.Fatalf("%v", err)
	}
}

func (ds *DataSource) GetTimeline(abs bool) []byte {
	b := ds.GetBucket("jenkins")
	rows := ds.QueryView(b, "jenkins", "by_build", map[string]interface{}{})

	failed := map[string]float64{}
	total := map[string]float64{}
	builds := []string{}
	for i := range rows {
		build := rows[i].Key.([]interface{})[0].(string)
		failCount, ok := rows[i].Value.([]interface{})[0].(float64)
		if !ok {
			continue
		}
		totalCount, ok := rows[i].Value.([]interface{})[1].(float64)
		if !ok {
			continue
		}
		if _, ok := failed[build]; ok {
			failed[build] += failCount
		} else {
			failed[build] = failCount
		}
		if _, ok := total[build]; ok {
			total[build] += totalCount
		} else {
			total[build] = totalCount
		}
		builds = appendIfUnique(builds, build)
	}

	timeline := []map[string]interface{}{}
	passedValues := []interface{}{}
	failedValues := []interface{}{}
	sort.Strings(builds)
	if abs {
		for _, build := range builds {
			passedValues = append(passedValues, []interface{}{
				build,
				total[build] - failed[build],
			})
			failedValues = append(failedValues, []interface{}{
				build,
				-failed[build],
			})
		}
		timeline = append(timeline,
			map[string]interface{}{"key": "Passed", "values": passedValues})
		timeline = append(timeline,
			map[string]interface{}{"key": "Failed", "values": failedValues})
	} else {
		for _, build := range builds {
			passedValues = append(passedValues, []interface{}{
				build,
				100.0 * (total[build] - failed[build]) / total[build],
			})
			failedValues = append(failedValues, []interface{}{
				build,
				100.0 * failed[build] / total[build],
			})
		}
		timeline = append(timeline,
			map[string]interface{}{"key": "Passed, %", "values": passedValues})
		timeline = append(timeline,
			map[string]interface{}{"key": "Failed, %", "values": failedValues})
	}

	j, _ := json.Marshal(timeline)
	return j
}

func appendIfUnique(slice []string, s string) []string {
	for i := range slice {
		if slice[i] == s {
			return slice
		}
	}
	return append(slice, s)
}

func (ds *DataSource) GetBreakdown(build string, by_platform bool) []byte {
	b := ds.GetBucket("jenkins")
	params := map[string]interface{}{"startkey": []string{build}}
	rows := ds.QueryView(b, "jenkins", "by_build", params)

	keys := []string{}
	failed := map[string]float64{}
	total := map[string]float64{}
	for i := range rows {
		var key string
		if by_platform {
			key = rows[i].Key.([]interface{})[2].(string)
		} else {
			key = rows[i].Key.([]interface{})[1].(string)
		}
		if key == "N/A" {
			continue
		}

		failCount, ok := rows[i].Value.([]interface{})[0].(float64)
		if !ok {
			continue
		}
		totalCount, ok := rows[i].Value.([]interface{})[1].(float64)
		if !ok {
			continue
		}
		if _, ok := failed[key]; ok {
			failed[key] += failCount
		} else {
			failed[key] = failCount
		}
		if _, ok := total[key]; ok {
			total[key] += totalCount
		} else {
			total[key] = totalCount
		}
		keys = appendIfUnique(keys, key)
	}

	sort.Strings(keys)
	data := map[string]interface{}{}
	for _, key := range keys {
		title := strings.Title(strings.ToLower(key))
		data[title] = []map[string]interface{}{
			{"key": "Passed", "value": total[key] - failed[key]},
			{"key": "Failed", "value": failed[key]},
		}
	}

	j, _ := json.Marshal(data)
	return j
}
<|endoftext|>


<commit_before>// Copyright ©2015 The Gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package path

import (
	"container/heap"

	"gonum.org/v1/gonum/graph"
	"gonum.org/v1/gonum/graph/traverse"
)

// DijkstraFrom returns a shortest-path tree for a shortest path from u to all nodes in
// the graph g. If the graph does not implement Weighted, UniformCost is used.
// DijkstraFrom will panic if g has a u-reachable negative edge weight.
//
// If g is a graph.Graph, all nodes of the graph will be stored in the shortest-path
// tree, otherwise only nodes reachable from u will be stored.
//
// The time complexity of DijkstrFrom is O(|E|.log|V|).
func DijkstraFrom(u graph.Node, g traverse.Graph) Shortest {
	var path Shortest
	if h, ok := g.(graph.Graph); ok {
		if h.Node(u.ID()) == nil {
			return Shortest{from: u}
		}
		path = newShortestFrom(u, graph.NodesOf(h.Nodes()))
	} else {
		if g.From(u.ID()) == nil {
			return Shortest{from: u}
		}
		path = newShortestFrom(u, []graph.Node{u})
	}

	var weight Weighting
	if wg, ok := g.(Weighted); ok {
		weight = wg.Weight
	} else {
		weight = UniformCost(g)
	}

	// Dijkstra's algorithm here is implemented essentially as
	// described in Function B.2 in figure 6 of UTCS Technical
	// Report TR-07-54.
	//
	// This implementation deviates from the report as follows:
	// - the value of path.dist for the start vertex u is initialized to 0;
	// - outdated elements from the priority queue (i.e. with respect to the dist value)
	//   are skipped.
	//
	// http://www.cs.utexas.edu/ftp/techreports/tr07-54.pdf
	Q := priorityQueue{{node: u, dist: 0}}
	for Q.Len() != 0 {
		mid := heap.Pop(&Q).(distanceNode)
		k := path.indexOf[mid.node.ID()]
		if mid.dist > path.dist[k] {
			continue
		}
		mnid := mid.node.ID()
		to := g.From(mnid)
		for to.Next() {
			v := to.Node()
			vid := v.ID()
			j, ok := path.indexOf[vid]
			if !ok {
				j = path.add(v)
			}
			w, ok := weight(mnid, vid)
			if !ok {
				panic("dijkstra: unexpected invalid weight")
			}
			if w < 0 {
				panic("dijkstra: negative edge weight")
			}
			joint := path.dist[k] + w
			if joint < path.dist[j] {
				heap.Push(&Q, distanceNode{node: v, dist: joint})
				path.set(j, joint, k)
			}
		}
	}

	return path
}

// DijkstraAllFrom returns a shortest-path tree for shortest paths from u to all nodes in
// the graph g. If the graph does not implement Weighted, UniformCost is used.
// DijkstraAllFrom will panic if g has a u-reachable negative edge weight.
//
// If g is a graph.Graph, all nodes of the graph will be stored in the shortest-path
// tree, otherwise only nodes reachable from u will be stored.
//
// The time complexity of DijkstrAllFrom is O(|E|.log|V|).
func DijkstraAllFrom(u graph.Node, g traverse.Graph) ShortestAlts {
	var path ShortestAlts
	if h, ok := g.(graph.Graph); ok {
		if h.Node(u.ID()) == nil {
			return ShortestAlts{from: u}
		}
		path = newShortestAltsFrom(u, graph.NodesOf(h.Nodes()))
	} else {
		if g.From(u.ID()) == nil {
			return ShortestAlts{from: u}
		}
		path = newShortestAltsFrom(u, []graph.Node{u})
	}

	var weight Weighting
	if wg, ok := g.(Weighted); ok {
		weight = wg.Weight
	} else {
		weight = UniformCost(g)
	}

	// Dijkstra's algorithm here is implemented essentially as
	// described in Function B.2 in figure 6 of UTCS Technical
	// Report TR-07-54.
	//
	// This implementation deviates from the report as follows:
	// - the value of path.dist for the start vertex u is initialized to 0;
	// - outdated elements from the priority queue (i.e. with respect to the dist value)
	//   are skipped.
	//
	// http://www.cs.utexas.edu/ftp/techreports/tr07-54.pdf
	Q := priorityQueue{{node: u, dist: 0}}
	for Q.Len() != 0 {
		mid := heap.Pop(&Q).(distanceNode)
		k := path.indexOf[mid.node.ID()]
		if mid.dist > path.dist[k] {
			continue
		}
		mnid := mid.node.ID()
		for _, v := range graph.NodesOf(g.From(mnid)) {
			vid := v.ID()
			j, ok := path.indexOf[vid]
			if !ok {
				j = path.add(v)
			}
			w, ok := weight(mnid, vid)
			if !ok {
				panic("dijkstra: unexpected invalid weight")
			}
			if w < 0 {
				panic("dijkstra: negative edge weight")
			}
			joint := path.dist[k] + w
			if joint < path.dist[j] {
				heap.Push(&Q, distanceNode{node: v, dist: joint})
				path.set(j, joint, k)
			} else if joint == path.dist[j] {
				path.addPath(j, k)
			}
		}
	}

	return path
}

// DijkstraAllPaths returns a shortest-path tree for shortest paths in the graph g.
// If the graph does not implement graph.Weighter, UniformCost is used.
// DijkstraAllPaths will panic if g has a negative edge weight.
//
// The time complexity of DijkstrAllPaths is O(|V|.|E|+|V|^2.log|V|).
func DijkstraAllPaths(g graph.Graph) (paths AllShortest) {
	paths = newAllShortest(graph.NodesOf(g.Nodes()), false)
	dijkstraAllPaths(g, paths)
	return paths
}

// dijkstraAllPaths is the all-paths implementation of Dijkstra. It is shared
// between DijkstraAllPaths and JohnsonAllPaths to avoid repeated allocation
// of the nodes slice and the indexOf map. It returns nothing, but stores the
// result of the work in the paths parameter which is a reference type.
func dijkstraAllPaths(g graph.Graph, paths AllShortest) {
	var weight Weighting
	if wg, ok := g.(graph.Weighted); ok {
		weight = wg.Weight
	} else {
		weight = UniformCost(g)
	}

	var Q priorityQueue
	for i, u := range paths.nodes {
		// Dijkstra's algorithm here is implemented essentially as
		// described in Function B.2 in figure 6 of UTCS Technical
		// Report TR-07-54 with the addition of handling multiple
		// co-equal paths.
		//
		// http://www.cs.utexas.edu/ftp/techreports/tr07-54.pdf

		// Q must be empty at this point.
		heap.Push(&Q, distanceNode{node: u, dist: 0})
		for Q.Len() != 0 {
			mid := heap.Pop(&Q).(distanceNode)
			k := paths.indexOf[mid.node.ID()]
			if mid.dist < paths.dist.At(i, k) {
				paths.dist.Set(i, k, mid.dist)
			}
			mnid := mid.node.ID()
			to := g.From(mnid)
			for to.Next() {
				v := to.Node()
				vid := v.ID()
				j := paths.indexOf[vid]
				w, ok := weight(mnid, vid)
				if !ok {
					panic("dijkstra: unexpected invalid weight")
				}
				if w < 0 {
					panic("dijkstra: negative edge weight")
				}
				joint := paths.dist.At(i, k) + w
				if joint < paths.dist.At(i, j) {
					heap.Push(&Q, distanceNode{node: v, dist: joint})
					paths.set(i, j, joint, k)
				} else if joint == paths.dist.At(i, j) {
					paths.add(i, j, k)
				}
			}
		}
	}
}

type distanceNode struct {
	node graph.Node
	dist float64
}

// priorityQueue implements a no-dec priority queue.
type priorityQueue []distanceNode

func (q priorityQueue) Len() int            { return len(q) }
func (q priorityQueue) Less(i, j int) bool  { return q[i].dist < q[j].dist }
func (q priorityQueue) Swap(i, j int)       { q[i], q[j] = q[j], q[i] }
func (q *priorityQueue) Push(n interface{}) { *q = append(*q, n.(distanceNode)) }
func (q *priorityQueue) Pop() interface{} {
	t := *q
	var n interface{}
	n, *q = t[len(t)-1], t[:len(t)-1]
	return n
}
<commit_msg>graph/path: fix empty iterator comparison<commit_after>// Copyright ©2015 The Gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package path

import (
	"container/heap"

	"gonum.org/v1/gonum/graph"
	"gonum.org/v1/gonum/graph/traverse"
)

// DijkstraFrom returns a shortest-path tree for a shortest path from u to all nodes in
// the graph g. If the graph does not implement Weighted, UniformCost is used.
// DijkstraFrom will panic if g has a u-reachable negative edge weight.
//
// If g is a graph.Graph, all nodes of the graph will be stored in the shortest-path
// tree, otherwise only nodes reachable from u will be stored.
//
// The time complexity of DijkstrFrom is O(|E|.log|V|).
func DijkstraFrom(u graph.Node, g traverse.Graph) Shortest {
	var path Shortest
	if h, ok := g.(graph.Graph); ok {
		if h.Node(u.ID()) == nil {
			return Shortest{from: u}
		}
		path = newShortestFrom(u, graph.NodesOf(h.Nodes()))
	} else {
		if g.From(u.ID()) == graph.Empty {
			return Shortest{from: u}
		}
		path = newShortestFrom(u, []graph.Node{u})
	}

	var weight Weighting
	if wg, ok := g.(Weighted); ok {
		weight = wg.Weight
	} else {
		weight = UniformCost(g)
	}

	// Dijkstra's algorithm here is implemented essentially as
	// described in Function B.2 in figure 6 of UTCS Technical
	// Report TR-07-54.
	//
	// This implementation deviates from the report as follows:
	// - the value of path.dist for the start vertex u is initialized to 0;
	// - outdated elements from the priority queue (i.e. with respect to the dist value)
	//   are skipped.
	//
	// http://www.cs.utexas.edu/ftp/techreports/tr07-54.pdf
	Q := priorityQueue{{node: u, dist: 0}}
	for Q.Len() != 0 {
		mid := heap.Pop(&Q).(distanceNode)
		k := path.indexOf[mid.node.ID()]
		if mid.dist > path.dist[k] {
			continue
		}
		mnid := mid.node.ID()
		to := g.From(mnid)
		for to.Next() {
			v := to.Node()
			vid := v.ID()
			j, ok := path.indexOf[vid]
			if !ok {
				j = path.add(v)
			}
			w, ok := weight(mnid, vid)
			if !ok {
				panic("dijkstra: unexpected invalid weight")
			}
			if w < 0 {
				panic("dijkstra: negative edge weight")
			}
			joint := path.dist[k] + w
			if joint < path.dist[j] {
				heap.Push(&Q, distanceNode{node: v, dist: joint})
				path.set(j, joint, k)
			}
		}
	}

	return path
}

// DijkstraAllFrom returns a shortest-path tree for shortest paths from u to all nodes in
// the graph g. If the graph does not implement Weighted, UniformCost is used.
// DijkstraAllFrom will panic if g has a u-reachable negative edge weight.
//
// If g is a graph.Graph, all nodes of the graph will be stored in the shortest-path
// tree, otherwise only nodes reachable from u will be stored.
//
// The time complexity of DijkstrAllFrom is O(|E|.log|V|).
func DijkstraAllFrom(u graph.Node, g traverse.Graph) ShortestAlts {
	var path ShortestAlts
	if h, ok := g.(graph.Graph); ok {
		if h.Node(u.ID()) == nil {
			return ShortestAlts{from: u}
		}
		path = newShortestAltsFrom(u, graph.NodesOf(h.Nodes()))
	} else {
		if g.From(u.ID()) == graph.Empty {
			return ShortestAlts{from: u}
		}
		path = newShortestAltsFrom(u, []graph.Node{u})
	}

	var weight Weighting
	if wg, ok := g.(Weighted); ok {
		weight = wg.Weight
	} else {
		weight = UniformCost(g)
	}

	// Dijkstra's algorithm here is implemented essentially as
	// described in Function B.2 in figure 6 of UTCS Technical
	// Report TR-07-54.
	//
	// This implementation deviates from the report as follows:
	// - the value of path.dist for the start vertex u is initialized to 0;
	// - outdated elements from the priority queue (i.e. with respect to the dist value)
	//   are skipped.
	//
	// http://www.cs.utexas.edu/ftp/techreports/tr07-54.pdf
	Q := priorityQueue{{node: u, dist: 0}}
	for Q.Len() != 0 {
		mid := heap.Pop(&Q).(distanceNode)
		k := path.indexOf[mid.node.ID()]
		if mid.dist > path.dist[k] {
			continue
		}
		mnid := mid.node.ID()
		for _, v := range graph.NodesOf(g.From(mnid)) {
			vid := v.ID()
			j, ok := path.indexOf[vid]
			if !ok {
				j = path.add(v)
			}
			w, ok := weight(mnid, vid)
			if !ok {
				panic("dijkstra: unexpected invalid weight")
			}
			if w < 0 {
				panic("dijkstra: negative edge weight")
			}
			joint := path.dist[k] + w
			if joint < path.dist[j] {
				heap.Push(&Q, distanceNode{node: v, dist: joint})
				path.set(j, joint, k)
			} else if joint == path.dist[j] {
				path.addPath(j, k)
			}
		}
	}

	return path
}

// DijkstraAllPaths returns a shortest-path tree for shortest paths in the graph g.
// If the graph does not implement graph.Weighter, UniformCost is used.
// DijkstraAllPaths will panic if g has a negative edge weight.
//
// The time complexity of DijkstrAllPaths is O(|V|.|E|+|V|^2.log|V|).
func DijkstraAllPaths(g graph.Graph) (paths AllShortest) {
	paths = newAllShortest(graph.NodesOf(g.Nodes()), false)
	dijkstraAllPaths(g, paths)
	return paths
}

// dijkstraAllPaths is the all-paths implementation of Dijkstra. It is shared
// between DijkstraAllPaths and JohnsonAllPaths to avoid repeated allocation
// of the nodes slice and the indexOf map. It returns nothing, but stores the
// result of the work in the paths parameter which is a reference type.
func dijkstraAllPaths(g graph.Graph, paths AllShortest) {
	var weight Weighting
	if wg, ok := g.(graph.Weighted); ok {
		weight = wg.Weight
	} else {
		weight = UniformCost(g)
	}

	var Q priorityQueue
	for i, u := range paths.nodes {
		// Dijkstra's algorithm here is implemented essentially as
		// described in Function B.2 in figure 6 of UTCS Technical
		// Report TR-07-54 with the addition of handling multiple
		// co-equal paths.
		//
		// http://www.cs.utexas.edu/ftp/techreports/tr07-54.pdf

		// Q must be empty at this point.
		heap.Push(&Q, distanceNode{node: u, dist: 0})
		for Q.Len() != 0 {
			mid := heap.Pop(&Q).(distanceNode)
			k := paths.indexOf[mid.node.ID()]
			if mid.dist < paths.dist.At(i, k) {
				paths.dist.Set(i, k, mid.dist)
			}
			mnid := mid.node.ID()
			to := g.From(mnid)
			for to.Next() {
				v := to.Node()
				vid := v.ID()
				j := paths.indexOf[vid]
				w, ok := weight(mnid, vid)
				if !ok {
					panic("dijkstra: unexpected invalid weight")
				}
				if w < 0 {
					panic("dijkstra: negative edge weight")
				}
				joint := paths.dist.At(i, k) + w
				if joint < paths.dist.At(i, j) {
					heap.Push(&Q, distanceNode{node: v, dist: joint})
					paths.set(i, j, joint, k)
				} else if joint == paths.dist.At(i, j) {
					paths.add(i, j, k)
				}
			}
		}
	}
}

type distanceNode struct {
	node graph.Node
	dist float64
}

// priorityQueue implements a no-dec priority queue.
type priorityQueue []distanceNode

func (q priorityQueue) Len() int            { return len(q) }
func (q priorityQueue) Less(i, j int) bool  { return q[i].dist < q[j].dist }
func (q priorityQueue) Swap(i, j int)       { q[i], q[j] = q[j], q[i] }
func (q *priorityQueue) Push(n interface{}) { *q = append(*q, n.(distanceNode)) }
func (q *priorityQueue) Pop() interface{} {
	t := *q
	var n interface{}
	n, *q = t[len(t)-1], t[:len(t)-1]
	return n
}
<|endoftext|>


<commit_before>// Copyright 2014 ISRG.  All rights reserved
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package main

import (
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/letsencrypt/boulder"
	"github.com/streadway/amqp"
	"net/http"
	"os"
)

// Exit and print error message if we encountered a problem
func failOnError(err error, msg string) {
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s: %s\n", msg, err)
		os.Exit(1)
	}
}

// This is the same as amqpConnect in boulder, but with even
// more aggressive error dropping
func amqpChannel(url string) (ch *amqp.Channel) {
	conn, err := amqp.Dial(url)
	failOnError(err, "Unable to connect to AMQP server")

	ch, err = conn.Channel()
	failOnError(err, "Unable to establish channel to AMQP server")
	return
}

// Start the server and wait around
func runForever(server *boulder.AmqpRpcServer) {
	forever := make(chan bool)
	server.Start()
	fmt.Fprintf(os.Stderr, "Server running...\n")
	<-forever
}

func main() {
	app := cli.NewApp()
	app.Name = "boulder-start"
	app.Usage = "Command-line utility to start Boulder's servers in stand-alone mode"
	app.Version = "0.0.0"

	// Specify AMQP Server
	app.Flags = []cli.Flag{
		cli.StringFlag{
				Name: "amqp",
				Value: "amqp://guest:guest@localhost:5672",
				Usage: "AMQP Broker String",
			},
	}

	// One command per element of the system
	// * WebFrontEnd
	// * RegistrationAuthority
	// * ValidationAuthority
	// * CertificateAuthority
	// * StorageAuthority
	//
	// Once started, we just run until killed
	//
	// AMQP queue names are hard-coded for now
	app.Commands = []cli.Command{
		{
			Name:  "monolithic",
			Usage: "Start the CA in monolithic mode, without using AMQP",
			Action: func(c *cli.Context) {
				// Create the components
				wfe := boulder.NewWebFrontEndImpl()
				sa := boulder.NewSimpleStorageAuthorityImpl()
				ra := boulder.NewRegistrationAuthorityImpl()
				va := boulder.NewValidationAuthorityImpl()
				ca, err := boulder.NewCertificateAuthorityImpl()
				failOnError(err, "Unable to create CA")

				// Wire them up
				wfe.RA = &ra
				wfe.SA = &sa
				ra.CA = &ca
				ra.SA = &sa
				ra.VA = &va
				va.RA = &ra

				// Go!
				authority := "localhost:4000"
				authzPath := "/acme/authz/"
				certPath := "/acme/cert/"
				wfe.SetAuthzBase("http://" + authority + authzPath)
				wfe.SetCertBase("http://" + authority + certPath)
				http.HandleFunc("/acme/new-authz", wfe.NewAuthz)
				http.HandleFunc("/acme/new-cert", wfe.NewCert)
				http.HandleFunc("/acme/authz/", wfe.Authz)
				http.HandleFunc("/acme/cert/", wfe.Cert)

				fmt.Fprintf(os.Stderr, "Server running...\n")
				err = http.ListenAndServe(authority, nil)
				failOnError(err, "Error starting HTTP server")
			},
		},
		{
			Name:  "monolithic-amqp",
			Usage: "Start the CA in monolithic mode, using AMQP",
			Action: func(c *cli.Context) {
				// Create an AMQP channel
				ch := amqpChannel(c.GlobalString("amqp"))

				// Create AMQP-RPC clients for CA, VA, RA, SA
				cac, err := boulder.NewCertificateAuthorityClient("CA.client", "CA.server", ch)
				failOnError(err, "Failed to create CA client")
				vac, err := boulder.NewValidationAuthorityClient("VA.client", "VA.server", ch)
				failOnError(err, "Failed to create VA client")
				rac, err := boulder.NewRegistrationAuthorityClient("RA.client", "RA.server", ch)
				failOnError(err, "Failed to create RA client")
				sac, err := boulder.NewStorageAuthorityClient("SA.client", "SA.server", ch)
				failOnError(err, "Failed to create SA client")

				// ... and corresponding servers
				// (We need this order so that we can give the servers
				//  references to the clients)
				cas, err := boulder.NewCertificateAuthorityServer("CA.server", ch)
				failOnError(err, "Failed to create CA server")
				vas, err := boulder.NewValidationAuthorityServer("VA.server", ch, &rac)
				failOnError(err, "Failed to create VA server")
				ras, err := boulder.NewRegistrationAuthorityServer("RA.server", ch, &vac, &cac, &sac)
				failOnError(err, "Failed to create RA server")
				sas := boulder.NewStorageAuthorityServer("SA.server", ch)

				// Start the servers
				cas.Start()
				vas.Start()
				ras.Start()
				sas.Start()

				// Wire up the front end (wrappers are already wired)
				wfe := boulder.NewWebFrontEndImpl()
				wfe.RA = &rac
				wfe.SA = &sac

				// Go!
				authority := "localhost:4000"
				authzPath := "/acme/authz/"
				certPath := "/acme/cert/"
				wfe.SetAuthzBase("http://" + authority + authzPath)
				wfe.SetCertBase("http://" + authority + certPath)
				http.HandleFunc("/acme/new-authz", wfe.NewAuthz)
				http.HandleFunc("/acme/new-cert", wfe.NewCert)
				http.HandleFunc("/acme/authz/", wfe.Authz)
				http.HandleFunc("/acme/cert/", wfe.Cert)

				fmt.Fprintf(os.Stderr, "Server running...\n")
				err = http.ListenAndServe(authority, nil)
				failOnError(err, "Error starting HTTP server")
			},
		},
		{
			Name:  "wfe",
			Usage: "Start the WebFrontEnd",
			Action: func(c *cli.Context) {
				// Create necessary clients
				ch := amqpChannel(c.GlobalString("amqp"))

				rac, err := boulder.NewRegistrationAuthorityClient("RA.client", "RA.server", ch)
				failOnError(err, "Unable to create RA client")

				sac, err := boulder.NewStorageAuthorityClient("SA.client", "SA.server", ch)
				failOnError(err, "Unable to create SA client")

				// Create the front-end and wire in its resources
				wfe := boulder.NewWebFrontEndImpl()
				wfe.RA = &rac
				wfe.SA = &sac

				// Connect the front end to HTTP
				authority := "localhost:4000"
				authzPath := "/acme/authz/"
				certPath := "/acme/cert/"
				wfe.SetAuthzBase("http://" + authority + authzPath)
				wfe.SetCertBase("http://" + authority + certPath)
				http.HandleFunc("/acme/new-authz", wfe.NewAuthz)
				http.HandleFunc("/acme/new-cert", wfe.NewCert)
				http.HandleFunc("/acme/authz/", wfe.Authz)
				http.HandleFunc("/acme/cert/", wfe.Cert)

				fmt.Fprintf(os.Stderr, "Server running...\n")
				http.ListenAndServe(authority, nil)
			},
		},
		{
			Name:  "ca",
			Usage: "Start the CertificateAuthority",
			Action: func(c *cli.Context) {
				ch := amqpChannel(c.GlobalString("amqp"))

				cas, err := boulder.NewCertificateAuthorityServer("CA.server", ch)
				failOnError(err, "Unable to create CA server")
				runForever(cas)
			},
		},
		{
			Name:  "sa",
			Usage: "Start the StorageAuthority",
			Action: func(c *cli.Context) {
				ch := amqpChannel(c.GlobalString("amqp"))

				sas := boulder.NewStorageAuthorityServer("SA.server", ch)
				runForever(sas)
			},
		},
		{
			Name:  "va",
			Usage: "Start the ValidationAuthority",
			Action: func(c *cli.Context) {
				ch := amqpChannel(c.GlobalString("amqp"))

				rac, err := boulder.NewRegistrationAuthorityClient("RA.client", "RA.server", ch)
				failOnError(err, "Unable to create RA client")

				vas, err := boulder.NewValidationAuthorityServer("VA.server", ch, &rac)
				failOnError(err, "Unable to create VA server")
				runForever(vas)
			},
		},
		{
			Name:  "ra",
			Usage: "Start the RegistrationAuthority",
			Action: func(c *cli.Context) {
				// TODO
				ch := amqpChannel(c.GlobalString("amqp"))

				vac, err := boulder.NewValidationAuthorityClient("VA.client", "VA.server", ch)
				failOnError(err, "Unable to create VA client")

				cac, err := boulder.NewCertificateAuthorityClient("CA.client", "CA.server", ch)
				failOnError(err, "Unable to create CA client")

				sac, err := boulder.NewStorageAuthorityClient("SA.client", "SA.server", ch)
				failOnError(err, "Unable to create SA client")

				ras, err := boulder.NewRegistrationAuthorityServer("RA.server", ch, &vac, &cac, &sac)
				failOnError(err, "Unable to create RA server")
				runForever(ras)
			},
		},
	}

	err := app.Run(os.Args)
	failOnError(err, "Failed to run application")
}
<commit_msg>Change the defaults to open ports bound to all interfaces, not loopback.<commit_after>// Copyright 2014 ISRG.  All rights reserved
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package main

import (
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/letsencrypt/boulder"
	"github.com/streadway/amqp"
	"net/http"
	"os"
)

// Exit and print error message if we encountered a problem
func failOnError(err error, msg string) {
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s: %s\n", msg, err)
		os.Exit(1)
	}
}

// This is the same as amqpConnect in boulder, but with even
// more aggressive error dropping
func amqpChannel(url string) (ch *amqp.Channel) {
	conn, err := amqp.Dial(url)
	failOnError(err, "Unable to connect to AMQP server")

	ch, err = conn.Channel()
	failOnError(err, "Unable to establish channel to AMQP server")
	return
}

// Start the server and wait around
func runForever(server *boulder.AmqpRpcServer) {
	forever := make(chan bool)
	server.Start()
	fmt.Fprintf(os.Stderr, "Server running...\n")
	<-forever
}

func main() {
	app := cli.NewApp()
	app.Name = "boulder-start"
	app.Usage = "Command-line utility to start Boulder's servers in stand-alone mode"
	app.Version = "0.0.0"

	// Specify AMQP Server
	app.Flags = []cli.Flag{
		cli.StringFlag{
				Name: "amqp",
				Value: "amqp://guest:guest@localhost:5672",
				Usage: "AMQP Broker String",
			},
	}

	// One command per element of the system
	// * WebFrontEnd
	// * RegistrationAuthority
	// * ValidationAuthority
	// * CertificateAuthority
	// * StorageAuthority
	//
	// Once started, we just run until killed
	//
	// AMQP queue names are hard-coded for now
	app.Commands = []cli.Command{
		{
			Name:  "monolithic",
			Usage: "Start the CA in monolithic mode, without using AMQP",
			Action: func(c *cli.Context) {
				// Create the components
				wfe := boulder.NewWebFrontEndImpl()
				sa := boulder.NewSimpleStorageAuthorityImpl()
				ra := boulder.NewRegistrationAuthorityImpl()
				va := boulder.NewValidationAuthorityImpl()
				ca, err := boulder.NewCertificateAuthorityImpl()
				failOnError(err, "Unable to create CA")

				// Wire them up
				wfe.RA = &ra
				wfe.SA = &sa
				ra.CA = &ca
				ra.SA = &sa
				ra.VA = &va
				va.RA = &ra

				// Go!
				authority := "0.0.0.0:4000"
				authzPath := "/acme/authz/"
				certPath := "/acme/cert/"
				wfe.SetAuthzBase("http://" + authority + authzPath)
				wfe.SetCertBase("http://" + authority + certPath)
				http.HandleFunc("/acme/new-authz", wfe.NewAuthz)
				http.HandleFunc("/acme/new-cert", wfe.NewCert)
				http.HandleFunc("/acme/authz/", wfe.Authz)
				http.HandleFunc("/acme/cert/", wfe.Cert)

				fmt.Fprintf(os.Stderr, "Server running...\n")
				err = http.ListenAndServe(authority, nil)
				failOnError(err, "Error starting HTTP server")
			},
		},
		{
			Name:  "monolithic-amqp",
			Usage: "Start the CA in monolithic mode, using AMQP",
			Action: func(c *cli.Context) {
				// Create an AMQP channel
				ch := amqpChannel(c.GlobalString("amqp"))

				// Create AMQP-RPC clients for CA, VA, RA, SA
				cac, err := boulder.NewCertificateAuthorityClient("CA.client", "CA.server", ch)
				failOnError(err, "Failed to create CA client")
				vac, err := boulder.NewValidationAuthorityClient("VA.client", "VA.server", ch)
				failOnError(err, "Failed to create VA client")
				rac, err := boulder.NewRegistrationAuthorityClient("RA.client", "RA.server", ch)
				failOnError(err, "Failed to create RA client")
				sac, err := boulder.NewStorageAuthorityClient("SA.client", "SA.server", ch)
				failOnError(err, "Failed to create SA client")

				// ... and corresponding servers
				// (We need this order so that we can give the servers
				//  references to the clients)
				cas, err := boulder.NewCertificateAuthorityServer("CA.server", ch)
				failOnError(err, "Failed to create CA server")
				vas, err := boulder.NewValidationAuthorityServer("VA.server", ch, &rac)
				failOnError(err, "Failed to create VA server")
				ras, err := boulder.NewRegistrationAuthorityServer("RA.server", ch, &vac, &cac, &sac)
				failOnError(err, "Failed to create RA server")
				sas := boulder.NewStorageAuthorityServer("SA.server", ch)

				// Start the servers
				cas.Start()
				vas.Start()
				ras.Start()
				sas.Start()

				// Wire up the front end (wrappers are already wired)
				wfe := boulder.NewWebFrontEndImpl()
				wfe.RA = &rac
				wfe.SA = &sac

				// Go!
				authority := "0.0.0.0:4000"
				authzPath := "/acme/authz/"
				certPath := "/acme/cert/"
				wfe.SetAuthzBase("http://" + authority + authzPath)
				wfe.SetCertBase("http://" + authority + certPath)
				http.HandleFunc("/acme/new-authz", wfe.NewAuthz)
				http.HandleFunc("/acme/new-cert", wfe.NewCert)
				http.HandleFunc("/acme/authz/", wfe.Authz)
				http.HandleFunc("/acme/cert/", wfe.Cert)

				fmt.Fprintf(os.Stderr, "Server running...\n")
				err = http.ListenAndServe(authority, nil)
				failOnError(err, "Error starting HTTP server")
			},
		},
		{
			Name:  "wfe",
			Usage: "Start the WebFrontEnd",
			Action: func(c *cli.Context) {
				// Create necessary clients
				ch := amqpChannel(c.GlobalString("amqp"))

				rac, err := boulder.NewRegistrationAuthorityClient("RA.client", "RA.server", ch)
				failOnError(err, "Unable to create RA client")

				sac, err := boulder.NewStorageAuthorityClient("SA.client", "SA.server", ch)
				failOnError(err, "Unable to create SA client")

				// Create the front-end and wire in its resources
				wfe := boulder.NewWebFrontEndImpl()
				wfe.RA = &rac
				wfe.SA = &sac

				// Connect the front end to HTTP
				authority := "0.0.0.0:4000"
				authzPath := "/acme/authz/"
				certPath := "/acme/cert/"
				wfe.SetAuthzBase("http://" + authority + authzPath)
				wfe.SetCertBase("http://" + authority + certPath)
				http.HandleFunc("/acme/new-authz", wfe.NewAuthz)
				http.HandleFunc("/acme/new-cert", wfe.NewCert)
				http.HandleFunc("/acme/authz/", wfe.Authz)
				http.HandleFunc("/acme/cert/", wfe.Cert)

				fmt.Fprintf(os.Stderr, "Server running...\n")
				http.ListenAndServe(authority, nil)
			},
		},
		{
			Name:  "ca",
			Usage: "Start the CertificateAuthority",
			Action: func(c *cli.Context) {
				ch := amqpChannel(c.GlobalString("amqp"))

				cas, err := boulder.NewCertificateAuthorityServer("CA.server", ch)
				failOnError(err, "Unable to create CA server")
				runForever(cas)
			},
		},
		{
			Name:  "sa",
			Usage: "Start the StorageAuthority",
			Action: func(c *cli.Context) {
				ch := amqpChannel(c.GlobalString("amqp"))

				sas := boulder.NewStorageAuthorityServer("SA.server", ch)
				runForever(sas)
			},
		},
		{
			Name:  "va",
			Usage: "Start the ValidationAuthority",
			Action: func(c *cli.Context) {
				ch := amqpChannel(c.GlobalString("amqp"))

				rac, err := boulder.NewRegistrationAuthorityClient("RA.client", "RA.server", ch)
				failOnError(err, "Unable to create RA client")

				vas, err := boulder.NewValidationAuthorityServer("VA.server", ch, &rac)
				failOnError(err, "Unable to create VA server")
				runForever(vas)
			},
		},
		{
			Name:  "ra",
			Usage: "Start the RegistrationAuthority",
			Action: func(c *cli.Context) {
				// TODO
				ch := amqpChannel(c.GlobalString("amqp"))

				vac, err := boulder.NewValidationAuthorityClient("VA.client", "VA.server", ch)
				failOnError(err, "Unable to create VA client")

				cac, err := boulder.NewCertificateAuthorityClient("CA.client", "CA.server", ch)
				failOnError(err, "Unable to create CA client")

				sac, err := boulder.NewStorageAuthorityClient("SA.client", "SA.server", ch)
				failOnError(err, "Unable to create SA client")

				ras, err := boulder.NewRegistrationAuthorityServer("RA.server", ch, &vac, &cac, &sac)
				failOnError(err, "Unable to create RA server")
				runForever(ras)
			},
		},
	}

	err := app.Run(os.Args)
	failOnError(err, "Failed to run application")
}
<|endoftext|>


<commit_before>package main

import (
	"flag"

	"github.com/xshellinc/iotit/device"

	"fmt"
)

const ProgName = "iotit"

const helpInfo = `
NAME:
   iotit - Flashing Tool for iot devices used by Isaax Cloud

USAGE:
   iotit [global options]

GLOBAL OPTIONS:
   -dev [device-type] executes iotit with specified deviceType
   -help, -h          show help
   -version, -v       print the version
`

var Version string

func main() {
	deviceType := flag.String("dev", "", "-dev=[device-type]")
	h1 := flag.Bool("help", false, helpInfo)
	h2 := flag.Bool("h", false, helpInfo)

	v1 := flag.Bool("version", false, helpInfo)
	v2 := flag.Bool("v", false, helpInfo)

	flag.Parse()

	if *h1 || *h2 {
		fmt.Println(helpInfo)
		return
	}

	if *v1 || *v2 {
		if Version == "" {
			Version = "-1 not set"
		}
		fmt.Println(ProgName, Version)
		return
	}

	device.DeviceInit(*deviceType)
}
<commit_msg>Log to file, failover to stdout<commit_after>package main

import (
	"flag"

	"github.com/xshellinc/iotit/device"

	"fmt"
	"os"

	"github.com/Sirupsen/logrus"
)

const ProgName = "iotit"

const helpInfo = `
NAME:
   iotit - Flashing Tool for iot devices used by Isaax Cloud

USAGE:
   iotit [global options]

GLOBAL OPTIONS:
   -dev [device-type] executes iotit with specified deviceType
   -help, -h          show help
   -version, -v       print the version
`

var Version string

func init() {
	f, err := os.OpenFile(fmt.Sprintf("/tmp/%s.log", ProgName), os.O_APPEND|os.O_CREATE|os.O_RDWR, 0666)
	logrus.SetLevel(logrus.DebugLevel)
	if err != nil {
		logrus.Error("error opening file: %v", err)
		return
	}
	logrus.SetOutput(f)
}

func main() {
	deviceType := flag.String("dev", "", "-dev=[device-type]")
	h1 := flag.Bool("help", false, helpInfo)
	h2 := flag.Bool("h", false, helpInfo)

	v1 := flag.Bool("version", false, helpInfo)
	v2 := flag.Bool("v", false, helpInfo)

	flag.Parse()

	if *h1 || *h2 {
		fmt.Println(helpInfo)
		return
	}

	if *v1 || *v2 {
		if Version == "" {
			Version = "-1 not set"
		}
		fmt.Println(ProgName, Version)
		return
	}

	device.DeviceInit(*deviceType)
}
<|endoftext|>


<commit_before>// Copyright 2020 The Inet.Af AUTHORS. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package netaddr

import (
	"errors"
	"fmt"
	"sort"
	"strings"
)

// IPSetBuilder builds an immutable IPSet.
//
// The zero value is a valid value representing a set of no IPs.
//
// The Add and Remove methods add or remove IPs to/from the set.
// Removals only affect the current membership of the set, so in
// general Adds should be called first. Input ranges may overlap in
// any way.
type IPSetBuilder struct {
	// in are the ranges in the set.
	in []IPRange

	// out are the ranges to be removed from 'in'.
	out []IPRange

	// errs are errors accumulated during construction.
	errs multiErr
}

// normalize normalizes s: s.in becomes the minimal sorted list of
// ranges required to describe s, and s.out becomes empty.
func (s *IPSetBuilder) normalize() {
	const debug = false
	if debug {
		debugf("ranges start in=%v out=%v", s.in, s.out)
	}
	in, ok := mergeIPRanges(s.in)
	if !ok {
		return
	}
	out, ok := mergeIPRanges(s.out)
	if !ok {
		return
	}
	if debug {
		debugf("ranges sort  in=%v out=%v", in, out)
	}

	// in and out are sorted in ascending range order, and have no
	// overlaps within each other. We can run a merge of the two lists
	// in one pass.

	min := make([]IPRange, 0, len(in))
	for len(in) > 0 && len(out) > 0 {
		rin, rout := in[0], out[0]
		if debug {
			debugf("step in=%v out=%v", rin, rout)
		}

		switch {
		case !rout.Valid() || !rin.Valid():
			// mergeIPRanges should have prevented invalid ranges from
			// sneaking in.
			panic("invalid IPRanges during Ranges merge")
		case rout.entirelyBefore(rin):
			// "out" is entirely before "in".
			//
			//    out         in
			// f-------t   f-------t
			out = out[1:]
			if debug {
				debugf("out before in; drop out")
			}
		case rin.entirelyBefore(rout):
			// "in" is entirely before "out".
			//
			//    in         out
			// f------t   f-------t
			min = append(min, rin)
			in = in[1:]
			if debug {
				debugf("in before out; append in")
				debugf("min=%v", min)
			}
		case rin.coveredBy(rout):
			// "out" entirely covers "in".
			//
			//       out
			// f-------------t
			//    f------t
			//       in
			in = in[1:]
			if debug {
				debugf("in inside out; drop in")
			}
		case rout.inMiddleOf(rin):
			// "in" entirely covers "out".
			//
			//       in
			// f-------------t
			//    f------t
			//       out
			min = append(min, IPRange{from: rin.from, to: rout.from.Prior()})
			// Adjust in[0], not ir, because we want to consider the
			// mutated range on the next iteration.
			in[0].from = rout.to.Next()
			out = out[1:]
			if debug {
				debugf("out inside in; split in, append first in, drop out, adjust second in")
				debugf("min=%v", min)
			}
		case rout.overlapsStartOf(rin):
			// "out" overlaps start of "in".
			//
			//   out
			// f------t
			//    f------t
			//       in
			in[0].from = rout.to.Next()
			// Can't move ir onto min yet, another later out might
			// trim it further. Just discard or and continue.
			out = out[1:]
			if debug {
				debugf("out cuts start of in; adjust in, drop out")
			}
		case rout.overlapsEndOf(rin):
			// "out" overlaps end of "in".
			//
			//           out
			//        f------t
			//    f------t
			//       in
			min = append(min, IPRange{from: rin.from, to: rout.from.Prior()})
			in = in[1:]
			if debug {
				debugf("merge out cuts end of in; append shortened in")
				debugf("min=%v", min)
			}
		default:
			// The above should account for all combinations of in and
			// out overlapping, but insert a panic to be sure.
			panic("unexpected additional overlap scenario")
		}
	}
	if len(in) > 0 {
		// Ran out of removals before the end of in.
		min = append(min, in...)
		if debug {
			debugf("min=%v", min)
		}
	}

	s.in = min
	s.out = nil
}

// Clone returns a copy of s that shares no memory with s.
func (s *IPSetBuilder) Clone() *IPSetBuilder {
	return &IPSetBuilder{
		in:  append([]IPRange(nil), s.in...),
		out: append([]IPRange(nil), s.out...),
	}
}

// Add adds ip to s.
func (s *IPSetBuilder) Add(ip IP) {
	if ip.IsZero() {
		return
	}
	s.AddRange(IPRangeFrom(ip, ip))
}

// AddPrefix adds all IPs in p to s.
func (s *IPSetBuilder) AddPrefix(p IPPrefix) {
	if r := p.Range(); r.Valid() {
		s.AddRange(r)
	} else {
		s.errs = append(s.errs, fmt.Errorf("AddPrefix of invalid prefix %q", p))
	}
}

// AddRange adds r to s.
// If r is not Valid, AddRange does nothing.
func (s *IPSetBuilder) AddRange(r IPRange) {
	if !r.Valid() {
		s.errs = append(s.errs, fmt.Errorf("AddRange of invalid range %q", r))
		return
	}
	// If there are any removals (s.out), then we need to compact the set
	// first to get the order right.
	if len(s.out) > 0 {
		s.normalize()
	}
	s.in = append(s.in, r)
}

// AddSet adds all IPs in b to s.
func (s *IPSetBuilder) AddSet(b *IPSet) {
	if b == nil {
		return
	}
	for _, r := range b.rr {
		s.AddRange(r)
	}
}

// Remove removes ip from s.
func (s *IPSetBuilder) Remove(ip IP) {
	if ip.IsZero() {
		s.errs = append(s.errs, errors.New("ignored Remove of zero IP"))
	} else {
		s.RemoveRange(IPRangeFrom(ip, ip))
	}
}

// RemovePrefix removes all IPs in p from s.
func (s *IPSetBuilder) RemovePrefix(p IPPrefix) {
	if r := p.Range(); r.Valid() {
		s.RemoveRange(r)
	} else {
		s.errs = append(s.errs, fmt.Errorf("RemovePrefix of invalid prefix %q", p))
	}
}

// RemoveRange removes all IPs in r from s.
func (s *IPSetBuilder) RemoveRange(r IPRange) {
	if r.Valid() {
		s.out = append(s.out, r)
	} else {
		s.errs = append(s.errs, fmt.Errorf("RemoveRange of invalid range %q", r))
	}
}

// RemoveSet removes all IPs in o from s.
func (s *IPSetBuilder) RemoveSet(b *IPSet) {
	if b == nil {
		return
	}
	for _, r := range b.rr {
		s.RemoveRange(r)
	}
}

// removeBuilder removes all IPs in b from s.
func (s *IPSetBuilder) removeBuilder(b *IPSetBuilder) {
	b.normalize()
	for _, r := range b.in {
		s.RemoveRange(r)
	}
}

// Complement updates s to contain the complement of its current
// contents.
func (s *IPSetBuilder) Complement() {
	s.normalize()
	s.out = s.in
	s.in = []IPRange{
		IPPrefix{ip: IPv4(0, 0, 0, 0), bits: 0}.Range(),
		IPPrefix{ip: IPv6Unspecified(), bits: 0}.Range(),
	}
}

// Intersect updates s to the set intersection of s and b.
func (s *IPSetBuilder) Intersect(b *IPSet) {
	var o IPSetBuilder
	o.Complement()
	o.RemoveSet(b)
	s.removeBuilder(&o)
}

func discardf(format string, args ...interface{}) {}

// debugf is reassigned by tests.
var debugf = discardf

// IPSet returns an immutable IPSet representing the current state of s.
//
// Most IPSetBuilder methods do not return errors.
// Rather, the builder ignores any invalid inputs (such as an invalid IPPrefix),
// and accumulates a list of any such errors that it encountered.
//
// IPSet also reports any such accumulated errors.
// Even if the returned error is non-nil, the returned IPSet is usable
// and contains all modifications made with valid inputs.
//
// The builder remains usable after calling IPSet.
// Calling IPSet clears any accumulated errors.
func (s *IPSetBuilder) IPSet() (*IPSet, error) {
	s.normalize()
	ret := &IPSet{
		rr: append([]IPRange{}, s.in...),
	}
	if len(s.errs) == 0 {
		return ret, nil
	} else {
		errs := s.errs
		s.errs = nil
		return ret, errs
	}
}

// IPSet represents a set of IP addresses.
//
// IPSet is safe for concurrent use.
// The zero value is a valid value representing a set of no IPs.
// Use IPSetBuilder to construct IPSets.
type IPSet struct {
	// rr is the set of IPs that belong to this IPSet. The IPRanges
	// are normalized according to IPSetBuilder.normalize, meaning
	// they are a sorted, minimal representation (no overlapping
	// ranges, no contiguous ranges). The implementation of various
	// methods rely on this property.
	rr []IPRange
}

// Ranges returns the minimum and sorted set of IP
// ranges that covers s.
func (s *IPSet) Ranges() []IPRange {
	return append([]IPRange{}, s.rr...)
}

// Prefixes returns the minimum and sorted set of IP prefixes
// that covers s.
func (s *IPSet) Prefixes() []IPPrefix {
	out := make([]IPPrefix, 0, len(s.rr))
	for _, r := range s.rr {
		out = append(out, r.Prefixes()...)
	}
	return out
}

// Equal reports whether s and o represent the same set of IP
// addresses.
func (s *IPSet) Equal(o *IPSet) bool {
	if len(s.rr) != len(o.rr) {
		return false
	}
	for i := range s.rr {
		if s.rr[i] != o.rr[i] {
			return false
		}
	}
	return true
}

// Contains reports whether ip is in s.
// If ip has an IPv6 zone, Contains returns false,
// because IPSets do not track zones.
func (s *IPSet) Contains(ip IP) bool {
	if ip.hasZone() {
		return false
	}
	// TODO: data structure permitting more efficient lookups:
	// https://github.com/inetaf/netaddr/issues/139
	i := sort.Search(len(s.rr), func(i int) bool {
		return ip.Less(s.rr[i].from)
	})
	if i == 0 {
		return false
	}
	i--
	return s.rr[i].contains(ip)
}

// ContainsRange reports whether all IPs in r are in s.
func (s *IPSet) ContainsRange(r IPRange) bool {
	for _, x := range s.rr {
		if r.coveredBy(x) {
			return true
		}
	}
	return false
}

// ContainsPrefix reports whether all IPs in p are in s.
func (s *IPSet) ContainsPrefix(p IPPrefix) bool {
	return s.ContainsRange(p.Range())
}

// Overlaps reports whether any IP in b is also in s.
func (s *IPSet) Overlaps(b *IPSet) bool {
	// TODO: sorted ranges lets us do this in O(n+m)
	for _, r := range s.rr {
		for _, or := range b.rr {
			if r.Overlaps(or) {
				return true
			}
		}
	}
	return false
}

// OverlapsRange reports whether any IP in r is also in s.
func (s *IPSet) OverlapsRange(r IPRange) bool {
	// TODO: sorted ranges lets us do this more efficiently.
	for _, x := range s.rr {
		if x.Overlaps(r) {
			return true
		}
	}
	return false
}

// OverlapsPrefix reports whether any IP in p is also in s.
func (s *IPSet) OverlapsPrefix(p IPPrefix) bool {
	return s.OverlapsRange(p.Range())
}

// RemoveFreePrefix splits s into a Prefix of length bitLen and a new
// IPSet with that prefix removed.
//
// If no contiguous prefix of length bitLen exists in s,
// RemoveFreePrefix returns ok=false.
func (s *IPSet) RemoveFreePrefix(bitLen uint8) (p IPPrefix, newSet *IPSet, ok bool) {
	var bestFit IPPrefix
	for _, r := range s.rr {
		for _, prefix := range r.Prefixes() {
			if prefix.bits > bitLen {
				continue
			}
			if bestFit.ip.IsZero() || prefix.bits > bestFit.bits {
				bestFit = prefix
				if bestFit.bits == bitLen {
					// exact match, done.
					break
				}
			}
		}
	}

	if bestFit.ip.IsZero() {
		return IPPrefix{}, s, false
	}

	prefix := IPPrefix{ip: bestFit.ip, bits: bitLen}

	var b IPSetBuilder
	b.AddSet(s)
	b.RemovePrefix(prefix)
	newSet, _ = b.IPSet()
	return prefix, newSet, true
}

type multiErr []error

func (e multiErr) Error() string {
	var ret []string
	for _, err := range e {
		ret = append(ret, err.Error())
	}
	return strings.Join(ret, ", ")
}
<commit_msg>note unusual API in IPSetBuilder type documentation<commit_after>// Copyright 2020 The Inet.Af AUTHORS. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package netaddr

import (
	"errors"
	"fmt"
	"sort"
	"strings"
)

// IPSetBuilder builds an immutable IPSet.
//
// The zero value is a valid value representing a set of no IPs.
//
// The Add and Remove methods add or remove IPs to/from the set.
// Removals only affect the current membership of the set, so in
// general Adds should be called first. Input ranges may overlap in
// any way.
//
// Most IPSetBuilder methods do not return errors.
// Instead, errors are accumulated and reported by IPSetBuilder.IPSet.
type IPSetBuilder struct {
	// in are the ranges in the set.
	in []IPRange

	// out are the ranges to be removed from 'in'.
	out []IPRange

	// errs are errors accumulated during construction.
	errs multiErr
}

// normalize normalizes s: s.in becomes the minimal sorted list of
// ranges required to describe s, and s.out becomes empty.
func (s *IPSetBuilder) normalize() {
	const debug = false
	if debug {
		debugf("ranges start in=%v out=%v", s.in, s.out)
	}
	in, ok := mergeIPRanges(s.in)
	if !ok {
		return
	}
	out, ok := mergeIPRanges(s.out)
	if !ok {
		return
	}
	if debug {
		debugf("ranges sort  in=%v out=%v", in, out)
	}

	// in and out are sorted in ascending range order, and have no
	// overlaps within each other. We can run a merge of the two lists
	// in one pass.

	min := make([]IPRange, 0, len(in))
	for len(in) > 0 && len(out) > 0 {
		rin, rout := in[0], out[0]
		if debug {
			debugf("step in=%v out=%v", rin, rout)
		}

		switch {
		case !rout.Valid() || !rin.Valid():
			// mergeIPRanges should have prevented invalid ranges from
			// sneaking in.
			panic("invalid IPRanges during Ranges merge")
		case rout.entirelyBefore(rin):
			// "out" is entirely before "in".
			//
			//    out         in
			// f-------t   f-------t
			out = out[1:]
			if debug {
				debugf("out before in; drop out")
			}
		case rin.entirelyBefore(rout):
			// "in" is entirely before "out".
			//
			//    in         out
			// f------t   f-------t
			min = append(min, rin)
			in = in[1:]
			if debug {
				debugf("in before out; append in")
				debugf("min=%v", min)
			}
		case rin.coveredBy(rout):
			// "out" entirely covers "in".
			//
			//       out
			// f-------------t
			//    f------t
			//       in
			in = in[1:]
			if debug {
				debugf("in inside out; drop in")
			}
		case rout.inMiddleOf(rin):
			// "in" entirely covers "out".
			//
			//       in
			// f-------------t
			//    f------t
			//       out
			min = append(min, IPRange{from: rin.from, to: rout.from.Prior()})
			// Adjust in[0], not ir, because we want to consider the
			// mutated range on the next iteration.
			in[0].from = rout.to.Next()
			out = out[1:]
			if debug {
				debugf("out inside in; split in, append first in, drop out, adjust second in")
				debugf("min=%v", min)
			}
		case rout.overlapsStartOf(rin):
			// "out" overlaps start of "in".
			//
			//   out
			// f------t
			//    f------t
			//       in
			in[0].from = rout.to.Next()
			// Can't move ir onto min yet, another later out might
			// trim it further. Just discard or and continue.
			out = out[1:]
			if debug {
				debugf("out cuts start of in; adjust in, drop out")
			}
		case rout.overlapsEndOf(rin):
			// "out" overlaps end of "in".
			//
			//           out
			//        f------t
			//    f------t
			//       in
			min = append(min, IPRange{from: rin.from, to: rout.from.Prior()})
			in = in[1:]
			if debug {
				debugf("merge out cuts end of in; append shortened in")
				debugf("min=%v", min)
			}
		default:
			// The above should account for all combinations of in and
			// out overlapping, but insert a panic to be sure.
			panic("unexpected additional overlap scenario")
		}
	}
	if len(in) > 0 {
		// Ran out of removals before the end of in.
		min = append(min, in...)
		if debug {
			debugf("min=%v", min)
		}
	}

	s.in = min
	s.out = nil
}

// Clone returns a copy of s that shares no memory with s.
func (s *IPSetBuilder) Clone() *IPSetBuilder {
	return &IPSetBuilder{
		in:  append([]IPRange(nil), s.in...),
		out: append([]IPRange(nil), s.out...),
	}
}

// Add adds ip to s.
func (s *IPSetBuilder) Add(ip IP) {
	if ip.IsZero() {
		return
	}
	s.AddRange(IPRangeFrom(ip, ip))
}

// AddPrefix adds all IPs in p to s.
func (s *IPSetBuilder) AddPrefix(p IPPrefix) {
	if r := p.Range(); r.Valid() {
		s.AddRange(r)
	} else {
		s.errs = append(s.errs, fmt.Errorf("AddPrefix of invalid prefix %q", p))
	}
}

// AddRange adds r to s.
// If r is not Valid, AddRange does nothing.
func (s *IPSetBuilder) AddRange(r IPRange) {
	if !r.Valid() {
		s.errs = append(s.errs, fmt.Errorf("AddRange of invalid range %q", r))
		return
	}
	// If there are any removals (s.out), then we need to compact the set
	// first to get the order right.
	if len(s.out) > 0 {
		s.normalize()
	}
	s.in = append(s.in, r)
}

// AddSet adds all IPs in b to s.
func (s *IPSetBuilder) AddSet(b *IPSet) {
	if b == nil {
		return
	}
	for _, r := range b.rr {
		s.AddRange(r)
	}
}

// Remove removes ip from s.
func (s *IPSetBuilder) Remove(ip IP) {
	if ip.IsZero() {
		s.errs = append(s.errs, errors.New("ignored Remove of zero IP"))
	} else {
		s.RemoveRange(IPRangeFrom(ip, ip))
	}
}

// RemovePrefix removes all IPs in p from s.
func (s *IPSetBuilder) RemovePrefix(p IPPrefix) {
	if r := p.Range(); r.Valid() {
		s.RemoveRange(r)
	} else {
		s.errs = append(s.errs, fmt.Errorf("RemovePrefix of invalid prefix %q", p))
	}
}

// RemoveRange removes all IPs in r from s.
func (s *IPSetBuilder) RemoveRange(r IPRange) {
	if r.Valid() {
		s.out = append(s.out, r)
	} else {
		s.errs = append(s.errs, fmt.Errorf("RemoveRange of invalid range %q", r))
	}
}

// RemoveSet removes all IPs in o from s.
func (s *IPSetBuilder) RemoveSet(b *IPSet) {
	if b == nil {
		return
	}
	for _, r := range b.rr {
		s.RemoveRange(r)
	}
}

// removeBuilder removes all IPs in b from s.
func (s *IPSetBuilder) removeBuilder(b *IPSetBuilder) {
	b.normalize()
	for _, r := range b.in {
		s.RemoveRange(r)
	}
}

// Complement updates s to contain the complement of its current
// contents.
func (s *IPSetBuilder) Complement() {
	s.normalize()
	s.out = s.in
	s.in = []IPRange{
		IPPrefix{ip: IPv4(0, 0, 0, 0), bits: 0}.Range(),
		IPPrefix{ip: IPv6Unspecified(), bits: 0}.Range(),
	}
}

// Intersect updates s to the set intersection of s and b.
func (s *IPSetBuilder) Intersect(b *IPSet) {
	var o IPSetBuilder
	o.Complement()
	o.RemoveSet(b)
	s.removeBuilder(&o)
}

func discardf(format string, args ...interface{}) {}

// debugf is reassigned by tests.
var debugf = discardf

// IPSet returns an immutable IPSet representing the current state of s.
//
// Most IPSetBuilder methods do not return errors.
// Rather, the builder ignores any invalid inputs (such as an invalid IPPrefix),
// and accumulates a list of any such errors that it encountered.
//
// IPSet also reports any such accumulated errors.
// Even if the returned error is non-nil, the returned IPSet is usable
// and contains all modifications made with valid inputs.
//
// The builder remains usable after calling IPSet.
// Calling IPSet clears any accumulated errors.
func (s *IPSetBuilder) IPSet() (*IPSet, error) {
	s.normalize()
	ret := &IPSet{
		rr: append([]IPRange{}, s.in...),
	}
	if len(s.errs) == 0 {
		return ret, nil
	} else {
		errs := s.errs
		s.errs = nil
		return ret, errs
	}
}

// IPSet represents a set of IP addresses.
//
// IPSet is safe for concurrent use.
// The zero value is a valid value representing a set of no IPs.
// Use IPSetBuilder to construct IPSets.
type IPSet struct {
	// rr is the set of IPs that belong to this IPSet. The IPRanges
	// are normalized according to IPSetBuilder.normalize, meaning
	// they are a sorted, minimal representation (no overlapping
	// ranges, no contiguous ranges). The implementation of various
	// methods rely on this property.
	rr []IPRange
}

// Ranges returns the minimum and sorted set of IP
// ranges that covers s.
func (s *IPSet) Ranges() []IPRange {
	return append([]IPRange{}, s.rr...)
}

// Prefixes returns the minimum and sorted set of IP prefixes
// that covers s.
func (s *IPSet) Prefixes() []IPPrefix {
	out := make([]IPPrefix, 0, len(s.rr))
	for _, r := range s.rr {
		out = append(out, r.Prefixes()...)
	}
	return out
}

// Equal reports whether s and o represent the same set of IP
// addresses.
func (s *IPSet) Equal(o *IPSet) bool {
	if len(s.rr) != len(o.rr) {
		return false
	}
	for i := range s.rr {
		if s.rr[i] != o.rr[i] {
			return false
		}
	}
	return true
}

// Contains reports whether ip is in s.
// If ip has an IPv6 zone, Contains returns false,
// because IPSets do not track zones.
func (s *IPSet) Contains(ip IP) bool {
	if ip.hasZone() {
		return false
	}
	// TODO: data structure permitting more efficient lookups:
	// https://github.com/inetaf/netaddr/issues/139
	i := sort.Search(len(s.rr), func(i int) bool {
		return ip.Less(s.rr[i].from)
	})
	if i == 0 {
		return false
	}
	i--
	return s.rr[i].contains(ip)
}

// ContainsRange reports whether all IPs in r are in s.
func (s *IPSet) ContainsRange(r IPRange) bool {
	for _, x := range s.rr {
		if r.coveredBy(x) {
			return true
		}
	}
	return false
}

// ContainsPrefix reports whether all IPs in p are in s.
func (s *IPSet) ContainsPrefix(p IPPrefix) bool {
	return s.ContainsRange(p.Range())
}

// Overlaps reports whether any IP in b is also in s.
func (s *IPSet) Overlaps(b *IPSet) bool {
	// TODO: sorted ranges lets us do this in O(n+m)
	for _, r := range s.rr {
		for _, or := range b.rr {
			if r.Overlaps(or) {
				return true
			}
		}
	}
	return false
}

// OverlapsRange reports whether any IP in r is also in s.
func (s *IPSet) OverlapsRange(r IPRange) bool {
	// TODO: sorted ranges lets us do this more efficiently.
	for _, x := range s.rr {
		if x.Overlaps(r) {
			return true
		}
	}
	return false
}

// OverlapsPrefix reports whether any IP in p is also in s.
func (s *IPSet) OverlapsPrefix(p IPPrefix) bool {
	return s.OverlapsRange(p.Range())
}

// RemoveFreePrefix splits s into a Prefix of length bitLen and a new
// IPSet with that prefix removed.
//
// If no contiguous prefix of length bitLen exists in s,
// RemoveFreePrefix returns ok=false.
func (s *IPSet) RemoveFreePrefix(bitLen uint8) (p IPPrefix, newSet *IPSet, ok bool) {
	var bestFit IPPrefix
	for _, r := range s.rr {
		for _, prefix := range r.Prefixes() {
			if prefix.bits > bitLen {
				continue
			}
			if bestFit.ip.IsZero() || prefix.bits > bestFit.bits {
				bestFit = prefix
				if bestFit.bits == bitLen {
					// exact match, done.
					break
				}
			}
		}
	}

	if bestFit.ip.IsZero() {
		return IPPrefix{}, s, false
	}

	prefix := IPPrefix{ip: bestFit.ip, bits: bitLen}

	var b IPSetBuilder
	b.AddSet(s)
	b.RemovePrefix(prefix)
	newSet, _ = b.IPSet()
	return prefix, newSet, true
}

type multiErr []error

func (e multiErr) Error() string {
	var ret []string
	for _, err := range e {
		ret = append(ret, err.Error())
	}
	return strings.Join(ret, ", ")
}
<|endoftext|>