<commit_before>package btelegram

import (
	"html"
	"regexp"
	"strconv"
	"strings"

	"github.com/42wim/matterbridge/bridge/config"
	"github.com/42wim/matterbridge/bridge/helper"
	"github.com/go-telegram-bot-api/telegram-bot-api"
)

func (b *Btelegram) handleUpdate(rmsg *config.Message, message, posted, edited *tgbotapi.Message) *tgbotapi.Message {
	// handle channels
	if posted != nil {
		message = posted
		rmsg.Text = message.Text
	}

	// edited channel message
	if edited != nil && !b.GetBool("EditDisable") {
		message = edited
		rmsg.Text = rmsg.Text + message.Text + b.GetString("EditSuffix")
	}
	return message
}

// handleChannels checks if it's a channel message and if the message is a new or edited messages
func (b *Btelegram) handleChannels(rmsg *config.Message, message *tgbotapi.Message, update tgbotapi.Update) *tgbotapi.Message {
	return b.handleUpdate(rmsg, message, update.ChannelPost, update.EditedChannelPost)
}

// handleGroups checks if it's a group message and if the message is a new or edited messages
func (b *Btelegram) handleGroups(rmsg *config.Message, message *tgbotapi.Message, update tgbotapi.Update) *tgbotapi.Message {
	return b.handleUpdate(rmsg, message, update.Message, update.EditedMessage)
}

// handleForwarded handles forwarded messages
func (b *Btelegram) handleForwarded(rmsg *config.Message, message *tgbotapi.Message) {
	if message.ForwardFrom != nil {
		usernameForward := ""
		if b.GetBool("UseFirstName") {
			usernameForward = message.ForwardFrom.FirstName
		}
		if usernameForward == "" {
			usernameForward = message.ForwardFrom.UserName
			if usernameForward == "" {
				usernameForward = message.ForwardFrom.FirstName
			}
		}
		if usernameForward == "" {
			usernameForward = unknownUser
		}
		rmsg.Text = "Forwarded from " + usernameForward + ": " + rmsg.Text
	}
}

// handleQuoting handles quoting of previous messages
func (b *Btelegram) handleQuoting(rmsg *config.Message, message *tgbotapi.Message) {
	if message.ReplyToMessage != nil {
		usernameReply := ""
		if message.ReplyToMessage.From != nil {
			if b.GetBool("UseFirstName") {
				usernameReply = message.ReplyToMessage.From.FirstName
			}
			if usernameReply == "" {
				usernameReply = message.ReplyToMessage.From.UserName
				if usernameReply == "" {
					usernameReply = message.ReplyToMessage.From.FirstName
				}
			}
		}
		if usernameReply == "" {
			usernameReply = unknownUser
		}
		if !b.GetBool("QuoteDisable") {
			rmsg.Text = b.handleQuote(rmsg.Text, usernameReply, message.ReplyToMessage.Text)
		}
	}
}

// handleUsername handles the correct setting of the username
func (b *Btelegram) handleUsername(rmsg *config.Message, message *tgbotapi.Message) {
	if message.From != nil {
		rmsg.UserID = strconv.Itoa(message.From.ID)
		if b.GetBool("UseFirstName") {
			rmsg.Username = message.From.FirstName
		}
		if rmsg.Username == "" {
			rmsg.Username = message.From.UserName
			if rmsg.Username == "" {
				rmsg.Username = message.From.FirstName
			}
		}
		// only download avatars if we have a place to upload them (configured mediaserver)
		if b.General.MediaServerUpload != "" {
			b.handleDownloadAvatar(message.From.ID, rmsg.Channel)
		}
	}

	// if we really didn't find a username, set it to unknown
	if rmsg.Username == "" {
		rmsg.Username = unknownUser
	}
}

func (b *Btelegram) handleRecv(updates <-chan tgbotapi.Update) {
	for update := range updates {
		b.Log.Debugf("== Receiving event: %#v", update.Message)

		if update.Message == nil && update.ChannelPost == nil &&
			update.EditedMessage == nil && update.EditedChannelPost == nil {
			b.Log.Error("Getting nil messages, this shouldn't happen.")
			continue
		}

		var message *tgbotapi.Message

		rmsg := config.Message{Account: b.Account, Extra: make(map[string][]interface{})}

		// handle channels
		message = b.handleChannels(&rmsg, message, update)

		// handle groups
		message = b.handleGroups(&rmsg, message, update)

		// set the ID's from the channel or group message
		rmsg.ID = strconv.Itoa(message.MessageID)
		rmsg.Channel = strconv.FormatInt(message.Chat.ID, 10)

		// handle username
		b.handleUsername(&rmsg, message)

		// handle any downloads
		err := b.handleDownload(&rmsg, message)
		if err != nil {
			b.Log.Errorf("download failed: %s", err)
		}

		// handle forwarded messages
		b.handleForwarded(&rmsg, message)

		// quote the previous message
		b.handleQuoting(&rmsg, message)

		// handle entities (adding URLs)
		b.handleEntities(&rmsg, message)

		if rmsg.Text != "" || len(rmsg.Extra) > 0 {
			rmsg.Text = helper.RemoveEmptyNewLines(rmsg.Text)
			// channels don't have (always?) user information. see #410
			if message.From != nil {
				rmsg.Avatar = helper.GetAvatar(b.avatarMap, strconv.Itoa(message.From.ID), b.General)
			}

			b.Log.Debugf("<= Sending message from %s on %s to gateway", rmsg.Username, b.Account)
			b.Log.Debugf("<= Message is %#v", rmsg)
			b.Remote <- rmsg
		}
	}
}

// handleDownloadAvatar downloads the avatar of userid from channel
// sends a EVENT_AVATAR_DOWNLOAD message to the gateway if successful.
// logs an error message if it fails
func (b *Btelegram) handleDownloadAvatar(userid int, channel string) {
	rmsg := config.Message{Username: "system",
		Text:    "avatar",
		Channel: channel,
		Account: b.Account,
		UserID:  strconv.Itoa(userid),
		Event:   config.EventAvatarDownload,
		Extra:   make(map[string][]interface{})}

	if _, ok := b.avatarMap[strconv.Itoa(userid)]; !ok {
		photos, err := b.c.GetUserProfilePhotos(tgbotapi.UserProfilePhotosConfig{UserID: userid, Limit: 1})
		if err != nil {
			b.Log.Errorf("Userprofile download failed for %#v %s", userid, err)
		}

		if len(photos.Photos) > 0 {
			photo := photos.Photos[0][0]
			url := b.getFileDirectURL(photo.FileID)
			name := strconv.Itoa(userid) + ".png"
			b.Log.Debugf("trying to download %#v fileid %#v with size %#v", name, photo.FileID, photo.FileSize)

			err := helper.HandleDownloadSize(b.Log, &rmsg, name, int64(photo.FileSize), b.General)
			if err != nil {
				b.Log.Error(err)
				return
			}
			data, err := helper.DownloadFile(url)
			if err != nil {
				b.Log.Errorf("download %s failed %#v", url, err)
				return
			}
			helper.HandleDownloadData(b.Log, &rmsg, name, rmsg.Text, "", data, b.General)
			b.Remote <- rmsg
		}
	}
}

// handleDownloadFile handles file download
func (b *Btelegram) handleDownload(rmsg *config.Message, message *tgbotapi.Message) error {
	size := 0
	var url, name, text string
	switch {
	case message.Sticker != nil:
		text, name, url = b.getDownloadInfo(message.Sticker.FileID, ".webp", true)
		size = message.Sticker.FileSize
	case message.Voice != nil:
		text, name, url = b.getDownloadInfo(message.Voice.FileID, ".ogg", true)
		size = message.Voice.FileSize
	case message.Video != nil:
		text, name, url = b.getDownloadInfo(message.Video.FileID, "", true)
		size = message.Video.FileSize
	case message.Audio != nil:
		text, name, url = b.getDownloadInfo(message.Audio.FileID, "", true)
		size = message.Audio.FileSize
	case message.Document != nil:
		_, _, url = b.getDownloadInfo(message.Document.FileID, "", false)
		size = message.Document.FileSize
		name = message.Document.FileName
		text = " " + message.Document.FileName + " : " + url
	case message.Photo != nil:
		photos := *message.Photo
		size = photos[len(photos)-1].FileSize
		text, name, url = b.getDownloadInfo(photos[len(photos)-1].FileID, "", true)
	}

	// if name is empty we didn't match a thing to download
	if name == "" {
		return nil
	}
	// use the URL instead of native upload
	if b.GetBool("UseInsecureURL") {
		b.Log.Debugf("Setting message text to :%s", text)
		rmsg.Text += text
		return nil
	}
	// if we have a file attached, download it (in memory) and put a pointer to it in msg.Extra
	err := helper.HandleDownloadSize(b.Log, rmsg, name, int64(size), b.General)
	if err != nil {
		return err
	}
	data, err := helper.DownloadFile(url)
	if err != nil {
		return err
	}
	if strings.HasSuffix(name, ".webp") && b.GetBool("MediaConvertWebPToPNG") {
		b.Log.Debugf("WebP to PNG conversion enabled, converting %s", name)
		err := helper.ConvertWebPToPNG(data)
		if err != nil {
			b.Log.Errorf("conversion failed: %s", err)
		} else {
			name = strings.Replace(name, ".webp", ".png", 1)
		}
	}
	helper.HandleDownloadData(b.Log, rmsg, name, message.Caption, "", data, b.General)
	return nil
}

func (b *Btelegram) getDownloadInfo(id string, suffix string, urlpart bool) (string, string, string) {
	url := b.getFileDirectURL(id)
	name := ""
	if urlpart {
		urlPart := strings.Split(url, "/")
		name = urlPart[len(urlPart)-1]
	}
	if suffix != "" && !strings.HasSuffix(name, suffix) {
		name += suffix
	}
	text := " " + url
	return text, name, url
}

// handleDelete handles message deleting
func (b *Btelegram) handleDelete(msg *config.Message, chatid int64) (string, error) {
	if msg.ID == "" {
		return "", nil
	}
	msgid, err := strconv.Atoi(msg.ID)
	if err != nil {
		return "", err
	}
	_, err = b.c.DeleteMessage(tgbotapi.DeleteMessageConfig{ChatID: chatid, MessageID: msgid})
	return "", err
}

// handleEdit handles message editing.
func (b *Btelegram) handleEdit(msg *config.Message, chatid int64) (string, error) {
	msgid, err := strconv.Atoi(msg.ID)
	if err != nil {
		return "", err
	}
	if strings.ToLower(b.GetString("MessageFormat")) == HTMLNick {
		b.Log.Debug("Using mode HTML - nick only")
		msg.Text = html.EscapeString(msg.Text)
	}
	m := tgbotapi.NewEditMessageText(chatid, msgid, msg.Username+msg.Text)
	switch b.GetString("MessageFormat") {
	case HTMLFormat:
		b.Log.Debug("Using mode HTML")
		m.ParseMode = tgbotapi.ModeHTML
	case "Markdown":
		b.Log.Debug("Using mode markdown")
		m.ParseMode = tgbotapi.ModeMarkdown
	}
	if strings.ToLower(b.GetString("MessageFormat")) == HTMLNick {
		b.Log.Debug("Using mode HTML - nick only")
		m.ParseMode = tgbotapi.ModeHTML
	}
	_, err = b.c.Send(m)
	if err != nil {
		return "", err
	}
	return "", nil
}

// handleUploadFile handles native upload of files
func (b *Btelegram) handleUploadFile(msg *config.Message, chatid int64) string {
	var c tgbotapi.Chattable
	for _, f := range msg.Extra["file"] {
		fi := f.(config.FileInfo)
		file := tgbotapi.FileBytes{
			Name:  fi.Name,
			Bytes: *fi.Data,
		}
		re := regexp.MustCompile(".(jpg|png)$")
		if re.MatchString(fi.Name) {
			c = tgbotapi.NewPhotoUpload(chatid, file)
		} else {
			c = tgbotapi.NewDocumentUpload(chatid, file)
		}
		_, err := b.c.Send(c)
		if err != nil {
			b.Log.Errorf("file upload failed: %#v", err)
		}
		if fi.Comment != "" {
			if _, err := b.sendMessage(chatid, msg.Username, fi.Comment); err != nil {
				b.Log.Errorf("posting file comment %s failed: %s", fi.Comment, err)
			}
		}
	}
	return ""
}

func (b *Btelegram) handleQuote(message, quoteNick, quoteMessage string) string {
	format := b.GetString("quoteformat")
	if format == "" {
		format = "{MESSAGE} (re @{QUOTENICK}: {QUOTEMESSAGE})"
	}
	format = strings.Replace(format, "{MESSAGE}", message, -1)
	format = strings.Replace(format, "{QUOTENICK}", quoteNick, -1)
	format = strings.Replace(format, "{QUOTEMESSAGE}", quoteMessage, -1)
	return format
}

// handleEntities handles messageEntities
func (b *Btelegram) handleEntities(rmsg *config.Message, message *tgbotapi.Message) {
	if message.Entities == nil {
		return
	}
	// for now only do URL replacements
	for _, e := range *message.Entities {
		if e.Type == "text_link" {
			url, err := e.ParseURL()
			if err != nil {
				b.Log.Errorf("entity text_link url parse failed: %s", err)
				continue
			}
			link := rmsg.Text[e.Offset : e.Offset+e.Length]
			rmsg.Text = strings.Replace(rmsg.Text, link, url.String(), 1)
		}
	}
}
<commit_msg>Handle nil message (telegram). Fixes #777<commit_after>package btelegram

import (
	"html"
	"regexp"
	"strconv"
	"strings"

	"github.com/42wim/matterbridge/bridge/config"
	"github.com/42wim/matterbridge/bridge/helper"
	"github.com/go-telegram-bot-api/telegram-bot-api"
)

func (b *Btelegram) handleUpdate(rmsg *config.Message, message, posted, edited *tgbotapi.Message) *tgbotapi.Message {
	// handle channels
	if posted != nil {
		message = posted
		rmsg.Text = message.Text
	}

	// edited channel message
	if edited != nil && !b.GetBool("EditDisable") {
		message = edited
		rmsg.Text = rmsg.Text + message.Text + b.GetString("EditSuffix")
	}
	return message
}

// handleChannels checks if it's a channel message and if the message is a new or edited messages
func (b *Btelegram) handleChannels(rmsg *config.Message, message *tgbotapi.Message, update tgbotapi.Update) *tgbotapi.Message {
	return b.handleUpdate(rmsg, message, update.ChannelPost, update.EditedChannelPost)
}

// handleGroups checks if it's a group message and if the message is a new or edited messages
func (b *Btelegram) handleGroups(rmsg *config.Message, message *tgbotapi.Message, update tgbotapi.Update) *tgbotapi.Message {
	return b.handleUpdate(rmsg, message, update.Message, update.EditedMessage)
}

// handleForwarded handles forwarded messages
func (b *Btelegram) handleForwarded(rmsg *config.Message, message *tgbotapi.Message) {
	if message.ForwardFrom != nil {
		usernameForward := ""
		if b.GetBool("UseFirstName") {
			usernameForward = message.ForwardFrom.FirstName
		}
		if usernameForward == "" {
			usernameForward = message.ForwardFrom.UserName
			if usernameForward == "" {
				usernameForward = message.ForwardFrom.FirstName
			}
		}
		if usernameForward == "" {
			usernameForward = unknownUser
		}
		rmsg.Text = "Forwarded from " + usernameForward + ": " + rmsg.Text
	}
}

// handleQuoting handles quoting of previous messages
func (b *Btelegram) handleQuoting(rmsg *config.Message, message *tgbotapi.Message) {
	if message.ReplyToMessage != nil {
		usernameReply := ""
		if message.ReplyToMessage.From != nil {
			if b.GetBool("UseFirstName") {
				usernameReply = message.ReplyToMessage.From.FirstName
			}
			if usernameReply == "" {
				usernameReply = message.ReplyToMessage.From.UserName
				if usernameReply == "" {
					usernameReply = message.ReplyToMessage.From.FirstName
				}
			}
		}
		if usernameReply == "" {
			usernameReply = unknownUser
		}
		if !b.GetBool("QuoteDisable") {
			rmsg.Text = b.handleQuote(rmsg.Text, usernameReply, message.ReplyToMessage.Text)
		}
	}
}

// handleUsername handles the correct setting of the username
func (b *Btelegram) handleUsername(rmsg *config.Message, message *tgbotapi.Message) {
	if message.From != nil {
		rmsg.UserID = strconv.Itoa(message.From.ID)
		if b.GetBool("UseFirstName") {
			rmsg.Username = message.From.FirstName
		}
		if rmsg.Username == "" {
			rmsg.Username = message.From.UserName
			if rmsg.Username == "" {
				rmsg.Username = message.From.FirstName
			}
		}
		// only download avatars if we have a place to upload them (configured mediaserver)
		if b.General.MediaServerUpload != "" {
			b.handleDownloadAvatar(message.From.ID, rmsg.Channel)
		}
	}

	// if we really didn't find a username, set it to unknown
	if rmsg.Username == "" {
		rmsg.Username = unknownUser
	}
}

func (b *Btelegram) handleRecv(updates <-chan tgbotapi.Update) {
	for update := range updates {
		b.Log.Debugf("== Receiving event: %#v", update.Message)

		if update.Message == nil && update.ChannelPost == nil &&
			update.EditedMessage == nil && update.EditedChannelPost == nil {
			b.Log.Error("Getting nil messages, this shouldn't happen.")
			continue
		}

		var message *tgbotapi.Message

		rmsg := config.Message{Account: b.Account, Extra: make(map[string][]interface{})}

		// handle channels
		message = b.handleChannels(&rmsg, message, update)

		// handle groups
		message = b.handleGroups(&rmsg, message, update)

		if message == nil {
			b.Log.Error("message is nil, this shouldn't happen.")
			continue
		}

		// set the ID's from the channel or group message
		rmsg.ID = strconv.Itoa(message.MessageID)
		rmsg.Channel = strconv.FormatInt(message.Chat.ID, 10)

		// handle username
		b.handleUsername(&rmsg, message)

		// handle any downloads
		err := b.handleDownload(&rmsg, message)
		if err != nil {
			b.Log.Errorf("download failed: %s", err)
		}

		// handle forwarded messages
		b.handleForwarded(&rmsg, message)

		// quote the previous message
		b.handleQuoting(&rmsg, message)

		// handle entities (adding URLs)
		b.handleEntities(&rmsg, message)

		if rmsg.Text != "" || len(rmsg.Extra) > 0 {
			rmsg.Text = helper.RemoveEmptyNewLines(rmsg.Text)
			// channels don't have (always?) user information. see #410
			if message.From != nil {
				rmsg.Avatar = helper.GetAvatar(b.avatarMap, strconv.Itoa(message.From.ID), b.General)
			}

			b.Log.Debugf("<= Sending message from %s on %s to gateway", rmsg.Username, b.Account)
			b.Log.Debugf("<= Message is %#v", rmsg)
			b.Remote <- rmsg
		}
	}
}

// handleDownloadAvatar downloads the avatar of userid from channel
// sends a EVENT_AVATAR_DOWNLOAD message to the gateway if successful.
// logs an error message if it fails
func (b *Btelegram) handleDownloadAvatar(userid int, channel string) {
	rmsg := config.Message{Username: "system",
		Text:    "avatar",
		Channel: channel,
		Account: b.Account,
		UserID:  strconv.Itoa(userid),
		Event:   config.EventAvatarDownload,
		Extra:   make(map[string][]interface{})}

	if _, ok := b.avatarMap[strconv.Itoa(userid)]; !ok {
		photos, err := b.c.GetUserProfilePhotos(tgbotapi.UserProfilePhotosConfig{UserID: userid, Limit: 1})
		if err != nil {
			b.Log.Errorf("Userprofile download failed for %#v %s", userid, err)
		}

		if len(photos.Photos) > 0 {
			photo := photos.Photos[0][0]
			url := b.getFileDirectURL(photo.FileID)
			name := strconv.Itoa(userid) + ".png"
			b.Log.Debugf("trying to download %#v fileid %#v with size %#v", name, photo.FileID, photo.FileSize)

			err := helper.HandleDownloadSize(b.Log, &rmsg, name, int64(photo.FileSize), b.General)
			if err != nil {
				b.Log.Error(err)
				return
			}
			data, err := helper.DownloadFile(url)
			if err != nil {
				b.Log.Errorf("download %s failed %#v", url, err)
				return
			}
			helper.HandleDownloadData(b.Log, &rmsg, name, rmsg.Text, "", data, b.General)
			b.Remote <- rmsg
		}
	}
}

// handleDownloadFile handles file download
func (b *Btelegram) handleDownload(rmsg *config.Message, message *tgbotapi.Message) error {
	size := 0
	var url, name, text string
	switch {
	case message.Sticker != nil:
		text, name, url = b.getDownloadInfo(message.Sticker.FileID, ".webp", true)
		size = message.Sticker.FileSize
	case message.Voice != nil:
		text, name, url = b.getDownloadInfo(message.Voice.FileID, ".ogg", true)
		size = message.Voice.FileSize
	case message.Video != nil:
		text, name, url = b.getDownloadInfo(message.Video.FileID, "", true)
		size = message.Video.FileSize
	case message.Audio != nil:
		text, name, url = b.getDownloadInfo(message.Audio.FileID, "", true)
		size = message.Audio.FileSize
	case message.Document != nil:
		_, _, url = b.getDownloadInfo(message.Document.FileID, "", false)
		size = message.Document.FileSize
		name = message.Document.FileName
		text = " " + message.Document.FileName + " : " + url
	case message.Photo != nil:
		photos := *message.Photo
		size = photos[len(photos)-1].FileSize
		text, name, url = b.getDownloadInfo(photos[len(photos)-1].FileID, "", true)
	}

	// if name is empty we didn't match a thing to download
	if name == "" {
		return nil
	}
	// use the URL instead of native upload
	if b.GetBool("UseInsecureURL") {
		b.Log.Debugf("Setting message text to :%s", text)
		rmsg.Text += text
		return nil
	}
	// if we have a file attached, download it (in memory) and put a pointer to it in msg.Extra
	err := helper.HandleDownloadSize(b.Log, rmsg, name, int64(size), b.General)
	if err != nil {
		return err
	}
	data, err := helper.DownloadFile(url)
	if err != nil {
		return err
	}
	if strings.HasSuffix(name, ".webp") && b.GetBool("MediaConvertWebPToPNG") {
		b.Log.Debugf("WebP to PNG conversion enabled, converting %s", name)
		err := helper.ConvertWebPToPNG(data)
		if err != nil {
			b.Log.Errorf("conversion failed: %s", err)
		} else {
			name = strings.Replace(name, ".webp", ".png", 1)
		}
	}
	helper.HandleDownloadData(b.Log, rmsg, name, message.Caption, "", data, b.General)
	return nil
}

func (b *Btelegram) getDownloadInfo(id string, suffix string, urlpart bool) (string, string, string) {
	url := b.getFileDirectURL(id)
	name := ""
	if urlpart {
		urlPart := strings.Split(url, "/")
		name = urlPart[len(urlPart)-1]
	}
	if suffix != "" && !strings.HasSuffix(name, suffix) {
		name += suffix
	}
	text := " " + url
	return text, name, url
}

// handleDelete handles message deleting
func (b *Btelegram) handleDelete(msg *config.Message, chatid int64) (string, error) {
	if msg.ID == "" {
		return "", nil
	}
	msgid, err := strconv.Atoi(msg.ID)
	if err != nil {
		return "", err
	}
	_, err = b.c.DeleteMessage(tgbotapi.DeleteMessageConfig{ChatID: chatid, MessageID: msgid})
	return "", err
}

// handleEdit handles message editing.
func (b *Btelegram) handleEdit(msg *config.Message, chatid int64) (string, error) {
	msgid, err := strconv.Atoi(msg.ID)
	if err != nil {
		return "", err
	}
	if strings.ToLower(b.GetString("MessageFormat")) == HTMLNick {
		b.Log.Debug("Using mode HTML - nick only")
		msg.Text = html.EscapeString(msg.Text)
	}
	m := tgbotapi.NewEditMessageText(chatid, msgid, msg.Username+msg.Text)
	switch b.GetString("MessageFormat") {
	case HTMLFormat:
		b.Log.Debug("Using mode HTML")
		m.ParseMode = tgbotapi.ModeHTML
	case "Markdown":
		b.Log.Debug("Using mode markdown")
		m.ParseMode = tgbotapi.ModeMarkdown
	}
	if strings.ToLower(b.GetString("MessageFormat")) == HTMLNick {
		b.Log.Debug("Using mode HTML - nick only")
		m.ParseMode = tgbotapi.ModeHTML
	}
	_, err = b.c.Send(m)
	if err != nil {
		return "", err
	}
	return "", nil
}

// handleUploadFile handles native upload of files
func (b *Btelegram) handleUploadFile(msg *config.Message, chatid int64) string {
	var c tgbotapi.Chattable
	for _, f := range msg.Extra["file"] {
		fi := f.(config.FileInfo)
		file := tgbotapi.FileBytes{
			Name:  fi.Name,
			Bytes: *fi.Data,
		}
		re := regexp.MustCompile(".(jpg|png)$")
		if re.MatchString(fi.Name) {
			c = tgbotapi.NewPhotoUpload(chatid, file)
		} else {
			c = tgbotapi.NewDocumentUpload(chatid, file)
		}
		_, err := b.c.Send(c)
		if err != nil {
			b.Log.Errorf("file upload failed: %#v", err)
		}
		if fi.Comment != "" {
			if _, err := b.sendMessage(chatid, msg.Username, fi.Comment); err != nil {
				b.Log.Errorf("posting file comment %s failed: %s", fi.Comment, err)
			}
		}
	}
	return ""
}

func (b *Btelegram) handleQuote(message, quoteNick, quoteMessage string) string {
	format := b.GetString("quoteformat")
	if format == "" {
		format = "{MESSAGE} (re @{QUOTENICK}: {QUOTEMESSAGE})"
	}
	format = strings.Replace(format, "{MESSAGE}", message, -1)
	format = strings.Replace(format, "{QUOTENICK}", quoteNick, -1)
	format = strings.Replace(format, "{QUOTEMESSAGE}", quoteMessage, -1)
	return format
}

// handleEntities handles messageEntities
func (b *Btelegram) handleEntities(rmsg *config.Message, message *tgbotapi.Message) {
	if message.Entities == nil {
		return
	}
	// for now only do URL replacements
	for _, e := range *message.Entities {
		if e.Type == "text_link" {
			url, err := e.ParseURL()
			if err != nil {
				b.Log.Errorf("entity text_link url parse failed: %s", err)
				continue
			}
			link := rmsg.Text[e.Offset : e.Offset+e.Length]
			rmsg.Text = strings.Replace(rmsg.Text, link, url.String(), 1)
		}
	}
}
<|endoftext|>


<commit_before>package model

import (
	"github.com/Konstantin8105/GoFea/input/element"
	"github.com/Konstantin8105/GoFea/input/point"
	"github.com/Konstantin8105/GoFea/solver/dof"
	"github.com/Konstantin8105/GoFea/solver/finiteElement"
)

// generateDof - create degree's of freedom for model
func (m *Dim2) generateDof() {

	// Generate degreeForPoint - degree of freedom in global system for each point of finite element
	{
		pointIndexes := make([]point.Index, len(m.points))
		for i := range m.points {
			pointIndexes = append(pointIndexes, m.points[i].Index)
		}
		m.degreeForPoint = dof.DoF{
			DofArray:  pointIndexes,
			Dimension: dof.Dim2d,
		}
	}

	// Generate degreeInGlobalMatrix - degree of freedom for creating global matrix of stiffnes, mass, ...
	{
		var axes []dof.AxeNumber
		for _, ele := range m.elements {
			switch ele.(type) {
			case element.Beam:
				beam := ele.(element.Beam)
				fe := m.getFiniteElement(beam.GetIndex())
				_, localAxes := finiteElement.GetStiffinerGlobalK(fe, &m.degreeForPoint, finiteElement.WithoutZeroStiffiner)
				axes = append(axes, localAxes...)
			default:
				panic("")
			}
		}
		dof.UniqueAxeNumber(&axes)
		m.degreeInGlobalMatrix = axes
	}

	// Create convertor axe number to position in global matrix
	m.indexsInGlobalMatrix = dof.NewMapIndex(&m.degreeInGlobalMatrix)
}
<commit_msg>remove panic<commit_after>package model

import (
	"github.com/Konstantin8105/GoFea/input/point"
	"github.com/Konstantin8105/GoFea/solver/dof"
	"github.com/Konstantin8105/GoFea/solver/finiteElement"
)

// generateDof - create degree's of freedom for model
func (m *Dim2) generateDof() {

	// Generate degreeForPoint - degree of freedom in global system for each point of finite element
	{
		pointIndexes := make([]point.Index, len(m.points))
		for i := range m.points {
			pointIndexes = append(pointIndexes, m.points[i].Index)
		}
		m.degreeForPoint = dof.DoF{
			DofArray:  pointIndexes,
			Dimension: dof.Dim2d,
		}
	}

	// Generate degreeInGlobalMatrix - degree of freedom for creating global matrix of stiffnes, mass, ...
	{
		var axes []dof.AxeNumber
		for _, ele := range m.elements {
			fe := m.getFiniteElement(ele.GetIndex())
			_, localAxes := finiteElement.GetStiffinerGlobalK(fe, &m.degreeForPoint, finiteElement.WithoutZeroStiffiner)
			axes = append(axes, localAxes...)
		}
		dof.UniqueAxeNumber(&axes)
		m.degreeInGlobalMatrix = axes
	}

	// Create convertor axe number to position in global matrix
	m.indexsInGlobalMatrix = dof.NewMapIndex(&m.degreeInGlobalMatrix)
}
<|endoftext|>


<commit_before>// +build live

package bubbles

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"testing"
	"time"
)

const (
	addr  = "localhost:9200"
	index = "bubbles"
)

func TestLiveIndex(t *testing.T) {
	b, err := New([]string{addr}, 2, 10*time.Millisecond)
	if err != nil {
		t.Fatal(err.Error())
	}

	ins := Action{
		Type: Index,
		MetaData: MetaData{
			Index: index,
			Type:  "type1",
			ID:    "1",
		},
		Document: `{"field1": "value1"}`,
	}

	b.Enqueue(&ins)
	time.Sleep(1000 * time.Millisecond)
	pending := b.Stop()
	if have, want := len(pending), 0; have != want {
		t.Fatalf("have %d, want %d: %v", have, want, pending)
	}
}

func TestLiveIndexError(t *testing.T) {
	// Index with errors.

	b, err := New([]string{addr}, 2, 10*time.Millisecond)
	if err != nil {
		t.Fatal(err.Error())
	}

	ins1 := Action{
		Type: Index,
		MetaData: MetaData{
			Index: index,
			Type:  "type1",
			ID:    "1",
		},
		Document: `{"field1": "value1"}`,
	}
	ins2 := Action{
		Type: Index,
		MetaData: MetaData{
			Index: index,
			Type:  "type1",
			ID:    "2",
		},
		Document: `{"field1": `, // <-- invalid!
	}

	b.Enqueue(&ins1)
	b.Enqueue(&ins2)
	time.Sleep(1000 * time.Millisecond)
	pending := b.Stop()
	if have, want := len(pending), 0; have != want {
		t.Fatalf("have %d, want %d: %v", have, want, pending)
	}
}

func TestLiveMany(t *testing.T) {
	b, err := New([]string{addr}, 2, 10*time.Millisecond)
	if err != nil {
		t.Fatal(err.Error())
	}
	go func() {
		for e := range b.Errors() {
			fmt.Printf("error: %v\n", e)
		}
	}()

	var (
		clients = 10
		// documents = 1000
	)
	req, err := http.NewRequest("DELETE", fmt.Sprintf("http://%s/%s", addr, index), nil)
	if err != nil {
		panic(err)
	}
	res, err := http.DefaultClient.Do(req)
	if err != nil {
		panic(err)
	}

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		panic(err)
	}
	fmt.Printf("res: %d - %s\n", res.StatusCode, string(body))

	for i := 0; i < clients; i++ {
		ins := Action{
			Type: Create,
			MetaData: MetaData{
				Index: index,
				Type:  "type1",
				ID:    "1",
			},
			Document: `{"field1": "value1"}`,
		}
		b.Enqueue(&ins)
	}

	time.Sleep(1000 * time.Millisecond)
	fmt.Printf("go stop\n")
	pending := b.Stop()
	fmt.Printf("done stop\n")
	if have, want := len(pending), 0; have != want {
		t.Fatalf("have %d, want %d: %v", have, want, pending)
	}
}
<commit_msg>live test compile fixes<commit_after>// +build live

package bubbles

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"testing"
	"time"
)

const (
	addr  = "localhost:9200"
	index = "bubbles"
)

func TestLiveIndex(t *testing.T) {
	b, err := New([]string{addr}, OptFlush(10*time.Millisecond))
	if err != nil {
		t.Fatal(err.Error())
	}

	ins := Action{
		Type: Index,
		MetaData: MetaData{
			Index: index,
			Type:  "type1",
			ID:    "1",
		},
		Document: `{"field1": "value1"}`,
	}

	b.Enqueue(&ins)
	time.Sleep(1000 * time.Millisecond)
	pending := b.Stop()
	if have, want := len(pending), 0; have != want {
		t.Fatalf("have %d, want %d: %v", have, want, pending)
	}
}

func TestLiveIndexError(t *testing.T) {
	// Index with errors.

	b, err := New([]string{addr}, OptFlush(10*time.Millisecond))
	if err != nil {
		t.Fatal(err.Error())
	}

	ins1 := Action{
		Type: Index,
		MetaData: MetaData{
			Index: index,
			Type:  "type1",
			ID:    "1",
		},
		Document: `{"field1": "value1"}`,
	}
	ins2 := Action{
		Type: Index,
		MetaData: MetaData{
			Index: index,
			Type:  "type1",
			ID:    "2",
		},
		Document: `{"field1": `, // <-- invalid!
	}

	b.Enqueue(&ins1)
	b.Enqueue(&ins2)
	time.Sleep(1000 * time.Millisecond)
	pending := b.Stop()
	if have, want := len(pending), 0; have != want {
		t.Fatalf("have %d, want %d: %v", have, want, pending)
	}
}

func TestLiveMany(t *testing.T) {
	b, err := New([]string{addr}, OptFlush(10*time.Millisecond))
	if err != nil {
		t.Fatal(err.Error())
	}
	go func() {
		for e := range b.Errors() {
			fmt.Printf("error: %v\n", e)
		}
	}()

	var (
		clients = 10
		// documents = 1000
	)
	req, err := http.NewRequest("DELETE", fmt.Sprintf("http://%s/%s", addr, index), nil)
	if err != nil {
		panic(err)
	}
	res, err := http.DefaultClient.Do(req)
	if err != nil {
		panic(err)
	}

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		panic(err)
	}
	fmt.Printf("res: %d - %s\n", res.StatusCode, string(body))

	for i := 0; i < clients; i++ {
		ins := Action{
			Type: Create,
			MetaData: MetaData{
				Index: index,
				Type:  "type1",
				ID:    "1",
			},
			Document: `{"field1": "value1"}`,
		}
		b.Enqueue(&ins)
	}

	time.Sleep(1000 * time.Millisecond)
	fmt.Printf("go stop\n")
	pending := b.Stop()
	fmt.Printf("done stop\n")
	if have, want := len(pending), 0; have != want {
		t.Fatalf("have %d, want %d: %v", have, want, pending)
	}
}
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build windows

package svc_test

import (
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"golang.org/x/sys/windows/svc"
	"golang.org/x/sys/windows/svc/mgr"
)

func getState(t *testing.T, s *mgr.Service) svc.State {
	status, err := s.Query()
	if err != nil {
		t.Fatalf("Query(%s) failed: %s", s.Name, err)
	}
	return status.State
}

func testState(t *testing.T, s *mgr.Service, want svc.State) {
	have := getState(t, s)
	if have != want {
		t.Fatalf("%s state is=%d want=%d", s.Name, have, want)
	}
}

func waitState(t *testing.T, s *mgr.Service, want svc.State) {
	for i := 0; ; i++ {
		have := getState(t, s)
		if have == want {
			return
		}
		if i > 10 {
			t.Fatalf("%s state is=%d, waiting timeout", s.Name, have)
		}
		time.Sleep(300 * time.Millisecond)
	}
}

// stopAndDeleteIfInstalled stops and deletes service name,
// if the service is running and / or installed.
func stopAndDeleteIfInstalled(t *testing.T, m *mgr.Mgr, name string) {
	s, err := m.OpenService(name)
	if err != nil {
		// Service is not installed.
		return

	}
	defer s.Close()

	// Make sure the service is not running, otherwise we won't be able to delete it.
	if getState(t, s) == svc.Running {
		_, err = s.Control(svc.Stop)
		if err != nil {
			t.Fatalf("Control(%s) failed: %s", s.Name, err)
		}
		waitState(t, s, svc.Stopped)
	}

	err = s.Delete()
	if err != nil {
		t.Fatalf("Delete failed: %s", err)
	}
}

func TestExample(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping test in short mode - it modifies system services")
	}

	const name = "myservice"

	m, err := mgr.Connect()
	if err != nil {
		t.Fatalf("SCM connection failed: %s", err)
	}
	defer m.Disconnect()

	dir, err := ioutil.TempDir("", "svc")
	if err != nil {
		t.Fatalf("failed to create temp directory: %v", err)
	}
	defer os.RemoveAll(dir)

	exepath := filepath.Join(dir, "a.exe")
	o, err := exec.Command("go", "build", "-o", exepath, "golang.org/x/sys/windows/svc/example").CombinedOutput()
	if err != nil {
		t.Fatalf("failed to build service program: %v\n%v", err, string(o))
	}

	stopAndDeleteIfInstalled(t, m, name)

	s, err := m.CreateService(name, exepath, mgr.Config{DisplayName: "my service"}, "is", "auto-started")
	if err != nil {
		t.Fatalf("CreateService(%s) failed: %v", name, err)
	}
	defer s.Close()

	args := []string{"is", "manual-started", fmt.Sprintf("%d", rand.Int())}

	testState(t, s, svc.Stopped)
	err = s.Start(args...)
	if err != nil {
		t.Fatalf("Start(%s) failed: %s", s.Name, err)
	}
	waitState(t, s, svc.Running)
	time.Sleep(1 * time.Second)

	// testing deadlock from issues 4.
	_, err = s.Control(svc.Interrogate)
	if err != nil {
		t.Fatalf("Control(%s) failed: %s", s.Name, err)
	}
	_, err = s.Control(svc.Interrogate)
	if err != nil {
		t.Fatalf("Control(%s) failed: %s", s.Name, err)
	}
	time.Sleep(1 * time.Second)

	_, err = s.Control(svc.Stop)
	if err != nil {
		t.Fatalf("Control(%s) failed: %s", s.Name, err)
	}
	waitState(t, s, svc.Stopped)

	err = s.Delete()
	if err != nil {
		t.Fatalf("Delete failed: %s", err)
	}

	out, err := exec.Command("wevtutil.exe", "qe", "Application", "/q:*[System[Provider[@Name='myservice']]]", "/rd:true", "/c:10").CombinedOutput()
	if err != nil {
		t.Fatalf("wevtutil failed: %v\n%v", err, string(out))
	}
	want := strings.Join(append([]string{name}, args...), "-")
	// Test context passing (see servicemain in sys_386.s and sys_amd64.s).
	want += "-123456"
	if !strings.Contains(string(out), want) {
		t.Errorf("%q string does not contain %q", string(out), want)
	}
}

func TestIsAnInteractiveSession(t *testing.T) {
	isInteractive, err := svc.IsAnInteractiveSession()
	if err != nil {
		t.Fatal(err)
	}
	if !isInteractive {
		t.Error("IsAnInteractiveSession retuns false when running interactively.")
	}
}

func TestIsWindowsService(t *testing.T) {
	isSvc, err := svc.IsWindowsService()
	if err != nil {
		t.Fatal(err)
	}
	if isSvc {
		t.Error("IsWindowsService retuns true when not running in a service.")
	}
}

func TestIsWindowsServiceWhenParentExits(t *testing.T) {
	if os.Getenv("GO_WANT_HELPER_PROCESS") == "parent" {
		// in parent process

		// Start the child and exit quickly.
		child := exec.Command(os.Args[0], "-test.run=TestIsWindowsServiceWhenParentExits")
		child.Env = append(os.Environ(), "GO_WANT_HELPER_PROCESS=child")
		err := child.Start()
		if err != nil {
			fmt.Fprintf(os.Stderr, fmt.Sprintf("child start failed: %v", err))
			os.Exit(1)
		}
		os.Exit(0)
	}

	if os.Getenv("GO_WANT_HELPER_PROCESS") == "child" {
		// in child process
		dumpPath := os.Getenv("GO_WANT_HELPER_PROCESS_FILE")
		if dumpPath == "" {
			// We cannot report this error. But main test will notice
			// that we did not create dump file.
			os.Exit(1)
		}
		var msg string
		isSvc, err := svc.IsWindowsService()
		if err != nil {
			msg = err.Error()
		}
		if isSvc {
			msg = "IsWindowsService retuns true when not running in a service."
		}
		err = os.WriteFile(dumpPath, []byte(msg), 0644)
		if err != nil {
			// We cannot report this error. But main test will notice
			// that we did not create dump file.
			os.Exit(2)
		}
		os.Exit(0)
	}

	// Run in a loop until it fails.
	for i := 0; i < 10; i++ {
		childDumpPath := filepath.Join(t.TempDir(), "issvc.txt")

		parent := exec.Command(os.Args[0], "-test.run=TestIsWindowsServiceWhenParentExits")
		parent.Env = append(os.Environ(),
			"GO_WANT_HELPER_PROCESS=parent",
			"GO_WANT_HELPER_PROCESS_FILE="+childDumpPath)
		parentOutput, err := parent.CombinedOutput()
		if err != nil {
			t.Errorf("parent failed: %v: %v", err, string(parentOutput))
		}
		for i := 0; ; i++ {
			if _, err := os.Stat(childDumpPath); err == nil {
				break
			}
			time.Sleep(100 * time.Millisecond)
			if i > 10 {
				t.Fatal("timed out waiting for child ouput file to be created.")
			}
		}
		childOutput, err := ioutil.ReadFile(childDumpPath)
		if err != nil {
			t.Fatalf("reading child ouput failed: %v", err)
		}
		if got, want := string(childOutput), ""; got != want {
			t.Fatalf("child output: want %q, got %q", want, got)
		}
	}
}
<commit_msg>windows/svc: use ioutil.WriteFile instead of os.WriteFile<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build windows
// +build windows

package svc_test

import (
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"golang.org/x/sys/windows/svc"
	"golang.org/x/sys/windows/svc/mgr"
)

func getState(t *testing.T, s *mgr.Service) svc.State {
	status, err := s.Query()
	if err != nil {
		t.Fatalf("Query(%s) failed: %s", s.Name, err)
	}
	return status.State
}

func testState(t *testing.T, s *mgr.Service, want svc.State) {
	have := getState(t, s)
	if have != want {
		t.Fatalf("%s state is=%d want=%d", s.Name, have, want)
	}
}

func waitState(t *testing.T, s *mgr.Service, want svc.State) {
	for i := 0; ; i++ {
		have := getState(t, s)
		if have == want {
			return
		}
		if i > 10 {
			t.Fatalf("%s state is=%d, waiting timeout", s.Name, have)
		}
		time.Sleep(300 * time.Millisecond)
	}
}

// stopAndDeleteIfInstalled stops and deletes service name,
// if the service is running and / or installed.
func stopAndDeleteIfInstalled(t *testing.T, m *mgr.Mgr, name string) {
	s, err := m.OpenService(name)
	if err != nil {
		// Service is not installed.
		return

	}
	defer s.Close()

	// Make sure the service is not running, otherwise we won't be able to delete it.
	if getState(t, s) == svc.Running {
		_, err = s.Control(svc.Stop)
		if err != nil {
			t.Fatalf("Control(%s) failed: %s", s.Name, err)
		}
		waitState(t, s, svc.Stopped)
	}

	err = s.Delete()
	if err != nil {
		t.Fatalf("Delete failed: %s", err)
	}
}

func TestExample(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping test in short mode - it modifies system services")
	}

	const name = "myservice"

	m, err := mgr.Connect()
	if err != nil {
		t.Fatalf("SCM connection failed: %s", err)
	}
	defer m.Disconnect()

	dir, err := ioutil.TempDir("", "svc")
	if err != nil {
		t.Fatalf("failed to create temp directory: %v", err)
	}
	defer os.RemoveAll(dir)

	exepath := filepath.Join(dir, "a.exe")
	o, err := exec.Command("go", "build", "-o", exepath, "golang.org/x/sys/windows/svc/example").CombinedOutput()
	if err != nil {
		t.Fatalf("failed to build service program: %v\n%v", err, string(o))
	}

	stopAndDeleteIfInstalled(t, m, name)

	s, err := m.CreateService(name, exepath, mgr.Config{DisplayName: "my service"}, "is", "auto-started")
	if err != nil {
		t.Fatalf("CreateService(%s) failed: %v", name, err)
	}
	defer s.Close()

	args := []string{"is", "manual-started", fmt.Sprintf("%d", rand.Int())}

	testState(t, s, svc.Stopped)
	err = s.Start(args...)
	if err != nil {
		t.Fatalf("Start(%s) failed: %s", s.Name, err)
	}
	waitState(t, s, svc.Running)
	time.Sleep(1 * time.Second)

	// testing deadlock from issues 4.
	_, err = s.Control(svc.Interrogate)
	if err != nil {
		t.Fatalf("Control(%s) failed: %s", s.Name, err)
	}
	_, err = s.Control(svc.Interrogate)
	if err != nil {
		t.Fatalf("Control(%s) failed: %s", s.Name, err)
	}
	time.Sleep(1 * time.Second)

	_, err = s.Control(svc.Stop)
	if err != nil {
		t.Fatalf("Control(%s) failed: %s", s.Name, err)
	}
	waitState(t, s, svc.Stopped)

	err = s.Delete()
	if err != nil {
		t.Fatalf("Delete failed: %s", err)
	}

	out, err := exec.Command("wevtutil.exe", "qe", "Application", "/q:*[System[Provider[@Name='myservice']]]", "/rd:true", "/c:10").CombinedOutput()
	if err != nil {
		t.Fatalf("wevtutil failed: %v\n%v", err, string(out))
	}
	want := strings.Join(append([]string{name}, args...), "-")
	// Test context passing (see servicemain in sys_386.s and sys_amd64.s).
	want += "-123456"
	if !strings.Contains(string(out), want) {
		t.Errorf("%q string does not contain %q", string(out), want)
	}
}

func TestIsAnInteractiveSession(t *testing.T) {
	isInteractive, err := svc.IsAnInteractiveSession()
	if err != nil {
		t.Fatal(err)
	}
	if !isInteractive {
		t.Error("IsAnInteractiveSession retuns false when running interactively.")
	}
}

func TestIsWindowsService(t *testing.T) {
	isSvc, err := svc.IsWindowsService()
	if err != nil {
		t.Fatal(err)
	}
	if isSvc {
		t.Error("IsWindowsService retuns true when not running in a service.")
	}
}

func TestIsWindowsServiceWhenParentExits(t *testing.T) {
	if os.Getenv("GO_WANT_HELPER_PROCESS") == "parent" {
		// in parent process

		// Start the child and exit quickly.
		child := exec.Command(os.Args[0], "-test.run=TestIsWindowsServiceWhenParentExits")
		child.Env = append(os.Environ(), "GO_WANT_HELPER_PROCESS=child")
		err := child.Start()
		if err != nil {
			fmt.Fprintf(os.Stderr, fmt.Sprintf("child start failed: %v", err))
			os.Exit(1)
		}
		os.Exit(0)
	}

	if os.Getenv("GO_WANT_HELPER_PROCESS") == "child" {
		// in child process
		dumpPath := os.Getenv("GO_WANT_HELPER_PROCESS_FILE")
		if dumpPath == "" {
			// We cannot report this error. But main test will notice
			// that we did not create dump file.
			os.Exit(1)
		}
		var msg string
		isSvc, err := svc.IsWindowsService()
		if err != nil {
			msg = err.Error()
		}
		if isSvc {
			msg = "IsWindowsService retuns true when not running in a service."
		}
		err = ioutil.WriteFile(dumpPath, []byte(msg), 0644)
		if err != nil {
			// We cannot report this error. But main test will notice
			// that we did not create dump file.
			os.Exit(2)
		}
		os.Exit(0)
	}

	// Run in a loop until it fails.
	for i := 0; i < 10; i++ {
		childDumpPath := filepath.Join(t.TempDir(), "issvc.txt")

		parent := exec.Command(os.Args[0], "-test.run=TestIsWindowsServiceWhenParentExits")
		parent.Env = append(os.Environ(),
			"GO_WANT_HELPER_PROCESS=parent",
			"GO_WANT_HELPER_PROCESS_FILE="+childDumpPath)
		parentOutput, err := parent.CombinedOutput()
		if err != nil {
			t.Errorf("parent failed: %v: %v", err, string(parentOutput))
		}
		for i := 0; ; i++ {
			if _, err := os.Stat(childDumpPath); err == nil {
				break
			}
			time.Sleep(100 * time.Millisecond)
			if i > 10 {
				t.Fatal("timed out waiting for child ouput file to be created.")
			}
		}
		childOutput, err := ioutil.ReadFile(childDumpPath)
		if err != nil {
			t.Fatalf("reading child ouput failed: %v", err)
		}
		if got, want := string(childOutput), ""; got != want {
			t.Fatalf("child output: want %q, got %q", want, got)
		}
	}
}
<|endoftext|>


<commit_before><commit_msg>useless file<commit_after><|endoftext|>


<commit_before>package main

import (
	log "github.com/Sirupsen/logrus"
	"gopkg.in/vmihailenco/msgpack.v2"
	"cs262-project/common"
	"net"
	"sync"
)

var emptyList = []common.UUID{}

// Handles the subscriptions, updates, forwarding
type Broker struct {
	metadata *MetadataStore

	// map queries to clients
	subscriber_lock sync.RWMutex
	subscribers     map[string][]Client

	// map of producer ids to queries
	forwarding_lock sync.RWMutex
	forwarding      map[common.UUID]*queryList

	// index of producers
	producers_lock sync.RWMutex
	producers      map[common.UUID]*Producer

	// map query string to query struct
	query_lock sync.RWMutex
	queries    map[string]*Query
}

//TODO: config for broker?
func NewBroker(metadata *MetadataStore) *Broker {
	return &Broker{
		metadata:    metadata,
		subscribers: make(map[string][]Client),
		forwarding:  make(map[common.UUID]*queryList),
		producers:   make(map[common.UUID]*Producer),
		queries:     make(map[string]*Query),
	}
}

// safely adds entry to map[query][]Client map
func (b *Broker) mapQueryToClient(query string, c *Client) {
	b.subscriber_lock.Lock()
	if list, found := b.subscribers[query]; found {
		// check if we are already in the list
		for _, c2 := range list {
			if c2 == *c {
				b.subscriber_lock.Unlock()
				return
			}
		}
		// if we aren't in the list, but list exists, append to the end
		b.subscribers[query] = append(list, *c)
	} else {
		// otherwise, create a new list with us in it
		b.subscribers[query] = []Client{*c}
	}
	b.subscriber_lock.Unlock()
}

func (b *Broker) updateForwardingTable(query *Query) {
	b.forwarding_lock.Lock()
Loop:
	for producerID, _ := range query.MatchingProducers {
		if list, found := b.forwarding[producerID]; found {
			// check if we are already in the list
			for _, q2 := range *list {
				if q2 == query.Query {
					continue Loop
				}
			}
			log.WithFields(log.Fields{
				"producerID": producerID, "query": query.Query, "list": list,
			}).Debug("Adding query to existing query list")
			b.forwarding[producerID].addQuery(query.Query)
		} else {
			log.WithFields(log.Fields{
				"producerID": producerID, "query": query.Query,
			}).Debug("Adding query to NEW query list")
			b.forwarding[producerID] = &queryList{query.Query}
		}
	}
	log.Debugf("forwarding table %v", b.forwarding)
	b.forwarding_lock.Unlock()
}

// we have the list of new and removed UUIDs for a query,
// so we update the forwarding table to match that
func (b *Broker) updateForwardingDiffs(query *Query, added, removed []common.UUID) {
	var (
		list  *queryList
		found bool
	)
	if len(removed) > 0 {
		b.forwarding_lock.Lock()
		for _, rm_uuid := range removed {
			if list, found = b.forwarding[rm_uuid]; !found {
				// no subscribers for this uuid
				continue
			}
			for _, tmp_query := range *list {
				if tmp_query == query.Query {
					list.removeQuery(query.Query)
					continue
				}
			}
			b.forwarding[rm_uuid] = list // TODO ETK why is this step necessary?

		}
		b.forwarding_lock.Unlock()
	}
	b.updateForwardingTable(query)
}

func (b *Broker) ForwardMessage(m *common.Message) {
	var (
		matchingQueries *queryList
		found           bool
	)
	log.Debugf("forwarding msg? %v", m)
	b.forwarding_lock.RLock()
	// return if we can't find anyone to forward to
	matchingQueries, found = b.forwarding[m.UUID]
	b.forwarding_lock.RUnlock()

	if !found || (matchingQueries != nil && len(*matchingQueries) == 0) {
		log.Debugf("no forwarding targets")
		return
	}

	var clientList []Client
	for _, query := range *matchingQueries {
		b.subscriber_lock.RLock()
		clientList, found = b.subscribers[query]
		b.subscriber_lock.RUnlock()
		if !found || len(clientList) == 0 {
			log.Debugf("found no clients")
			break
		}
		for _, client := range clientList {
			go client.Send(m)
		}
	}
}

func (b *Broker) SendSubscriptionDiffs(query string, added, removed []common.UUID) {
	// if we don't do this, then empty lists show up as None
	// when we pack them
	if len(added) == 0 {
		added = emptyList
	}
	if len(removed) == 0 {
		removed = emptyList
	}
	msg := map[string][]common.UUID{"New": added, "Del": removed}
	// send to subscribers
	b.subscriber_lock.RLock()
	subscribers := b.subscribers[query]
	b.subscriber_lock.RUnlock()
	for _, sub := range subscribers {
		go sub.Send(msg)
	}
}

// Evaluates the query and establishes the forwarding decisions.
// Returns the client
func (b *Broker) NewSubscription(querystring string, conn net.Conn) *Client {
	var (
		query *Query
		found bool
		err   error
	)
	b.query_lock.RLock()
	query, found = b.queries[querystring]
	b.query_lock.RUnlock()

	if !found {
		// parse it!
		queryAST := Parse(querystring)
		query, err = b.metadata.Query(queryAST)
		if err != nil {
			log.WithFields(log.Fields{
				"error": err, "query": querystring,
			}).Error("Error evaluating mongo query")
		}

		// add to our map of queries
		b.query_lock.Lock()
		b.queries[querystring] = query
		b.query_lock.Unlock()

		log.WithFields(log.Fields{
			"query": querystring, "results": query.MatchingProducers,
		}).Debug("Evaluated query")
	}

	c := NewClient(querystring, &conn)
	go c.dosend()

	// set up forwarding for all initial producers
	b.updateForwardingTable(query)
	b.mapQueryToClient(querystring, c)
	c.Send(query.MatchingProducers)

	return c
}

func (b *Broker) HandleProducer(msg *common.Message, dec *msgpack.Decoder, conn net.Conn) {
	// use uuid to find old producer or create new one
	// add producer.C to a list of channels to select from
	// when we receive a message from a producer, save the
	// metadata and evaluate it, then forward it.
	// decode the incoming message
	var (
		err   error
		found bool
		p     *Producer
	)

	// save the metadata
	err = b.metadata.Save(msg)
	if err != nil {
		log.WithFields(log.Fields{
			"message": msg, "error": err,
		}).Error("Could not save metadata")
		conn.Close()
		return
	}

	// find the producer
	b.producers_lock.RLock()
	p, found = b.producers[msg.UUID]
	b.producers_lock.RUnlock()
	if !found {
		p = NewProducer(msg.UUID, dec)
	}
	b.RemapProducer(p, msg)

	// queue first message to be sent
	b.ForwardMessage(msg)

	go func(p *Producer) {
		for p.C != nil { // TODO ETK not sure what this for-loop achieves
			select {
			case <-p.stop:
				return
			case msg := <-p.C:
				if len(msg.Metadata) > 0 {
					err = b.metadata.Save(msg)
					b.RemapProducer(p, msg)
				}
				b.ForwardMessage(msg)
			}
		}
	}(p)
}

// 1. Firstly, have a method that given a producer (which is an implicit pointer to its
//    metadata) reevaluates all queries.

// when we receive a new producer, find related queries and update the forwarding
// tables that match. If newMetadata is nil, it will just reevaluate using current
// producer metadata across *all* queries, else it can use metadata in the provided
// Message to filter which queries to reevaluate
func (b *Broker) RemapProducer(p *Producer, newMetadata *common.Message) {
	//TODO: make a list of queries to update so that its unique THEN actually reevaluate
	//      to get added/removed lists. Once we have added/removed lists we use
	//      that to update the forwarding table.
	//      Question: Do i save the loop over to make OLD until after the forwarding table
	//      is updated? NO this should be encapsulated?
	var queriesToReevaluate = make(map[string]*Query)
	// TODO ETK this `true` shouldn't be here, right? Debugging purposes?
	if newMetadata == nil || true { // reevaluate ALL queries
		b.subscriber_lock.RLock()
		for querystring, _ := range b.subscribers {
			if _, found := queriesToReevaluate[querystring]; !found {
				b.query_lock.RLock()
				queriesToReevaluate[querystring] = b.queries[querystring]
				b.query_lock.RUnlock()
			}
		}
		b.subscriber_lock.RUnlock()

		for _, query := range queriesToReevaluate {
			added, removed := b.metadata.Reevaluate(query)
			log.WithFields(log.Fields{
				"query": query.Query, "added": added, "removed": removed,
			}).Info("Reevaluated query")
			log.Debugf("remapped? %v %v %v", p, query)
			b.updateForwardingDiffs(query, added, removed)
			b.SendSubscriptionDiffs(query.Query, added, removed)
		}
	}
}
<commit_msg>Fixed a small synchronization bug in the broker<commit_after>package main

import (
	log "github.com/Sirupsen/logrus"
	"gopkg.in/vmihailenco/msgpack.v2"
	"cs262-project/common"
	"net"
	"sync"
)

var emptyList = []common.UUID{}

// Handles the subscriptions, updates, forwarding
type Broker struct {
	metadata *MetadataStore

	// map queries to clients
	subscriber_lock sync.RWMutex
	subscribers     map[string][]Client

	// map of producer ids to queries
	forwarding_lock sync.RWMutex
	forwarding      map[common.UUID]*queryList

	// index of producers
	producers_lock sync.RWMutex
	producers      map[common.UUID]*Producer

	// map query string to query struct
	query_lock sync.RWMutex
	queries    map[string]*Query
}

//TODO: config for broker?
func NewBroker(metadata *MetadataStore) *Broker {
	return &Broker{
		metadata:    metadata,
		subscribers: make(map[string][]Client),
		forwarding:  make(map[common.UUID]*queryList),
		producers:   make(map[common.UUID]*Producer),
		queries:     make(map[string]*Query),
	}
}

// safely adds entry to map[query][]Client map
func (b *Broker) mapQueryToClient(query string, c *Client) {
	b.subscriber_lock.Lock()
	if list, found := b.subscribers[query]; found {
		// check if we are already in the list
		for _, c2 := range list {
			if c2 == *c {
				b.subscriber_lock.Unlock()
				return
			}
		}
		// if we aren't in the list, but list exists, append to the end
		b.subscribers[query] = append(list, *c)
	} else {
		// otherwise, create a new list with us in it
		b.subscribers[query] = []Client{*c}
	}
	b.subscriber_lock.Unlock()
}

func (b *Broker) updateForwardingTable(query *Query) {
	b.forwarding_lock.Lock()
Loop:
	for producerID, _ := range query.MatchingProducers {
		if list, found := b.forwarding[producerID]; found {
			// check if we are already in the list
			for _, q2 := range *list {
				if q2 == query.Query {
					continue Loop
				}
			}
			log.WithFields(log.Fields{
				"producerID": producerID, "query": query.Query, "list": list,
			}).Debug("Adding query to existing query list")
			b.forwarding[producerID].addQuery(query.Query)
		} else {
			log.WithFields(log.Fields{
				"producerID": producerID, "query": query.Query,
			}).Debug("Adding query to NEW query list")
			b.forwarding[producerID] = &queryList{query.Query}
		}
	}
	log.Debugf("forwarding table %v", b.forwarding)
	b.forwarding_lock.Unlock()
}

// we have the list of new and removed UUIDs for a query,
// so we update the forwarding table to match that
func (b *Broker) updateForwardingDiffs(query *Query, added, removed []common.UUID) {
	var (
		list  *queryList
		found bool
	)
	if len(removed) > 0 {
		b.forwarding_lock.Lock()
		for _, rm_uuid := range removed {
			if list, found = b.forwarding[rm_uuid]; !found {
				// no subscribers for this uuid
				continue
			}
			for _, tmp_query := range *list {
				if tmp_query == query.Query {
					list.removeQuery(query.Query)
					continue
				}
			}
			b.forwarding[rm_uuid] = list // TODO ETK why is this step necessary?

		}
		b.forwarding_lock.Unlock()
	}
	b.updateForwardingTable(query)
}

func (b *Broker) ForwardMessage(m *common.Message) {
	var (
		matchingQueries *queryList
		found           bool
	)
	log.Debugf("forwarding msg? %v", m)
	b.forwarding_lock.RLock()
	// return if we can't find anyone to forward to
	matchingQueries, found = b.forwarding[m.UUID]
	b.forwarding_lock.RUnlock()

	if !found || (matchingQueries != nil && len(*matchingQueries) == 0) {
		log.Debugf("no forwarding targets")
		return
	}

	var clientList []Client
	for _, query := range *matchingQueries {
		b.subscriber_lock.RLock()
		clientList, found = b.subscribers[query]
		b.subscriber_lock.RUnlock()
		if !found || len(clientList) == 0 {
			log.Debugf("found no clients")
			break
		}
		for _, client := range clientList {
			go client.Send(m)
		}
	}
}

func (b *Broker) SendSubscriptionDiffs(query string, added, removed []common.UUID) {
	// if we don't do this, then empty lists show up as None
	// when we pack them
	if len(added) == 0 {
		added = emptyList
	}
	if len(removed) == 0 {
		removed = emptyList
	}
	msg := map[string][]common.UUID{"New": added, "Del": removed}
	// send to subscribers
	b.subscriber_lock.RLock()
	subscribers := b.subscribers[query]
	b.subscriber_lock.RUnlock()
	for _, sub := range subscribers {
		go sub.Send(msg)
	}
}

// Evaluates the query and establishes the forwarding decisions.
// Returns the client
func (b *Broker) NewSubscription(querystring string, conn net.Conn) *Client {
	var (
		query *Query
		found bool
		err   error
	)
	b.query_lock.RLock()
	query, found = b.queries[querystring]
	b.query_lock.RUnlock()

	if !found {
		// parse it!
		queryAST := Parse(querystring)
		query, err = b.metadata.Query(queryAST)
		if err != nil {
			log.WithFields(log.Fields{
				"error": err, "query": querystring,
			}).Error("Error evaluating mongo query")
		}

		// add to our map of queries
		b.query_lock.Lock()
		b.queries[querystring] = query
		b.query_lock.Unlock()

		log.WithFields(log.Fields{
			"query": querystring, "results": query.MatchingProducers,
		}).Debug("Evaluated query")
	}

	c := NewClient(querystring, &conn)
	go c.dosend()

	// set up forwarding for all initial producers
	b.updateForwardingTable(query)
	b.mapQueryToClient(querystring, c)
	c.Send(query.MatchingProducers)

	return c
}

func (b *Broker) HandleProducer(msg *common.Message, dec *msgpack.Decoder, conn net.Conn) {
	// use uuid to find old producer or create new one
	// add producer.C to a list of channels to select from
	// when we receive a message from a producer, save the
	// metadata and evaluate it, then forward it.
	// decode the incoming message
	var (
		err   error
		found bool
		p     *Producer
	)

	// save the metadata
	err = b.metadata.Save(msg)
	if err != nil {
		log.WithFields(log.Fields{
			"message": msg, "error": err,
		}).Error("Could not save metadata")
		conn.Close()
		return
	}

	// find the producer
	b.producers_lock.RLock()
	p, found = b.producers[msg.UUID]
	b.producers_lock.RUnlock()
	if !found {
		p = NewProducer(msg.UUID, dec)
	}
	b.RemapProducer(p, msg)

	// queue first message to be sent
	b.ForwardMessage(msg)

	go func(p *Producer) {
		for p.C != nil { // TODO ETK not sure what this for-loop achieves
			select {
			case <-p.stop:
				return
			case msg := <-p.C:
				if len(msg.Metadata) > 0 {
					err = b.metadata.Save(msg)
					b.RemapProducer(p, msg)
				}
				b.ForwardMessage(msg)
			}
		}
	}(p)
}

// 1. Firstly, have a method that given a producer (which is an implicit pointer to its
//    metadata) reevaluates all queries.

// when we receive a new producer, find related queries and update the forwarding
// tables that match. If newMetadata is nil, it will just reevaluate using current
// producer metadata across *all* queries, else it can use metadata in the provided
// Message to filter which queries to reevaluate
func (b *Broker) RemapProducer(p *Producer, newMetadata *common.Message) {
	//TODO: make a list of queries to update so that its unique THEN actually reevaluate
	//      to get added/removed lists. Once we have added/removed lists we use
	//      that to update the forwarding table.
	//      Question: Do i save the loop over to make OLD until after the forwarding table
	//      is updated? NO this should be encapsulated?
	var queriesToReevaluate = make(map[string]*Query)
	// TODO ETK this `true` shouldn't be here, right? Debugging purposes?
	if newMetadata == nil || true { // reevaluate ALL queries
		b.subscriber_lock.RLock()
		for querystring, _ := range b.subscribers {
			if _, found := queriesToReevaluate[querystring]; !found {
				b.query_lock.RLock()
				queriesToReevaluate[querystring] = b.queries[querystring]
				b.query_lock.RUnlock()
			}
		}
		b.subscriber_lock.RUnlock()

		for _, query := range queriesToReevaluate {
			added, removed := b.metadata.Reevaluate(query)
			log.WithFields(log.Fields{
				"query": query.Query, "added": added, "removed": removed,
			}).Info("Reevaluated query")
			query.RWMutex.RLock()
			log.Debugf("remapped? %v %v %v", p, query)
			query.RWMutex.RUnlock()
			b.updateForwardingDiffs(query, added, removed)
			b.SendSubscriptionDiffs(query.Query, added, removed)
		}
	}
}
<|endoftext|>


<commit_before>package wakatime

import (
	"net/http"
)

type BasicTransport struct {
	apiKey    string
	Transport http.RoundTripper
}

func NewBasicTransport(apiKey string) *BasicTransport {
	return &BasicTransport{
		apiKey: apiKey,
	}
}

func (bt *BasicTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	req = cloneRequest(req)
	req.Header.Set("Authorization", "Basic "+bt.apiKey)
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("User-Agent", "go-wakatime/"+Version)

	// Make the HTTP request.
	return bt.transport().RoundTrip(req)
}

func (bt *BasicTransport) transport() http.RoundTripper {
	if bt.Transport != nil {
		return bt.Transport
	}
	return http.DefaultTransport
}

// cloneRequest returns a clone of the provided *http.Request.
// The clone is a shallow copy of the struct and its Header map.
func cloneRequest(r *http.Request) *http.Request {
	// shallow copy of the struct
	r2 := new(http.Request)
	*r2 = *r
	// deep copy of the Header
	r2.Header = make(http.Header)
	for k, s := range r.Header {
		r2.Header[k] = s
	}
	return r2
}
<commit_msg>Encode key<commit_after>package wakatime

import (
	"encoding/base64"
	"net/http"
)

type BasicTransport struct {
	encodedApiKey string
	Transport     http.RoundTripper
}

func NewBasicTransport(apiKey string) *BasicTransport {
	return &BasicTransport{
		encodedApiKey: base64.StdEncoding.EncodeToString([]byte(apiKey)),
	}
}

func (bt *BasicTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	req = cloneRequest(req)
	req.Header.Set("Authorization", "Basic "+bt.encodedApiKey)
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("User-Agent", "go-wakatime/"+Version)

	// Make the HTTP request.
	return bt.transport().RoundTrip(req)
}

func (bt *BasicTransport) transport() http.RoundTripper {
	if bt.Transport != nil {
		return bt.Transport
	}
	return http.DefaultTransport
}

// cloneRequest returns a clone of the provided *http.Request.
// The clone is a shallow copy of the struct and its Header map.
func cloneRequest(r *http.Request) *http.Request {
	// shallow copy of the struct
	r2 := new(http.Request)
	*r2 = *r
	// deep copy of the Header
	r2.Header = make(http.Header)
	for k, s := range r.Header {
		r2.Header[k] = s
	}
	return r2
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strconv"
	"unicode"
)

/* XXX eventually i'll need to modify these functions so they take a command struct or a variable
 * number of interfaces so that i can pass arguments.
 */

// normal commands
var normalFns map[int]func(*GlobalState) = map[int]func(*GlobalState){
	'j': normalj,
	'k': normalk,
	'l': normall,
	';': normalSemiColon,
	'p': normalp,
	'P': normalP, // will fix later
	'G': normalG,
	'u': normalu,
	'a': normala,
	'i': normali,
	'o': normalo,
	'O': normalO,
	// 'n':  nextBuffer,
	// 'p':  prevBuffer,
	':': normalColon,
	'-': normalMinus,
	'+': normalPlus,
	'#': normalHash,
	' ': normalSpace,
	'!': normalBang,
	'<': normalLShift,
	'>': normalRShift,
	'$': normalDollar,
	'0': normal0,
	// XXX why the fuck did i do these in hex?
	1: normalCtlA, // ^A
	2: normalCtlB, // ^B
	// 3: normalCtlC, // ^C
	4:  normalCtlD,   // ^D
	5:  normalCtlE,   // ^E
	6:  normalCtlF,   // ^F
	7:  normalCtlG,   // ^G
	8:  normalCtlH,   // ^H
	9:  normalCtlI,   // ^I
	10: normalCtlJ,   // ^J
	11: normalCtlK,   // ^K
	12: normalCtlL,   // ^L
	13: normalCtlM,   // ^M
	16: normalCtlP,   // ^P
	20: normalCtlT,   // ^T
	21: normalCtlU,   // ^U
	23: normalCtlW,   // ^W
	25: normalCtlY,   // ^Y
	26: normalCtlZ,   // ^Z
	29: normalCtlRSB, // ^] (right square bracket)
	// x: normalCtlCaret
	ESC: cmdClear,
}

func normalj(gs *GlobalState) {
	switch c := gs.curBuf(); b := c.(type) {
	case *EditBuffer:
		// left
		ln := b.line()
		if ln.cursor()-gs.n.cnt < 0 {
			ln.move(0)
			Beep()
		} else {
			ln.move(ln.cursor() - gs.n.cnt)
		}
	}
}

func normalk(gs *GlobalState) {
	switch c := gs.curBuf(); b := c.(type) {
	case *EditBuffer:
		// down
		if b.lno+gs.n.cnt < len(b.lines)-1 {
			b.lno += gs.n.cnt
		} else {
			b.lno = len(b.lines) - 1
			Beep()
		}

		// TODO column needs to be maintained for the down/up commands (even if the line you
		// move to is not long enough).
	}
}

func normall(gs *GlobalState) {
	switch c := gs.curBuf(); b := c.(type) {
	case *EditBuffer:
		// up
		if b.lno-gs.n.cnt > 0 {
			b.lno -= gs.n.cnt
		} else {
			b.lno = 0
			Beep()
		}
	}
}

func normalSemiColon(gs *GlobalState) {
	switch c := gs.curBuf(); b := c.(type) {
	case *EditBuffer:
		// right
		ln := b.line()
		if ln.cursor()+gs.n.cnt < len(ln.raw()) {
			ln.move(ln.cursor() + gs.n.cnt)
		} else {
			ln.move(len(ln.raw()) - 1)
			Beep()
		}
	}
}

func normalp(gs *GlobalState) {
	switch c := gs.curBuf(); b := c.(type) {
	case *EditBuffer:
		// paste
		gs.queueMessage(&Message{
			"paste.",
			false,
		})
	}
}

func normalP(gs *GlobalState) {
}

func normalG(gs *GlobalState) {
	switch c := gs.curBuf(); b := c.(type) {
	case *EditBuffer:
		b.lno = len(b.lines) - 1
	}
}

func normalu(gs *GlobalState) {
	switch c := gs.curBuf(); b := c.(type) {
	case *EditBuffer:
		// rewind
	}
}

func normala(gs *GlobalState) {
	switch c := gs.curBuf(); b := c.(type) {
	case *EditBuffer:
		// appendInput
		appendInsert(gs)
	}
}

func normali(gs *GlobalState) {
	switch c := gs.curBuf(); b := c.(type) {
	case *EditBuffer:
		insert(gs)
	}
}

func normalo(gs *GlobalState) {
	switch c := gs.curBuf(); b := c.(type) {
	case *EditBuffer:
		openInsert(gs)
	}
}

func normalO(gs *GlobalState) {
	switch c := gs.curBuf(); b := c.(type) {
	case *EditBuffer:
		aboveOpenInsert(gs)
	}
}

func normalColon(gs *GlobalState) {
	switch c := gs.curBuf(); b := c.(type) {
	case *EditBuffer:
		ex(gs)
	}
}

func normalMinus(gs *GlobalState) {
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

func normalPlus(gs *GlobalState) {
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

func normalHash(gs *GlobalState) {
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

func normalSpace(gs *GlobalState) {
	normalSemiColon(gs)
}

func normalBang(gs *GlobalState) {
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

func normalLShift(gs *GlobalState) {
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

func normalRShift(gs *GlobalState) {
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

func normalDollar(gs *GlobalState) {
	switch c := gs.curBuf(); b := c.(type) {
	case *EditBuffer:
		cnt := gs.n.cnt - 1
		if b.lno+cnt > len(b.lines)-1 {
			return
		}

		b.lno += cnt
		ln := b.line()
		ln.move(len(ln.raw()) - 1)
	}
}

func normal0(gs *GlobalState) {
}

func normalCtlA(gs *GlobalState) {
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

func normalCtlB(gs *GlobalState) {
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

func normalCtlD(gs *GlobalState) {
	switch c := gs.curBuf(); b := c.(type) {
	case *EditBuffer:
		// scroll down
	}
}

func normalCtlE(gs *GlobalState) {
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

func normalCtlF(gs *GlobalState) {
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

func normalCtlG(gs *GlobalState) {
	switch c := gs.curBuf(); b := c.(type) {
	case *EditBuffer:
		mod := "modified"
		if !b.isDirty() {
			mod = "un" + mod
		}
		// XXX This is actual not correct.  When the file is empty, we want to show "empty
		// file" rather than file position information.
		gs.queueMessage(&Message{
			fmt.Sprintf("%s: %s: line %d of %d [%d%]", b.ident(), mod, b.lno+1,
				len(b.lines), b.lno/len(b.lines)),
			false,
		})
	}
}

func normalCtlH(gs *GlobalState) {
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

func normalCtlI(gs *GlobalState) {
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

func normalCtlJ(gs *GlobalState) {
	normalj(gs)
}

func normalCtlK(gs *GlobalState) {
	normalk(gs)
}

func normalCtlL(gs *GlobalState) {
	// repaint
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

func normalCtlM(gs *GlobalState) {
	normalPlus(gs)
}

func normalCtlP(gs *GlobalState) {
	normalk(gs)
}

func normalCtlT(gs *GlobalState) {
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

func normalCtlU(gs *GlobalState) {
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

func normalCtlW(gs *GlobalState) {
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

// XXX ^y and ^z are fucked
func normalCtlY(gs *GlobalState) {
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

func normalCtlZ(gs *GlobalState) {
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

func normalCtlRSB(gs *GlobalState) {
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

func nextBuffer(gs *GlobalState) {
	r := gs.NextBuffer()
	gs.queueMessage(&Message{
		gs.curBuf().ident(),
		r == nil,
	})
}

func prevBuffer(gs *GlobalState) {
	r := gs.PrevBuffer()
	gs.queueMessage(&Message{
		gs.curBuf().ident(),
		r == nil,
	})
}

func cmdClear(gs *GlobalState) {
	gs.cmd = ""
	gs.n.cnt = 1
}

type Nm struct {
	buf string
	cnt int
}

// normal mode
func NormalMode(gs *GlobalState) {
	gs.Mode = MODENORMAL

	m := NewNormalModeline()
	gs.SetModeline(m)

	// advertise the current buffer
	gs.queueMessage(&Message{
		gs.curBuf().ident(),
		false,
	})

	gs.n = new(Nm)

	buf := ""
	gs.n.cnt = 1
	for {
		window := gs.Window
		window.PaintMapper(0, window.Rows-1, true)
		gs.UpdateCh <- 1
		k := <-gs.InputCh // screen.Window.Getch()

		if !unicode.IsDigit(k) {
			if len(buf) == 0 {
				gs.n.cnt = 1
				buf = string(k)
			} else {
				if cnt, e := strconv.Atoi(buf); e == nil {
					gs.n.cnt = cnt
					buf = string(k)
				}
			}

			if fn, ok := normalFns[k]; ok {
				fn(gs)
			}
			buf = ""
		} else {
			buf += string(k)
		}

		if gs.Mode != MODENORMAL {
			gs.Mode = MODENORMAL
			gs.SetModeline(m)
		}
		m.Key = k
		gs.curbuf.Value.(*EditBuffer).redraw = true
	}
}
<commit_msg>remove stale comment<commit_after>package main

import (
	"fmt"
	"strconv"
	"unicode"
)

/* XXX eventually i'll need to modify these functions so they take a command struct or a variable
 * number of interfaces so that i can pass arguments.
 */

// normal commands
var normalFns map[int]func(*GlobalState) = map[int]func(*GlobalState){
	'j': normalj,
	'k': normalk,
	'l': normall,
	';': normalSemiColon,
	'p': normalp,
	'P': normalP, // will fix later
	'G': normalG,
	'u': normalu,
	'a': normala,
	'i': normali,
	'o': normalo,
	'O': normalO,
	// 'n':  nextBuffer,
	// 'p':  prevBuffer,
	':': normalColon,
	'-': normalMinus,
	'+': normalPlus,
	'#': normalHash,
	' ': normalSpace,
	'!': normalBang,
	'<': normalLShift,
	'>': normalRShift,
	'$': normalDollar,
	'0': normal0,
	1:   normalCtlA, // ^A
	2:   normalCtlB, // ^B
	// 3: normalCtlC, // ^C
	4:  normalCtlD,   // ^D
	5:  normalCtlE,   // ^E
	6:  normalCtlF,   // ^F
	7:  normalCtlG,   // ^G
	8:  normalCtlH,   // ^H
	9:  normalCtlI,   // ^I
	10: normalCtlJ,   // ^J
	11: normalCtlK,   // ^K
	12: normalCtlL,   // ^L
	13: normalCtlM,   // ^M
	16: normalCtlP,   // ^P
	20: normalCtlT,   // ^T
	21: normalCtlU,   // ^U
	23: normalCtlW,   // ^W
	25: normalCtlY,   // ^Y
	26: normalCtlZ,   // ^Z
	29: normalCtlRSB, // ^] (right square bracket)
	// x: normalCtlCaret
	ESC: cmdClear,
}

func normalj(gs *GlobalState) {
	switch c := gs.curBuf(); b := c.(type) {
	case *EditBuffer:
		// left
		ln := b.line()
		if ln.cursor()-gs.n.cnt < 0 {
			ln.move(0)
			Beep()
		} else {
			ln.move(ln.cursor() - gs.n.cnt)
		}
	}
}

func normalk(gs *GlobalState) {
	switch c := gs.curBuf(); b := c.(type) {
	case *EditBuffer:
		// down
		if b.lno+gs.n.cnt < len(b.lines)-1 {
			b.lno += gs.n.cnt
		} else {
			b.lno = len(b.lines) - 1
			Beep()
		}

		// TODO column needs to be maintained for the down/up commands (even if the line you
		// move to is not long enough).
	}
}

func normall(gs *GlobalState) {
	switch c := gs.curBuf(); b := c.(type) {
	case *EditBuffer:
		// up
		if b.lno-gs.n.cnt > 0 {
			b.lno -= gs.n.cnt
		} else {
			b.lno = 0
			Beep()
		}
	}
}

func normalSemiColon(gs *GlobalState) {
	switch c := gs.curBuf(); b := c.(type) {
	case *EditBuffer:
		// right
		ln := b.line()
		if ln.cursor()+gs.n.cnt < len(ln.raw()) {
			ln.move(ln.cursor() + gs.n.cnt)
		} else {
			ln.move(len(ln.raw()) - 1)
			Beep()
		}
	}
}

func normalp(gs *GlobalState) {
	switch c := gs.curBuf(); b := c.(type) {
	case *EditBuffer:
		// paste
		gs.queueMessage(&Message{
			"paste.",
			false,
		})
	}
}

func normalP(gs *GlobalState) {
}

func normalG(gs *GlobalState) {
	switch c := gs.curBuf(); b := c.(type) {
	case *EditBuffer:
		b.lno = len(b.lines) - 1
	}
}

func normalu(gs *GlobalState) {
	switch c := gs.curBuf(); b := c.(type) {
	case *EditBuffer:
		// rewind
	}
}

func normala(gs *GlobalState) {
	switch c := gs.curBuf(); b := c.(type) {
	case *EditBuffer:
		// appendInput
		appendInsert(gs)
	}
}

func normali(gs *GlobalState) {
	switch c := gs.curBuf(); b := c.(type) {
	case *EditBuffer:
		insert(gs)
	}
}

func normalo(gs *GlobalState) {
	switch c := gs.curBuf(); b := c.(type) {
	case *EditBuffer:
		openInsert(gs)
	}
}

func normalO(gs *GlobalState) {
	switch c := gs.curBuf(); b := c.(type) {
	case *EditBuffer:
		aboveOpenInsert(gs)
	}
}

func normalColon(gs *GlobalState) {
	switch c := gs.curBuf(); b := c.(type) {
	case *EditBuffer:
		ex(gs)
	}
}

func normalMinus(gs *GlobalState) {
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

func normalPlus(gs *GlobalState) {
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

func normalHash(gs *GlobalState) {
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

func normalSpace(gs *GlobalState) {
	normalSemiColon(gs)
}

func normalBang(gs *GlobalState) {
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

func normalLShift(gs *GlobalState) {
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

func normalRShift(gs *GlobalState) {
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

func normalDollar(gs *GlobalState) {
	switch c := gs.curBuf(); b := c.(type) {
	case *EditBuffer:
		cnt := gs.n.cnt - 1
		if b.lno+cnt > len(b.lines)-1 {
			return
		}

		b.lno += cnt
		ln := b.line()
		ln.move(len(ln.raw()) - 1)
	}
}

func normal0(gs *GlobalState) {
}

func normalCtlA(gs *GlobalState) {
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

func normalCtlB(gs *GlobalState) {
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

func normalCtlD(gs *GlobalState) {
	switch c := gs.curBuf(); b := c.(type) {
	case *EditBuffer:
		// scroll down
	}
}

func normalCtlE(gs *GlobalState) {
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

func normalCtlF(gs *GlobalState) {
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

func normalCtlG(gs *GlobalState) {
	switch c := gs.curBuf(); b := c.(type) {
	case *EditBuffer:
		mod := "modified"
		if !b.isDirty() {
			mod = "un" + mod
		}
		// XXX This is actual not correct.  When the file is empty, we want to show "empty
		// file" rather than file position information.
		gs.queueMessage(&Message{
			fmt.Sprintf("%s: %s: line %d of %d [%d%]", b.ident(), mod, b.lno+1,
				len(b.lines), b.lno/len(b.lines)),
			false,
		})
	}
}

func normalCtlH(gs *GlobalState) {
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

func normalCtlI(gs *GlobalState) {
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

func normalCtlJ(gs *GlobalState) {
	normalj(gs)
}

func normalCtlK(gs *GlobalState) {
	normalk(gs)
}

func normalCtlL(gs *GlobalState) {
	// repaint
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

func normalCtlM(gs *GlobalState) {
	normalPlus(gs)
}

func normalCtlP(gs *GlobalState) {
	normalk(gs)
}

func normalCtlT(gs *GlobalState) {
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

func normalCtlU(gs *GlobalState) {
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

func normalCtlW(gs *GlobalState) {
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

// XXX ^y and ^z are fucked
func normalCtlY(gs *GlobalState) {
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

func normalCtlZ(gs *GlobalState) {
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

func normalCtlRSB(gs *GlobalState) {
	gs.queueMessage(&Message{
		"not implemented",
		true,
	})
}

func nextBuffer(gs *GlobalState) {
	r := gs.NextBuffer()
	gs.queueMessage(&Message{
		gs.curBuf().ident(),
		r == nil,
	})
}

func prevBuffer(gs *GlobalState) {
	r := gs.PrevBuffer()
	gs.queueMessage(&Message{
		gs.curBuf().ident(),
		r == nil,
	})
}

func cmdClear(gs *GlobalState) {
	gs.cmd = ""
	gs.n.cnt = 1
}

type Nm struct {
	buf string
	cnt int
}

// normal mode
func NormalMode(gs *GlobalState) {
	gs.Mode = MODENORMAL

	m := NewNormalModeline()
	gs.SetModeline(m)

	// advertise the current buffer
	gs.queueMessage(&Message{
		gs.curBuf().ident(),
		false,
	})

	gs.n = new(Nm)

	buf := ""
	gs.n.cnt = 1
	for {
		window := gs.Window
		window.PaintMapper(0, window.Rows-1, true)
		gs.UpdateCh <- 1
		k := <-gs.InputCh // screen.Window.Getch()

		if !unicode.IsDigit(k) {
			if len(buf) == 0 {
				gs.n.cnt = 1
				buf = string(k)
			} else {
				if cnt, e := strconv.Atoi(buf); e == nil {
					gs.n.cnt = cnt
					buf = string(k)
				}
			}

			if fn, ok := normalFns[k]; ok {
				fn(gs)
			}
			buf = ""
		} else {
			buf += string(k)
		}

		if gs.Mode != MODENORMAL {
			gs.Mode = MODENORMAL
			gs.SetModeline(m)
		}
		m.Key = k
		gs.curbuf.Value.(*EditBuffer).redraw = true
	}
}
<|endoftext|>


<commit_before>package gohl7

import (
	//"bitbucket.org/yehezkel/gohl7"
	//"testing"
	//"io/ioutil"
	//"os"
)

/*var(
	file_path = "test.hl7"
)

func BenchmarkLongMessage(b *testing.B){

	file, err := os.Open(file_path) // For read access.
	if err != nil {
		b.Fatal(err)
	}

	data,err := ioutil.ReadAll(file)

	//b.Fatalf("length %s\n",data[:10])

	if err != nil {
		b.Fatal(err)
	}
	buffer := make([]byte, len(data))

	//reset timmer
	b.ResetTimer()

	defer file.Close()

	for i := 0; i < b.N; i++{

		//skip copy
		b.StopTimer()
		_ = copy(buffer, data)
		b.StartTimer()

		parser, err := gohl7.NewParser(buffer)
		if err != nil{
			b.Fatal(err)
		}

		//this is the function where all the logic happends
		_, err = parser.Parse()
		if err != nil{
			b.Fatal(err)
		}
	}
}*/
<commit_msg>Activating the old benchmark<commit_after>package gohl7

import (
	//"bitbucket.org/yehezkel/gohl7"
	"testing"
	"io/ioutil"
	"os"
)

var(
	sampleFile = "test.hl7"
)

func BenchmarkLongMessage(b *testing.B){

	file, err := os.Open(sampleFile) // For read access.
	if err != nil {
		b.Fatal(err)
	}

	raw,err := ioutil.ReadAll(file)
	if err != nil {
		b.Fatal(err)
	}

    err = file.Close()

	//reset timmer
	b.ResetTimer()
    if err != nil {
        b.Fatal(err)
    }

	for i := 0; i < b.N; i++{

		parser, err := NewHl7Parser(raw)
		if err != nil{
			b.Fatal(err)
		}

		//this is the function where all the logic happends
	    _, err = parser.Parse()
		if err != nil{
			b.Fatal(err)
		}
	}
}
<|endoftext|>


<commit_before>package test

import (
	"math/rand"
	"sort"
	"testing"

	"github.com/fzandona/goroar"
)

var bitmapSize = 1000000

type Values []uint32

func (v Values) Len() int           { return len(v) }
func (v Values) Swap(i, j int)      { v[i], v[j] = v[j], v[i] }
func (v Values) Less(i, j int) bool { return v[i] < v[j] }

func getBuffer(size, seed int) []uint32 {
	rand.Seed(int64(seed))

	set := make(map[uint32]struct{})
	buffer := make([]uint32, 0, size)

	for len(set) < size {
		set[rand.Uint32()] = struct{}{}
	}
	for v, _ := range set {
		buffer = append(buffer, v)
	}
	return buffer
}

func BenchmarkAdd(b *testing.B) {
	rb := goroar.New()
	var pos int
	buffer := getBuffer(bitmapSize, 42)

	b.N = bitmapSize
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		rb.Add(buffer[pos])
		pos++
	}
}

func BenchmarkAddSorted(b *testing.B) {
	rb := goroar.New()
	var pos int
	buffer := getBuffer(bitmapSize, 42)
	sort.Sort(Values(buffer))

	b.N = bitmapSize
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		rb.Add(buffer[pos])
		pos++
	}
}

func BenchmarkIterator(b *testing.B) {
	rb := goroar.New()
	var pos int
	buffer := getBuffer(bitmapSize, 42)
	sort.Sort(Values(buffer))

	for _, v := range buffer {
		rb.Add(v)
	}

	b.ResetTimer()
	for v := range rb.Iterator() {
		if v > 0 {
		}
		pos++
	}
}

func BenchmarkContains(b *testing.B) {
	rb := goroar.New()
	var pos int
	buffer := getBuffer(bitmapSize, 42)
	sort.Sort(Values(buffer))

	for _, v := range buffer {
		rb.Add(v)
	}

	testBuffer := getBuffer(bitmapSize, 24)
	b.N = bitmapSize
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		rb.Contains(testBuffer[pos])
		pos++
	}
}

func getTwoRB(size1, seed1, size2, seed2 int) (*goroar.RoaringBitmap, *goroar.RoaringBitmap) {
	rb1 := goroar.New()
	rb2 := goroar.New()

	buffer := getBuffer(size1, seed1)
	sort.Sort(Values(buffer))
	for _, v := range buffer {
		rb1.Add(v)
	}

	buffer = getBuffer(size2, seed2)
	sort.Sort(Values(buffer))
	for _, v := range buffer {
		rb2.Add(v)
	}

	return rb1, rb2
}

func BenchmarkAndSameContent(b *testing.B) {
	rb1, rb2 := getTwoRB(bitmapSize, 42, bitmapSize, 42)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		rb1.And(rb2)
	}
}

func BenchmarkAndDifferentContent(b *testing.B) {
	rb1, rb2 := getTwoRB(bitmapSize, 42, bitmapSize, 24)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		rb1.And(rb2)
	}
}

func BenchmarkOrSameContent(b *testing.B) {
	rb1, rb2 := getTwoRB(bitmapSize, 42, bitmapSize, 42)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		rb1.Or(rb2)
	}
}

func BenchmarkOrDifferentContent(b *testing.B) {
	rb1, rb2 := getTwoRB(bitmapSize, 42, bitmapSize, 24)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		rb1.Or(rb2)
	}
}

func BenchmarkXorSameContent(b *testing.B) {
	rb1, rb2 := getTwoRB(bitmapSize, 42, bitmapSize, 42)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		rb1.Xor(rb2)
	}
}

func BenchmarkXorDifferentContent(b *testing.B) {
	rb1, rb2 := getTwoRB(bitmapSize, 42, bitmapSize, 24)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		rb1.Xor(rb2)
	}
}

func BenchmarkAndNotSameContent(b *testing.B) {
	rb1, rb2 := getTwoRB(bitmapSize, 42, bitmapSize, 42)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		rb1.AndNot(rb2)
	}
}

func BenchmarkAndNotDifferentContent(b *testing.B) {
	rb1, rb2 := getTwoRB(bitmapSize, 42, bitmapSize, 24)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		rb1.AndNot(rb2)
	}
}

func getTwoSimpleRB(size1, size2 int) (*goroar.RoaringBitmap, *goroar.RoaringBitmap) {
	rb1 := goroar.New()
	rb2 := goroar.New()

	for i := 0; i < size1; i++ {
		rb1.Add(uint32(i))
	}

	for i := 0; i < size2; i++ {
		rb2.Add(uint32(i))
	}

	return rb1, rb2
}

func BenchmarkAndSimpleContent(b *testing.B) {
	rb1, rb2 := getTwoSimpleRB(bitmapSize, bitmapSize)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		rb1.And(rb2)
	}
}

func BenchmarkOrSimpleContent(b *testing.B) {
	rb1, rb2 := getTwoSimpleRB(bitmapSize, bitmapSize)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		rb1.Or(rb2)
	}
}

func BenchmarkXorSimpleContent(b *testing.B) {
	rb1, rb2 := getTwoSimpleRB(bitmapSize, bitmapSize)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		rb1.Xor(rb2)
	}
}

func BenchmarkAndNotSimpleContent(b *testing.B) {
	rb1, rb2 := getTwoSimpleRB(bitmapSize, bitmapSize)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		rb1.AndNot(rb2)
	}
}
<commit_msg>Fixing goroar import.<commit_after>package goroar

import (
	"math/rand"
	"sort"
	"testing"
)

var bitmapSize = 1000000

type Values []uint32

func (v Values) Len() int           { return len(v) }
func (v Values) Swap(i, j int)      { v[i], v[j] = v[j], v[i] }
func (v Values) Less(i, j int) bool { return v[i] < v[j] }

func getBuffer(size, seed int) []uint32 {
	rand.Seed(int64(seed))

	set := make(map[uint32]struct{})
	buffer := make([]uint32, 0, size)

	for len(set) < size {
		set[rand.Uint32()] = struct{}{}
	}
	for v, _ := range set {
		buffer = append(buffer, v)
	}
	return buffer
}

func BenchmarkAdd(b *testing.B) {
	rb := New()
	var pos int
	buffer := getBuffer(bitmapSize, 42)

	b.N = bitmapSize
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		rb.Add(buffer[pos])
		pos++
	}
}

func BenchmarkAddSorted(b *testing.B) {
	rb := New()
	var pos int
	buffer := getBuffer(bitmapSize, 42)
	sort.Sort(Values(buffer))

	b.N = bitmapSize
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		rb.Add(buffer[pos])
		pos++
	}
}

func BenchmarkIterator(b *testing.B) {
	rb := New()
	var pos int
	buffer := getBuffer(bitmapSize, 42)
	sort.Sort(Values(buffer))

	for _, v := range buffer {
		rb.Add(v)
	}

	b.ResetTimer()
	for v := range rb.Iterator() {
		if v > 0 {
		}
		pos++
	}
}

func BenchmarkContains(b *testing.B) {
	rb := New()
	var pos int
	buffer := getBuffer(bitmapSize, 42)
	sort.Sort(Values(buffer))

	for _, v := range buffer {
		rb.Add(v)
	}

	testBuffer := getBuffer(bitmapSize, 24)
	b.N = bitmapSize
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		rb.Contains(testBuffer[pos])
		pos++
	}
}

func getTwoRB(size1, seed1, size2, seed2 int) (*RoaringBitmap, *RoaringBitmap) {
	rb1 := New()
	rb2 := New()

	buffer := getBuffer(size1, seed1)
	sort.Sort(Values(buffer))
	for _, v := range buffer {
		rb1.Add(v)
	}

	buffer = getBuffer(size2, seed2)
	sort.Sort(Values(buffer))
	for _, v := range buffer {
		rb2.Add(v)
	}

	return rb1, rb2
}

func BenchmarkAndSameContent(b *testing.B) {
	rb1, rb2 := getTwoRB(bitmapSize, 42, bitmapSize, 42)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		rb1.And(rb2)
	}
}

func BenchmarkAndDifferentContent(b *testing.B) {
	rb1, rb2 := getTwoRB(bitmapSize, 42, bitmapSize, 24)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		rb1.And(rb2)
	}
}

func BenchmarkOrSameContent(b *testing.B) {
	rb1, rb2 := getTwoRB(bitmapSize, 42, bitmapSize, 42)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		rb1.Or(rb2)
	}
}

func BenchmarkOrDifferentContent(b *testing.B) {
	rb1, rb2 := getTwoRB(bitmapSize, 42, bitmapSize, 24)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		rb1.Or(rb2)
	}
}

func BenchmarkXorSameContent(b *testing.B) {
	rb1, rb2 := getTwoRB(bitmapSize, 42, bitmapSize, 42)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		rb1.Xor(rb2)
	}
}

func BenchmarkXorDifferentContent(b *testing.B) {
	rb1, rb2 := getTwoRB(bitmapSize, 42, bitmapSize, 24)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		rb1.Xor(rb2)
	}
}

func BenchmarkAndNotSameContent(b *testing.B) {
	rb1, rb2 := getTwoRB(bitmapSize, 42, bitmapSize, 42)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		rb1.AndNot(rb2)
	}
}

func BenchmarkAndNotDifferentContent(b *testing.B) {
	rb1, rb2 := getTwoRB(bitmapSize, 42, bitmapSize, 24)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		rb1.AndNot(rb2)
	}
}

func getTwoSimpleRB(size1, size2 int) (*RoaringBitmap, *RoaringBitmap) {
	rb1 := New()
	rb2 := New()

	for i := 0; i < size1; i++ {
		rb1.Add(uint32(i))
	}

	for i := 0; i < size2; i++ {
		rb2.Add(uint32(i))
	}

	return rb1, rb2
}

func BenchmarkAndSimpleContent(b *testing.B) {
	rb1, rb2 := getTwoSimpleRB(bitmapSize, bitmapSize)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		rb1.And(rb2)
	}
}

func BenchmarkOrSimpleContent(b *testing.B) {
	rb1, rb2 := getTwoSimpleRB(bitmapSize, bitmapSize)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		rb1.Or(rb2)
	}
}

func BenchmarkXorSimpleContent(b *testing.B) {
	rb1, rb2 := getTwoSimpleRB(bitmapSize, bitmapSize)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		rb1.Xor(rb2)
	}
}

func BenchmarkAndNotSimpleContent(b *testing.B) {
	rb1, rb2 := getTwoSimpleRB(bitmapSize, bitmapSize)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		rb1.AndNot(rb2)
	}
}
<|endoftext|>


<commit_before>package dsbldr

import (
	"encoding/csv"
	"io/ioutil"
	"net/http"
)

// Builder is main type for this tool.
type Builder struct {
	BaseURL        string
	RequestHeaders map[string]string // Custom Request Headers including auth
	featureMap     map[string]*Feature
	data           [][]string // Strings of Data to be read in to CSV
	records        int        // Number of records to be retrieved for dataset
}

// NewBuilder creates new Builder struct
func NewBuilder(featureCount, recordCount int) *Builder {
	// Add extra row for header
	preallocatedData := make([][]string, recordCount+1)
	for i := range preallocatedData {
		preallocatedData[i] = make([]string, featureCount)
	}
	return &Builder{
		RequestHeaders: make(map[string]string),
		featureMap:     make(map[string]*Feature),
		data:           preallocatedData,
		records:        recordCount,
	}
}

func (b *Builder) addDataFeature(featureName string, values []string) error {
	writeStringColumn(&b.data, featureName, values)
	return nil
}

func (b *Builder) getDataFeature(featureName string) []string {
	items := make([]string, b.records-1) // all items excluding header row
	var colIndex int

	for i := range b.data[0] {
		// Find first column with same header
		if b.data[0][i] == featureName {
			colIndex = i
			break
		}
	}

	// Add all the values as well (remember that Builder.data is pre-allocated)
	for i := 1; i < len(b.data); i++ {
		items[i-1] = b.data[i][colIndex]
	}

	return items
}

// GetFeature returns a feature in the detaset based on it's name
func (b *Builder) GetFeature(name string) *Feature {
	var feat *Feature
	if val, ok := b.featureMap[name]; ok {
		feat = val
		return feat
	}
	return nil
}

// Save commits the downloaded features to a file
func (b *Builder) Save(writer csv.Writer) error {
	for i := range b.data {
		var record []string
		for index, j := range b.data[i] {
			// if data header (feature name) has noSave == false, else don't write
			if !b.featureMap[b.data[0][index]].noSave {
				record = append(record, j)
			}
		}
		err := writer.Write(record)
		if err != nil {
			return err
		}
	}
	return nil
}

// AddFeatures adds a Feature struct to the "Features" Field on Builder
func (b *Builder) AddFeatures(features ...*Feature) {
	for _, feature := range features {
		feature.noSave = false
		b.featureMap[feature.Name] = feature
		// TODO: Return error if feature with same name has been added
	}
	// Increase size of data matrix if feature map is larger than initially allocated
	if len(b.featureMap) > len(b.data[0]) {
		for i := range b.data {
			b.data[i] = append(b.data[i], "")
		}
	}
}

func (b *Builder) createRequest(
	endpoint string,
	headers map[string]string,
) (*http.Request, error) {

	req, err := http.NewRequest("GET", b.BaseURL+endpoint, nil)
	if err != nil {
		return nil, err
	}

	for k, v := range headers {
		req.Header.Add(k, v)
	}
	return req, nil
}

// resolve endpoint templates to actual values for endpoints based on parent features
func (b *Builder) resolveFeatureEndpoints(feature *Feature) ([]string, error) {
	endpoints := make([]string, b.records)
	parents, err := feature.getParentNames()
	if err != nil {
		return nil, err
	}

	for i := 0; i < b.records; i++ {
		parentValuesMap := make(map[string]string)
		for _, j := range parents {
			parentValuesMap[j] = b.getDataFeature(j)[i]
		}

	}

	return endpoints, nil
}

// retrieveFeature returns string dumps of responses and an error if any
func (b *Builder) retrieveFeature(feature *Feature) ([]string, error) {
	client := &http.Client{}
	responses := make([]*http.Response, b.records)
	responseDumps := make([]string, b.records)
	endpoints, err := b.resolveFeatureEndpoints(feature)
	if err != nil {
		return nil, err
	}

	for i := 0; i < b.records; i++ {
		// endpoints should be the same as the number of
		req, err := b.createRequest(endpoints[i], b.RequestHeaders)
		if err != nil {
			return nil, err
		}

		resp, err := client.Do(req)
		if err != nil {
			return nil, err
		}

		defer resp.Body.Close()

		if resp.StatusCode == 200 { // OK
			bodyBytes, _ := ioutil.ReadAll(resp.Body)
			responseDumps[i] = string(bodyBytes)
		}
	}

	return responseDumps, nil
}

// Run Builder to aggregate all features and manage concurrent operations
func (b *Builder) Run() error {
	// flag.Parse()

	// Set noSave to true for any features listed in the call
	// for _, i := range ignoredFeatures {
	// 	b.GetFeature(i).noSave = true
	// }

	for _, feature := range b.featureMap {
		parents, err := feature.getParentNames()
		if err != nil {
			return err
		}

		var retrieveError error
		go func(feature *Feature) {
			for _, i := range parents {
				// Loop through parent features and wait till
				// they finish excecuting
				<-b.GetFeature(i).finished
			}

			parsedResponses := make([]string, b.records)
			parsedResponses, retrieveError = b.retrieveFeature(feature)

			output := feature.RunFunc(parsedResponses)
			b.addDataFeature(feature.Name, output)

			feature.finished <- true // Mark feature as true
		}(feature)

		if retrieveError != nil {
			return retrieveError
		}

	}
	return nil
}
<commit_msg>Finish implementing Builder.resolveFeatureEndpoints<commit_after>package dsbldr

import (
	"encoding/csv"
	"io/ioutil"
	"net/http"
)

// Builder is main type for this tool.
type Builder struct {
	BaseURL        string
	RequestHeaders map[string]string // Custom Request Headers including auth
	featureMap     map[string]*Feature
	data           [][]string // Strings of Data to be read in to CSV
	records        int        // Number of records to be retrieved for dataset
}

// NewBuilder creates new Builder struct
func NewBuilder(featureCount, recordCount int) *Builder {
	// Add extra row for header
	preallocatedData := make([][]string, recordCount+1)
	for i := range preallocatedData {
		preallocatedData[i] = make([]string, featureCount)
	}
	return &Builder{
		RequestHeaders: make(map[string]string),
		featureMap:     make(map[string]*Feature),
		data:           preallocatedData,
		records:        recordCount,
	}
}

func (b *Builder) addDataFeature(featureName string, values []string) error {
	writeStringColumn(&b.data, featureName, values)
	return nil
}

func (b *Builder) getDataFeature(featureName string) []string {
	items := make([]string, b.records-1) // all items excluding header row
	var colIndex int

	for i := range b.data[0] {
		// Find first column with same header
		if b.data[0][i] == featureName {
			colIndex = i
			break
		}
	}

	// Add all the values as well (remember that Builder.data is pre-allocated)
	for i := 1; i < len(b.data); i++ {
		items[i-1] = b.data[i][colIndex]
	}

	return items
}

// GetFeature returns a feature in the detaset based on it's name
func (b *Builder) GetFeature(name string) *Feature {
	var feat *Feature
	if val, ok := b.featureMap[name]; ok {
		feat = val
		return feat
	}
	return nil
}

// Save commits the downloaded features to a file
func (b *Builder) Save(writer csv.Writer) error {
	for i := range b.data {
		var record []string
		for index, j := range b.data[i] {
			// if data header (feature name) has noSave == false, else don't write
			if !b.featureMap[b.data[0][index]].noSave {
				record = append(record, j)
			}
		}
		err := writer.Write(record)
		if err != nil {
			return err
		}
	}
	return nil
}

// AddFeatures adds a Feature struct to the "Features" Field on Builder
func (b *Builder) AddFeatures(features ...*Feature) {
	for _, feature := range features {
		feature.noSave = false
		b.featureMap[feature.Name] = feature
		// TODO: Return error if feature with same name has been added
	}
	// Increase size of data matrix if feature map is larger than initially allocated
	if len(b.featureMap) > len(b.data[0]) {
		for i := range b.data {
			b.data[i] = append(b.data[i], "")
		}
	}
}

func (b *Builder) createRequest(
	endpoint string,
	headers map[string]string,
) (*http.Request, error) {

	req, err := http.NewRequest("GET", b.BaseURL+endpoint, nil)
	if err != nil {
		return nil, err
	}

	for k, v := range headers {
		req.Header.Add(k, v)
	}
	return req, nil
}

// resolve endpoint templates to actual values for endpoints based on parent features
func (b *Builder) resolveFeatureEndpoints(feature *Feature) ([]string, error) {
	endpoints := make([]string, b.records)
	parents, err := feature.getParentNames()
	if err != nil {
		return nil, err
	}

	for i := 0; i < b.records; i++ {
		parentValuesMap := make(map[string]string)
		for _, j := range parents {
			parentValuesMap[j] = b.getDataFeature(j)[i]
		}
		var resolveEndpointError error
		endpoints[i], resolveEndpointError = feature.resolveEndpoint(parentValuesMap)
		if resolveEndpointError != nil {
			return nil, resolveEndpointError
		}
	}

	return endpoints, nil
}

// retrieveFeature returns string dumps of responses and an error if any
func (b *Builder) retrieveFeature(feature *Feature) ([]string, error) {
	client := &http.Client{}
	responses := make([]*http.Response, b.records)
	responseDumps := make([]string, b.records)
	endpoints, err := b.resolveFeatureEndpoints(feature)
	if err != nil {
		return nil, err
	}

	for i := 0; i < b.records; i++ {
		// endpoints should be the same as the number of
		req, err := b.createRequest(endpoints[i], b.RequestHeaders)
		if err != nil {
			return nil, err
		}

		resp, err := client.Do(req)
		if err != nil {
			return nil, err
		}

		defer resp.Body.Close()

		if resp.StatusCode == 200 { // OK
			bodyBytes, _ := ioutil.ReadAll(resp.Body)
			responseDumps[i] = string(bodyBytes)
		}
	}

	return responseDumps, nil
}

// Run Builder to aggregate all features and manage concurrent operations
func (b *Builder) Run() error {
	// flag.Parse()

	// Set noSave to true for any features listed in the call
	// for _, i := range ignoredFeatures {
	// 	b.GetFeature(i).noSave = true
	// }

	for _, feature := range b.featureMap {
		parents, err := feature.getParentNames()
		if err != nil {
			return err
		}

		var retrieveError error
		go func(feature *Feature) {
			for _, i := range parents {
				// Loop through parent features and wait till
				// they finish excecuting
				<-b.GetFeature(i).finished
			}

			parsedResponses := make([]string, b.records)
			parsedResponses, retrieveError = b.retrieveFeature(feature)

			output := feature.RunFunc(parsedResponses)
			b.addDataFeature(feature.Name, output)

			feature.finished <- true // Mark feature as true
		}(feature)

		if retrieveError != nil {
			return retrieveError
		}

	}
	return nil
}
<|endoftext|>