<commit_before>package bxmpp

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
	"sync"
	"time"

	"github.com/42wim/matterbridge/bridge"
	"github.com/42wim/matterbridge/bridge/config"
	"github.com/42wim/matterbridge/bridge/helper"
	"github.com/jpillora/backoff"
	"github.com/matterbridge/go-xmpp"
	"github.com/rs/xid"
)

type Bxmpp struct {
	*bridge.Config

	startTime time.Time
	xc        *xmpp.Client
	xmppMap   map[string]string
	connected bool
	sync.RWMutex

	avatarAvailability map[string]bool
	avatarMap          map[string]string
}

func New(cfg *bridge.Config) bridge.Bridger {
	return &Bxmpp{
		Config:             cfg,
		xmppMap:            make(map[string]string),
		avatarAvailability: make(map[string]bool),
		avatarMap:          make(map[string]string),
	}
}

func (b *Bxmpp) Connect() error {
	b.Log.Infof("Connecting %s", b.GetString("Server"))
	if err := b.createXMPP(); err != nil {
		b.Log.Debugf("%#v", err)
		return err
	}

	b.Log.Info("Connection succeeded")
	go b.manageConnection()
	return nil
}

func (b *Bxmpp) Disconnect() error {
	return nil
}

func (b *Bxmpp) JoinChannel(channel config.ChannelInfo) error {
	if channel.Options.Key != "" {
		b.Log.Debugf("using key %s for channel %s", channel.Options.Key, channel.Name)
		b.xc.JoinProtectedMUC(channel.Name+"@"+b.GetString("Muc"), b.GetString("Nick"), channel.Options.Key, xmpp.NoHistory, 0, nil)
	} else {
		b.xc.JoinMUCNoHistory(channel.Name+"@"+b.GetString("Muc"), b.GetString("Nick"))
	}
	return nil
}

func (b *Bxmpp) Send(msg config.Message) (string, error) {
	// should be fixed by using a cache instead of dropping
	if !b.Connected() {
		return "", fmt.Errorf("bridge %s not connected, dropping message %#v to bridge", b.Account, msg)
	}
	// ignore delete messages
	if msg.Event == config.EventMsgDelete {
		return "", nil
	}

	b.Log.Debugf("=> Receiving %#v", msg)

	if msg.Event == config.EventAvatarDownload {
		return b.cacheAvatar(&msg), nil
	}

	// Make a action /me of the message, prepend the username with it.
	// https://xmpp.org/extensions/xep-0245.html
	if msg.Event == config.EventUserAction {
		msg.Username = "/me " + msg.Username
	}

	// Upload a file (in XMPP case send the upload URL because XMPP has no native upload support).
	var err error
	if msg.Extra != nil {
		for _, rmsg := range helper.HandleExtra(&msg, b.General) {
			b.Log.Debugf("=> Sending attachement message %#v", rmsg)
			if b.GetString("WebhookURL") != "" {
				err = b.postSlackCompatibleWebhook(msg)
			} else {
				_, err = b.xc.Send(xmpp.Chat{
					Type:   "groupchat",
					Remote: rmsg.Channel + "@" + b.GetString("Muc"),
					Text:   rmsg.Username + rmsg.Text,
				})
			}

			if err != nil {
				b.Log.WithError(err).Error("Unable to send message with share URL.")
			}
		}
		if len(msg.Extra["file"]) > 0 {
			return "", b.handleUploadFile(&msg)
		}
	}

	if b.GetString("WebhookURL") != "" {
		b.Log.Debugf("Sending message using Webhook")
		err := b.postSlackCompatibleWebhook(msg)
		if err != nil {
			b.Log.Errorf("Failed to send message using webhook: %s", err)
			return "", err
		}

		return "", nil
	}

	// Post normal message.
	var msgReplaceID string
	msgID := xid.New().String()
	if msg.ID != "" {
		msgID = msg.ID
		msgReplaceID = msg.ID
	}
	b.Log.Debugf("=> Sending message %#v", msg)
	if _, err := b.xc.Send(xmpp.Chat{
		Type:      "groupchat",
		Remote:    msg.Channel + "@" + b.GetString("Muc"),
		Text:      msg.Username + msg.Text,
		ID:        msgID,
		ReplaceID: msgReplaceID,
	}); err != nil {
		return "", err
	}
	return msgID, nil
}

func (b *Bxmpp) postSlackCompatibleWebhook(msg config.Message) error {
	type XMPPWebhook struct {
		Username string `json:"username"`
		Text     string `json:"text"`
	}
	webhookBody, err := json.Marshal(XMPPWebhook{
		Username: msg.Username,
		Text:     msg.Text,
	})
	if err != nil {
		b.Log.Errorf("Failed to marshal webhook: %s", err)
		return err
	}

	resp, err := http.Post(b.GetString("WebhookURL")+"/"+msg.Channel, "application/json", bytes.NewReader(webhookBody))
	resp.Body.Close()
	return err
}

func (b *Bxmpp) createXMPP() error {
	if !strings.Contains(b.GetString("Jid"), "@") {
		return fmt.Errorf("the Jid %s doesn't contain an @", b.GetString("Jid"))
	}
	tc := &tls.Config{
		ServerName:         strings.Split(b.GetString("Jid"), "@")[1],
		InsecureSkipVerify: b.GetBool("SkipTLSVerify"), // nolint: gosec
	}

	xmpp.DebugWriter = b.Log.Writer()

	options := xmpp.Options{
		Host:                         b.GetString("Server"),
		User:                         b.GetString("Jid"),
		Password:                     b.GetString("Password"),
		NoTLS:                        true,
		StartTLS:                     !b.GetBool("NoTLS"),
		TLSConfig:                    tc,
		Debug:                        b.GetBool("debug"),
		Session:                      true,
		Status:                       "",
		StatusMessage:                "",
		Resource:                     "",
		InsecureAllowUnencryptedAuth: b.GetBool("NoTLS"),
	}
	var err error
	b.xc, err = options.NewClient()
	return err
}

func (b *Bxmpp) manageConnection() {
	b.setConnected(true)
	initial := true
	bf := &backoff.Backoff{
		Min:    time.Second,
		Max:    5 * time.Minute,
		Jitter: true,
	}

	// Main connection loop. Each iteration corresponds to a successful
	// connection attempt and the subsequent handling of the connection.
	for {
		if initial {
			initial = false
		} else {
			b.Remote <- config.Message{
				Username: "system",
				Text:     "rejoin",
				Channel:  "",
				Account:  b.Account,
				Event:    config.EventRejoinChannels,
			}
		}

		if err := b.handleXMPP(); err != nil {
			b.Log.WithError(err).Error("Disconnected.")
			b.setConnected(false)
		}

		// Reconnection loop using an exponential back-off strategy. We
		// only break out of the loop if we have successfully reconnected.
		for {
			d := bf.Duration()
			b.Log.Infof("Reconnecting in %s.", d)
			time.Sleep(d)

			b.Log.Infof("Reconnecting now.")
			if err := b.createXMPP(); err == nil {
				b.setConnected(true)
				bf.Reset()
				break
			}
			b.Log.Warn("Failed to reconnect.")
		}
	}
}

func (b *Bxmpp) xmppKeepAlive() chan bool {
	done := make(chan bool)
	go func() {
		ticker := time.NewTicker(90 * time.Second)
		defer ticker.Stop()
		for {
			select {
			case <-ticker.C:
				b.Log.Debugf("PING")
				if err := b.xc.PingC2S("", ""); err != nil {
					b.Log.Debugf("PING failed %#v", err)
				}
			case <-done:
				return
			}
		}
	}()
	return done
}

func (b *Bxmpp) handleXMPP() error {
	b.startTime = time.Now()

	done := b.xmppKeepAlive()
	defer close(done)

	for {
		m, err := b.xc.Recv()
		if err != nil {
			return err
		}

		switch v := m.(type) {
		case xmpp.Chat:
			if v.Type == "groupchat" {
				b.Log.Debugf("== Receiving %#v", v)

				// Skip invalid messages.
				if b.skipMessage(v) {
					continue
				}

				var event string
				if strings.Contains(v.Text, "has set the subject to:") {
					event = config.EventTopicChange
				}

				available, sok := b.avatarAvailability[v.Remote]
				avatar := ""
				if !sok {
					b.Log.Debugf("Requesting avatar data")
					b.avatarAvailability[v.Remote] = false
					b.xc.AvatarRequestData(v.Remote)
				} else if available {
					avatar = getAvatar(b.avatarMap, v.Remote, b.General)
				}

				msgID := v.ID
				if v.ReplaceID != "" {
					msgID = v.ReplaceID
				}
				rmsg := config.Message{
					Username: b.parseNick(v.Remote),
					Text:     v.Text,
					Channel:  b.parseChannel(v.Remote),
					Account:  b.Account,
					Avatar:   avatar,
					UserID:   v.Remote,
					ID:       msgID,
					Event:    event,
				}

				// Check if we have an action event.
				var ok bool
				rmsg.Text, ok = b.replaceAction(rmsg.Text)
				if ok {
					rmsg.Event = config.EventUserAction
				}

				b.Log.Debugf("<= Sending message from %s on %s to gateway", rmsg.Username, b.Account)
				b.Log.Debugf("<= Message is %#v", rmsg)
				b.Remote <- rmsg
			}
		case xmpp.AvatarData:
			b.handleDownloadAvatar(v)
			b.avatarAvailability[v.From] = true
			b.Log.Debugf("Avatar for %s is now available", v.From)
		case xmpp.Presence:
			// Do nothing.
		}
	}
}

func (b *Bxmpp) replaceAction(text string) (string, bool) {
	if strings.HasPrefix(text, "/me ") {
		return strings.Replace(text, "/me ", "", -1), true
	}
	return text, false
}

// handleUploadFile handles native upload of files
func (b *Bxmpp) handleUploadFile(msg *config.Message) error {
	var urlDesc string

	for _, file := range msg.Extra["file"] {
		fileInfo := file.(config.FileInfo)
		if fileInfo.Comment != "" {
			msg.Text += fileInfo.Comment + ": "
		}
		if fileInfo.URL != "" {
			msg.Text = fileInfo.URL
			if fileInfo.Comment != "" {
				msg.Text = fileInfo.Comment + ": " + fileInfo.URL
				urlDesc = fileInfo.Comment
			}
		}
		if _, err := b.xc.Send(xmpp.Chat{
			Type:   "groupchat",
			Remote: msg.Channel + "@" + b.GetString("Muc"),
			Text:   msg.Username + msg.Text,
		}); err != nil {
			return err
		}

		if fileInfo.URL != "" {
			if _, err := b.xc.SendOOB(xmpp.Chat{
				Type:    "groupchat",
				Remote:  msg.Channel + "@" + b.GetString("Muc"),
				Ooburl:  fileInfo.URL,
				Oobdesc: urlDesc,
			}); err != nil {
				b.Log.WithError(err).Warn("Failed to send share URL.")
			}
		}
	}
	return nil
}

func (b *Bxmpp) parseNick(remote string) string {
	s := strings.Split(remote, "@")
	if len(s) > 0 {
		s = strings.Split(s[1], "/")
		if len(s) == 2 {
			return s[1] // nick
		}
	}
	return ""
}

func (b *Bxmpp) parseChannel(remote string) string {
	s := strings.Split(remote, "@")
	if len(s) >= 2 {
		return s[0] // channel
	}
	return ""
}

// skipMessage skips messages that need to be skipped
func (b *Bxmpp) skipMessage(message xmpp.Chat) bool {
	// skip messages from ourselves
	if b.parseNick(message.Remote) == b.GetString("Nick") {
		return true
	}

	// skip empty messages
	if message.Text == "" {
		return true
	}

	// skip subject messages
	if strings.Contains(message.Text, "</subject>") {
		return true
	}

	// do not show subjects on connect #732
	if strings.Contains(message.Text, "has set the subject to:") && time.Since(b.startTime) < time.Second*5 {
		return true
	}

	// Ignore messages posted by our webhook
	if b.GetString("WebhookURL") != "" && strings.Contains(message.ID, "webhookbot") {
		return true
	}

	// skip delayed messages
	return !message.Stamp.IsZero() && time.Since(message.Stamp).Minutes() > 5
}

func (b *Bxmpp) setConnected(state bool) {
	b.Lock()
	b.connected = state
	defer b.Unlock()
}

func (b *Bxmpp) Connected() bool {
	b.RLock()
	defer b.RUnlock()
	return b.connected
}
<commit_msg>Fix panic when the webhook fails (xmpp) (#1401)<commit_after>package bxmpp

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
	"sync"
	"time"

	"github.com/42wim/matterbridge/bridge"
	"github.com/42wim/matterbridge/bridge/config"
	"github.com/42wim/matterbridge/bridge/helper"
	"github.com/jpillora/backoff"
	"github.com/matterbridge/go-xmpp"
	"github.com/rs/xid"
)

type Bxmpp struct {
	*bridge.Config

	startTime time.Time
	xc        *xmpp.Client
	xmppMap   map[string]string
	connected bool
	sync.RWMutex

	avatarAvailability map[string]bool
	avatarMap          map[string]string
}

func New(cfg *bridge.Config) bridge.Bridger {
	return &Bxmpp{
		Config:             cfg,
		xmppMap:            make(map[string]string),
		avatarAvailability: make(map[string]bool),
		avatarMap:          make(map[string]string),
	}
}

func (b *Bxmpp) Connect() error {
	b.Log.Infof("Connecting %s", b.GetString("Server"))
	if err := b.createXMPP(); err != nil {
		b.Log.Debugf("%#v", err)
		return err
	}

	b.Log.Info("Connection succeeded")
	go b.manageConnection()
	return nil
}

func (b *Bxmpp) Disconnect() error {
	return nil
}

func (b *Bxmpp) JoinChannel(channel config.ChannelInfo) error {
	if channel.Options.Key != "" {
		b.Log.Debugf("using key %s for channel %s", channel.Options.Key, channel.Name)
		b.xc.JoinProtectedMUC(channel.Name+"@"+b.GetString("Muc"), b.GetString("Nick"), channel.Options.Key, xmpp.NoHistory, 0, nil)
	} else {
		b.xc.JoinMUCNoHistory(channel.Name+"@"+b.GetString("Muc"), b.GetString("Nick"))
	}
	return nil
}

func (b *Bxmpp) Send(msg config.Message) (string, error) {
	// should be fixed by using a cache instead of dropping
	if !b.Connected() {
		return "", fmt.Errorf("bridge %s not connected, dropping message %#v to bridge", b.Account, msg)
	}
	// ignore delete messages
	if msg.Event == config.EventMsgDelete {
		return "", nil
	}

	b.Log.Debugf("=> Receiving %#v", msg)

	if msg.Event == config.EventAvatarDownload {
		return b.cacheAvatar(&msg), nil
	}

	// Make a action /me of the message, prepend the username with it.
	// https://xmpp.org/extensions/xep-0245.html
	if msg.Event == config.EventUserAction {
		msg.Username = "/me " + msg.Username
	}

	// Upload a file (in XMPP case send the upload URL because XMPP has no native upload support).
	var err error
	if msg.Extra != nil {
		for _, rmsg := range helper.HandleExtra(&msg, b.General) {
			b.Log.Debugf("=> Sending attachement message %#v", rmsg)
			if b.GetString("WebhookURL") != "" {
				err = b.postSlackCompatibleWebhook(msg)
			} else {
				_, err = b.xc.Send(xmpp.Chat{
					Type:   "groupchat",
					Remote: rmsg.Channel + "@" + b.GetString("Muc"),
					Text:   rmsg.Username + rmsg.Text,
				})
			}

			if err != nil {
				b.Log.WithError(err).Error("Unable to send message with share URL.")
			}
		}
		if len(msg.Extra["file"]) > 0 {
			return "", b.handleUploadFile(&msg)
		}
	}

	if b.GetString("WebhookURL") != "" {
		b.Log.Debugf("Sending message using Webhook")
		err := b.postSlackCompatibleWebhook(msg)
		if err != nil {
			b.Log.Errorf("Failed to send message using webhook: %s", err)
			return "", err
		}

		return "", nil
	}

	// Post normal message.
	var msgReplaceID string
	msgID := xid.New().String()
	if msg.ID != "" {
		msgID = msg.ID
		msgReplaceID = msg.ID
	}
	b.Log.Debugf("=> Sending message %#v", msg)
	if _, err := b.xc.Send(xmpp.Chat{
		Type:      "groupchat",
		Remote:    msg.Channel + "@" + b.GetString("Muc"),
		Text:      msg.Username + msg.Text,
		ID:        msgID,
		ReplaceID: msgReplaceID,
	}); err != nil {
		return "", err
	}
	return msgID, nil
}

func (b *Bxmpp) postSlackCompatibleWebhook(msg config.Message) error {
	type XMPPWebhook struct {
		Username string `json:"username"`
		Text     string `json:"text"`
	}
	webhookBody, err := json.Marshal(XMPPWebhook{
		Username: msg.Username,
		Text:     msg.Text,
	})
	if err != nil {
		b.Log.Errorf("Failed to marshal webhook: %s", err)
		return err
	}

	resp, err := http.Post(b.GetString("WebhookURL")+"/"+msg.Channel, "application/json", bytes.NewReader(webhookBody))
	if err != nil {
		b.Log.Errorf("Failed to POST webhook: %s", err)
		return err
	}

	resp.Body.Close()
	return nil
}

func (b *Bxmpp) createXMPP() error {
	if !strings.Contains(b.GetString("Jid"), "@") {
		return fmt.Errorf("the Jid %s doesn't contain an @", b.GetString("Jid"))
	}
	tc := &tls.Config{
		ServerName:         strings.Split(b.GetString("Jid"), "@")[1],
		InsecureSkipVerify: b.GetBool("SkipTLSVerify"), // nolint: gosec
	}

	xmpp.DebugWriter = b.Log.Writer()

	options := xmpp.Options{
		Host:                         b.GetString("Server"),
		User:                         b.GetString("Jid"),
		Password:                     b.GetString("Password"),
		NoTLS:                        true,
		StartTLS:                     !b.GetBool("NoTLS"),
		TLSConfig:                    tc,
		Debug:                        b.GetBool("debug"),
		Session:                      true,
		Status:                       "",
		StatusMessage:                "",
		Resource:                     "",
		InsecureAllowUnencryptedAuth: b.GetBool("NoTLS"),
	}
	var err error
	b.xc, err = options.NewClient()
	return err
}

func (b *Bxmpp) manageConnection() {
	b.setConnected(true)
	initial := true
	bf := &backoff.Backoff{
		Min:    time.Second,
		Max:    5 * time.Minute,
		Jitter: true,
	}

	// Main connection loop. Each iteration corresponds to a successful
	// connection attempt and the subsequent handling of the connection.
	for {
		if initial {
			initial = false
		} else {
			b.Remote <- config.Message{
				Username: "system",
				Text:     "rejoin",
				Channel:  "",
				Account:  b.Account,
				Event:    config.EventRejoinChannels,
			}
		}

		if err := b.handleXMPP(); err != nil {
			b.Log.WithError(err).Error("Disconnected.")
			b.setConnected(false)
		}

		// Reconnection loop using an exponential back-off strategy. We
		// only break out of the loop if we have successfully reconnected.
		for {
			d := bf.Duration()
			b.Log.Infof("Reconnecting in %s.", d)
			time.Sleep(d)

			b.Log.Infof("Reconnecting now.")
			if err := b.createXMPP(); err == nil {
				b.setConnected(true)
				bf.Reset()
				break
			}
			b.Log.Warn("Failed to reconnect.")
		}
	}
}

func (b *Bxmpp) xmppKeepAlive() chan bool {
	done := make(chan bool)
	go func() {
		ticker := time.NewTicker(90 * time.Second)
		defer ticker.Stop()
		for {
			select {
			case <-ticker.C:
				b.Log.Debugf("PING")
				if err := b.xc.PingC2S("", ""); err != nil {
					b.Log.Debugf("PING failed %#v", err)
				}
			case <-done:
				return
			}
		}
	}()
	return done
}

func (b *Bxmpp) handleXMPP() error {
	b.startTime = time.Now()

	done := b.xmppKeepAlive()
	defer close(done)

	for {
		m, err := b.xc.Recv()
		if err != nil {
			return err
		}

		switch v := m.(type) {
		case xmpp.Chat:
			if v.Type == "groupchat" {
				b.Log.Debugf("== Receiving %#v", v)

				// Skip invalid messages.
				if b.skipMessage(v) {
					continue
				}

				var event string
				if strings.Contains(v.Text, "has set the subject to:") {
					event = config.EventTopicChange
				}

				available, sok := b.avatarAvailability[v.Remote]
				avatar := ""
				if !sok {
					b.Log.Debugf("Requesting avatar data")
					b.avatarAvailability[v.Remote] = false
					b.xc.AvatarRequestData(v.Remote)
				} else if available {
					avatar = getAvatar(b.avatarMap, v.Remote, b.General)
				}

				msgID := v.ID
				if v.ReplaceID != "" {
					msgID = v.ReplaceID
				}
				rmsg := config.Message{
					Username: b.parseNick(v.Remote),
					Text:     v.Text,
					Channel:  b.parseChannel(v.Remote),
					Account:  b.Account,
					Avatar:   avatar,
					UserID:   v.Remote,
					ID:       msgID,
					Event:    event,
				}

				// Check if we have an action event.
				var ok bool
				rmsg.Text, ok = b.replaceAction(rmsg.Text)
				if ok {
					rmsg.Event = config.EventUserAction
				}

				b.Log.Debugf("<= Sending message from %s on %s to gateway", rmsg.Username, b.Account)
				b.Log.Debugf("<= Message is %#v", rmsg)
				b.Remote <- rmsg
			}
		case xmpp.AvatarData:
			b.handleDownloadAvatar(v)
			b.avatarAvailability[v.From] = true
			b.Log.Debugf("Avatar for %s is now available", v.From)
		case xmpp.Presence:
			// Do nothing.
		}
	}
}

func (b *Bxmpp) replaceAction(text string) (string, bool) {
	if strings.HasPrefix(text, "/me ") {
		return strings.Replace(text, "/me ", "", -1), true
	}
	return text, false
}

// handleUploadFile handles native upload of files
func (b *Bxmpp) handleUploadFile(msg *config.Message) error {
	var urlDesc string

	for _, file := range msg.Extra["file"] {
		fileInfo := file.(config.FileInfo)
		if fileInfo.Comment != "" {
			msg.Text += fileInfo.Comment + ": "
		}
		if fileInfo.URL != "" {
			msg.Text = fileInfo.URL
			if fileInfo.Comment != "" {
				msg.Text = fileInfo.Comment + ": " + fileInfo.URL
				urlDesc = fileInfo.Comment
			}
		}
		if _, err := b.xc.Send(xmpp.Chat{
			Type:   "groupchat",
			Remote: msg.Channel + "@" + b.GetString("Muc"),
			Text:   msg.Username + msg.Text,
		}); err != nil {
			return err
		}

		if fileInfo.URL != "" {
			if _, err := b.xc.SendOOB(xmpp.Chat{
				Type:    "groupchat",
				Remote:  msg.Channel + "@" + b.GetString("Muc"),
				Ooburl:  fileInfo.URL,
				Oobdesc: urlDesc,
			}); err != nil {
				b.Log.WithError(err).Warn("Failed to send share URL.")
			}
		}
	}
	return nil
}

func (b *Bxmpp) parseNick(remote string) string {
	s := strings.Split(remote, "@")
	if len(s) > 0 {
		s = strings.Split(s[1], "/")
		if len(s) == 2 {
			return s[1] // nick
		}
	}
	return ""
}

func (b *Bxmpp) parseChannel(remote string) string {
	s := strings.Split(remote, "@")
	if len(s) >= 2 {
		return s[0] // channel
	}
	return ""
}

// skipMessage skips messages that need to be skipped
func (b *Bxmpp) skipMessage(message xmpp.Chat) bool {
	// skip messages from ourselves
	if b.parseNick(message.Remote) == b.GetString("Nick") {
		return true
	}

	// skip empty messages
	if message.Text == "" {
		return true
	}

	// skip subject messages
	if strings.Contains(message.Text, "</subject>") {
		return true
	}

	// do not show subjects on connect #732
	if strings.Contains(message.Text, "has set the subject to:") && time.Since(b.startTime) < time.Second*5 {
		return true
	}

	// Ignore messages posted by our webhook
	if b.GetString("WebhookURL") != "" && strings.Contains(message.ID, "webhookbot") {
		return true
	}

	// skip delayed messages
	return !message.Stamp.IsZero() && time.Since(message.Stamp).Minutes() > 5
}

func (b *Bxmpp) setConnected(state bool) {
	b.Lock()
	b.connected = state
	defer b.Unlock()
}

func (b *Bxmpp) Connected() bool {
	b.RLock()
	defer b.RUnlock()
	return b.connected
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kepctl

import (
	"fmt"
	"regexp"

	"github.com/pkg/errors"

	"k8s.io/enhancements/pkg/kepval/keps"
	"k8s.io/enhancements/pkg/kepval/keps/validations"
)

var (
	// SupportedOutputOpts stores all allowed query output formats
	SupportedOutputOpts = []string{
		"table",
		"json",
		"yaml",
	}

	// DefaultOutputOpt is the default output format for kepctl query
	DefaultOutputOpt = "table"

	StructuredOutputFormats = []string{
		"json",
		"yaml",
	}
)

type QueryOpts struct {
	CommonArgs
	SIG         []string
	Status      []string
	Stage       []string
	PRRApprover []string
	IncludePRs  bool
	Output      string
}

// Validate checks the args and cleans them up if needed
func (c *QueryOpts) Validate(args []string) error {
	if len(c.SIG) > 0 {
		sigs, err := selectByRegexp(validations.Sigs(), c.SIG)
		if err != nil {
			return err
		}
		if len(sigs) == 0 {
			return fmt.Errorf("No SIG matches any of the passed regular expressions")
		}
		c.SIG = sigs
	}

	// check if the Output specified is one of "", "json" or "yaml"
	if !sliceContains(SupportedOutputOpts, c.Output) {
		return fmt.Errorf("unsupported output format: %s. Valid values: %v", c.Output, SupportedOutputOpts)
	}

	//TODO: check the valid values of stage, status, etc.
	return nil
}

// Query searches the local repo and possibly GitHub for KEPs
// that match the search criteria.
func (c *Client) Query(opts QueryOpts) error {
	// if output format is json/yaml, suppress other outputs
	// json/yaml are structured formats, logging events which
	// do not conform to the spec will create formatting issues
	var suppressOutputs bool
	if sliceContains(StructuredOutputFormats, opts.Output) {
		suppressOutputs = true
	} else {
		suppressOutputs = false
	}

	if !suppressOutputs {
		fmt.Fprintf(c.Out, "Searching for KEPs...\n")
	}

	repoPath, err := c.findEnhancementsRepo(opts.CommonArgs)
	if err != nil {
		return errors.Wrap(err, "unable to search KEPs")
	}

	c.SetGitHubToken(opts.CommonArgs)

	var allKEPs []*keps.Proposal
	// load the KEPs for each listed SIG
	for _, sig := range opts.SIG {
		// KEPs in the local filesystem
		allKEPs = append(allKEPs, c.loadLocalKEPs(repoPath, sig)...)

		// Open PRs; existing KEPs with open PRs will be shown twice
		if opts.IncludePRs {
			prKeps, err := c.loadKEPPullRequests(sig)
			if err != nil {
				fmt.Fprintf(c.Err, "error searching for KEP PRs from %s: %s\n", sig, err)
			}
			if prKeps != nil {
				allKEPs = append(allKEPs, prKeps...)
			}
		}
	}

	// filter the KEPs by criteria
	allowedStatus := sliceToMap(opts.Status)
	allowedStage := sliceToMap(opts.Stage)
	allowedPRR := sliceToMap(opts.PRRApprover)

	var keep []*keps.Proposal
	for _, k := range allKEPs {
		if len(opts.Status) > 0 && !allowedStatus[k.Status] {
			continue
		}
		if len(opts.Stage) > 0 && !allowedStage[k.Stage] {
			continue
		}
		if len(opts.PRRApprover) > 0 && !atLeastOne(k.PRRApprovers, allowedPRR) {
			continue
		}
		keep = append(keep, k)
	}

	switch opts.Output {
	case "table":
		c.PrintTable(DefaultPrintConfigs("LastUpdated", "Stage", "Status", "SIG", "Authors", "Title", "Link"), keep)
	case "yaml":
		c.PrintYAML(keep)
	case "json":
		c.PrintJSON(keep)
	default:
		// this check happens as a validation step in cobra as well
		// added it for additional verbosity
		return fmt.Errorf("unsupported output format: %s. Valid values: %s", opts.Output, SupportedOutputOpts)
	}
	return nil
}

func sliceToMap(s []string) map[string]bool {
	m := make(map[string]bool, len(s))
	for _, v := range s {
		m[v] = true
	}
	return m
}

func sliceContains(s []string, e string) bool {
	for _, k := range s {
		if k == e {
			return true
		}
	}

	return false
}

// returns all strings in vals that match at least one
// regexp in regexps
func selectByRegexp(vals []string, regexps []string) ([]string, error) {
	var matches []string
	for _, s := range vals {
		for _, r := range regexps {
			found, err := regexp.MatchString(r, s)
			if err != nil {
				return matches, err
			}
			if found {
				matches = append(matches, s)
				break
			}
		}
	}
	return matches, nil
}

// returns true if at least one of vals is in the allowed map
func atLeastOne(vals []string, allowed map[string]bool) bool {
	for _, v := range vals {
		if allowed[v] {
			return true
		}
	}

	return false
}
<commit_msg>Fix reference to validations.Sigs() since refactor<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kepctl

import (
	"fmt"
	"regexp"

	"github.com/pkg/errors"

	"k8s.io/enhancements/pkg/kepval/keps"
	"k8s.io/enhancements/pkg/kepval/util"
)

var (
	// SupportedOutputOpts stores all allowed query output formats
	SupportedOutputOpts = []string{
		"table",
		"json",
		"yaml",
	}

	// DefaultOutputOpt is the default output format for kepctl query
	DefaultOutputOpt = "table"

	StructuredOutputFormats = []string{
		"json",
		"yaml",
	}
)

type QueryOpts struct {
	CommonArgs
	SIG         []string
	Status      []string
	Stage       []string
	PRRApprover []string
	IncludePRs  bool
	Output      string
}

// Validate checks the args and cleans them up if needed
func (c *QueryOpts) Validate(args []string) error {
	if len(c.SIG) > 0 {
		sigs, err := selectByRegexp(util.Groups(), c.SIG)
		if err != nil {
			return err
		}
		if len(sigs) == 0 {
			return fmt.Errorf("No SIG matches any of the passed regular expressions")
		}
		c.SIG = sigs
	}

	// check if the Output specified is one of "", "json" or "yaml"
	if !sliceContains(SupportedOutputOpts, c.Output) {
		return fmt.Errorf("unsupported output format: %s. Valid values: %v", c.Output, SupportedOutputOpts)
	}

	//TODO: check the valid values of stage, status, etc.
	return nil
}

// Query searches the local repo and possibly GitHub for KEPs
// that match the search criteria.
func (c *Client) Query(opts QueryOpts) error {
	// if output format is json/yaml, suppress other outputs
	// json/yaml are structured formats, logging events which
	// do not conform to the spec will create formatting issues
	var suppressOutputs bool
	if sliceContains(StructuredOutputFormats, opts.Output) {
		suppressOutputs = true
	} else {
		suppressOutputs = false
	}

	if !suppressOutputs {
		fmt.Fprintf(c.Out, "Searching for KEPs...\n")
	}

	repoPath, err := c.findEnhancementsRepo(opts.CommonArgs)
	if err != nil {
		return errors.Wrap(err, "unable to search KEPs")
	}

	c.SetGitHubToken(opts.CommonArgs)

	var allKEPs []*keps.Proposal
	// load the KEPs for each listed SIG
	for _, sig := range opts.SIG {
		// KEPs in the local filesystem
		allKEPs = append(allKEPs, c.loadLocalKEPs(repoPath, sig)...)

		// Open PRs; existing KEPs with open PRs will be shown twice
		if opts.IncludePRs {
			prKeps, err := c.loadKEPPullRequests(sig)
			if err != nil {
				fmt.Fprintf(c.Err, "error searching for KEP PRs from %s: %s\n", sig, err)
			}
			if prKeps != nil {
				allKEPs = append(allKEPs, prKeps...)
			}
		}
	}

	// filter the KEPs by criteria
	allowedStatus := sliceToMap(opts.Status)
	allowedStage := sliceToMap(opts.Stage)
	allowedPRR := sliceToMap(opts.PRRApprover)

	var keep []*keps.Proposal
	for _, k := range allKEPs {
		if len(opts.Status) > 0 && !allowedStatus[k.Status] {
			continue
		}
		if len(opts.Stage) > 0 && !allowedStage[k.Stage] {
			continue
		}
		if len(opts.PRRApprover) > 0 && !atLeastOne(k.PRRApprovers, allowedPRR) {
			continue
		}
		keep = append(keep, k)
	}

	switch opts.Output {
	case "table":
		c.PrintTable(DefaultPrintConfigs("LastUpdated", "Stage", "Status", "SIG", "Authors", "Title", "Link"), keep)
	case "yaml":
		c.PrintYAML(keep)
	case "json":
		c.PrintJSON(keep)
	default:
		// this check happens as a validation step in cobra as well
		// added it for additional verbosity
		return fmt.Errorf("unsupported output format: %s. Valid values: %s", opts.Output, SupportedOutputOpts)
	}
	return nil
}

func sliceToMap(s []string) map[string]bool {
	m := make(map[string]bool, len(s))
	for _, v := range s {
		m[v] = true
	}
	return m
}

func sliceContains(s []string, e string) bool {
	for _, k := range s {
		if k == e {
			return true
		}
	}

	return false
}

// returns all strings in vals that match at least one
// regexp in regexps
func selectByRegexp(vals []string, regexps []string) ([]string, error) {
	var matches []string
	for _, s := range vals {
		for _, r := range regexps {
			found, err := regexp.MatchString(r, s)
			if err != nil {
				return matches, err
			}
			if found {
				matches = append(matches, s)
				break
			}
		}
	}
	return matches, nil
}

// returns true if at least one of vals is in the allowed map
func atLeastOne(vals []string, allowed map[string]bool) bool {
	for _, v := range vals {
		if allowed[v] {
			return true
		}
	}

	return false
}
<|endoftext|>


<commit_before>package woothee

import (
  "strings"
)

type Parser struct {
  AgentDataSet DataSet
}

func NewParser() (*Parser) {
  return &Parser { DefaultDataSet }
}

func (p *Parser) LookupDataset(label string) (*Result, error) {
  result := p.AgentDataSet[label]
  if result == nil {
    return nil, ErrNoDataSet
  }

  return result, nil
}

func (p *Parser) TryCrawler(agent string) (result *Result, err error) {
  result, err = p.ChallengeGoogle(agent)
  if err == nil {
    return
  }

  result, err = p.ChallengeCrawlers(agent)
  if err == nil {
    return
  }

  err = ErrNoMatch
  return
}

func (p *Parser) TryMisc(agent string) (result *Result, err error) {
  result, err = p.ChallengeDesktopTools(agent)
  if err == nil {
    return
  }

  err = ErrNoMatch
  return
}

func (p *Parser) TryRareCases(agent string) (result *Result, err error) {
  result, err = p.ChallengeSmartphonePatterns(agent)
  if err == nil {
    return
  }

  result, err = p.ChallengeSleipnir(agent)
  if err == nil {
    return
  }

  result, err = p.ChallengeHttpLibrary(agent)
  if err == nil {
    return
  }

  result, err = p.ChallengeMaybeRssReader(agent)
  if err == nil {
    return
  }

  err = ErrNoMatch
  return
}

func (p *Parser) ChallengeGoogle(agent string) (*Result, error) {
  if ! strings.Contains(agent, "Google") {
    return nil, ErrNoMatch
  }

  if strings.Contains(agent, "compatible; Googlebot") {
    if strings.Contains(agent, "compatible; Googlebot-Mobile") {
      return p.LookupDataset("GoogleBotMobile")
    } else {
      return p.LookupDataset("GoogleBot")
    }
  }

  if strings.Contains(agent, "Googlebot-Image/") {
    return p.LookupDataset("GoogleBot")
  }

  if strings.Contains(agent, "Mediapartners-Google") {
    if strings.Contains(agent, "compatible; Mediapartners-Google") || agent == "Mediapartners-Google" {
      return p.LookupDataset("GoogleMediaPartners")
    }
  }

  if strings.Contains(agent, "Feedfetcher-Google;") {
    return p.LookupDataset("GoogleFeedFetcher")
  }

  if strings.Contains(agent, "AppEngine-Google") {
    return p.LookupDataset("GoogleAppEngine")
  }

  if strings.Contains(agent, "Google Web Preview") {
    return p.LookupDataset("GoogleWebPreview")
  }

  return nil, ErrNoMatch
}

func (p *Parser) ChallengeCrawlers(agent string) (*Result, error) {

  if strings.Contains(agent, "Yahoo") || strings.Contains(agent, "help.yahoo.co.jp/help/jp/") || strings.Contains(agent, "listing.yahoo.co.jp/support/faq/") {
    switch {
    case strings.Contains(agent, "compatible; Yahoo! Slurp"):
      return p.LookupDataset("YahooSlurp")
    case strings.Contains(agent, "YahooFeedSeekerJp") || strings.Contains(agent, "YahooFeedSeekerBetaJp"):
      return p.LookupDataset("YahooJP")
    case strings.Contains(agent, "crawler (http://listing.yahoo.co.jp/support/faq/") || strings.Contains(agent, "crawler (http://help.yahoo.co.jp/help/jp/"):
      return p.LookupDataset("YahooJP")
    case strings.Contains(agent, "Yahoo Pipes"):
      return p.LookupDataset("YahooPipes")
    }
  }

  if strings.Contains(agent, "msnbot") {
    return p.LookupDataset("msnbot")
  }

  if strings.Contains(agent, "bingbot") {
    if strings.Contains(agent, "compatible; bingbot") {
      return p.LookupDataset("bingbot")
    }
  }

  if strings.Contains(agent, "Baidu") {
    if strings.Contains(agent, "compatible; Baiduspider") || strings.Contains(agent, "Baiduspider+") || strings.Contains(agent, "Baiduspider-image+") {
      return p.LookupDataset("Baiduspider")
    }
  }

  if strings.Contains(agent, "Yeti") {
    if strings.Contains(agent, "http://help.naver.com/robots") {
      return p.LookupDataset("Yeti")
    }
  }

  if strings.Contains(agent, "FeedBurner/") {
    return p.LookupDataset("FeedBurner")
  }

  if strings.Contains(agent, "facebookexternalhit") {
    return p.LookupDataset("facebook")
  }

  if strings.Contains(agent, "ichiro") {
    if strings.Contains(agent, "http://help.goo.ne.jp/door/crawler.html") || strings.Contains(agent, "compatible; ichiro/mobile goo;") {
      return p.LookupDataset("goo")
    }
  }

  if strings.Contains(agent, "gooblogsearch/") {
    return p.LookupDataset("goo")
  }

  if strings.Contains(agent, "Apple-PubSub") {
    return p.LookupDataset("ApplePubSub")
  }

  if strings.Contains(agent, "(www.radian6.com/crawler)") {
    return p.LookupDataset("radian6")
  }

  if strings.Contains(agent, "Genieo/") {
    return p.LookupDataset("Genieo")
  }

  if strings.Contains(agent, "labs.topsy.com/butterfly/") {
    return p.LookupDataset("topsyButterfly")
  }

  if strings.Contains(agent, "rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot") {
    return p.LookupDataset("rogerbot")
  }

  if strings.Contains(agent, "compatible; AhrefsBot/") {
    return p.LookupDataset("AhrefsBot")
  }

  if strings.Contains(agent, "livedoor FeedFetcher") || strings.Contains(agent, "Fastladder FeedFetcher") {
    return p.LookupDataset("livedoorFeedFetcher")
  }

  if strings.Contains(agent, "Hatena ") {
    if strings.Contains(agent, "Hatena Antenna") || strings.Contains(agent, "Hatena Pagetitle Agent") || strings.Contains(agent, "Hatena Diary RSS") {
      return p.LookupDataset("Hatena")
    }
  }

  if strings.Contains(agent, "mixi-check") || strings.Contains(agent, "mixi-crawler") || strings.Contains(agent, "mixi-news-crawler") {
    return p.LookupDataset("mixi")
  }

  if strings.Contains(agent, "Indy Library") {
    if strings.Contains(agent, "compatible; Indy Library") {
      return p.LookupDataset("IndyLibrary")
    }
  }

  return nil, ErrNoMatch
}

func (p *Parser) ChallengeDesktopTools(agent string) (*Result, error) {
  if strings.Contains(agent, "AppleSyndication/") {
    return p.LookupDataset("SafariRSSReader")
  }

  if strings.Contains(agent, "compatible; Google Desktop/") {
    return p.LookupDataset("GoogleDesktop")
  }

  if strings.Contains(agent, "Windows-RSS-Platform") {
    return p.LookupDataset("WindowsRSSReader")
  }

  return nil, ErrNoMatch
}

func (p *Parser) ChallengeSmartphonePatterns(agent string) (*Result, error) {
  if strings.Contains(agent, "CFNetwork/") {
    // This is like iPhone, but only Category and Name are filled
    result, err := p.LookupDataset("iOS")
    if err != nil {
      return nil, err
    } else {
      r := EmptyResult.Clone()
      r.Name = result.Name
      r.Category = result.Category
      return r, nil
    }
  }

  return nil, ErrNoMatch
}

func (p *Parser) ChallengeSleipnir (agent string) (*Result, error) {
  start := strings.Index(agent, "Sleipnir/");
  if start < 0 {
    return nil, ErrNoMatch
  }

  // Absolutely refuse to use regexps
  agent_len := len(agent)

  end := start + 9
  for ; end < agent_len; end++ {
    switch agent[end] {
    case '0','1','2','3','4','5','6','7','8','9','.':
      // no op
    default:
      break
    }
  }
  version := agent[start:end]
  if version == "" {
    version = VALUE_UNKNOWN
  }

  result, err := p.LookupDataset("Sleipnir")
  if err != nil {
    return nil, err
  }

  win, err := p.LookupDataset("Win")
  if err != nil {
    return nil, err
  }

  r := result.Clone()
  r.Version = version
  r.Category = win.Category
  r.Os = win.Name

  return r, nil
}

func (p *Parser) ChallengeHttpLibrary (agent string) (*Result, error) {
  var version string

  if strings.HasPrefix(agent, "Apache-HttpClient/") || strings.HasPrefix(agent, "Jakarta Commons-HttpClient/") || strings.HasPrefix(agent, "Java/") {
    version = "Java"
  } else if i := strings.Index(agent, "HttpClient"); i > - 1 {
    // i should be greater than 0 so we can check for either "-" or " "
    // preceding the "HttpClient" string
    if i == 0 {
      return nil, ErrNoMatch
    }

    switch agent[i - 1] {
    case '-', ' ':
      // OK, no op
    default:
      return nil, ErrNoMatch
    }

    agent_len := len(agent)
    if agent_len > i + 10 { // Longer than "...HttpClient"
      if agent[i + 11] == '/' {
        goto MATCH_JAVA_MISC
      }
      return nil, ErrNoMatch
    }

MATCH_JAVA_MISC:
    version = "Java"
  } else if strings.Contains(agent, "Java(TM) 2 Runtime Environment,") {
    version = "Java"
  } else if strings.HasPrefix(agent, "Wget/") {
    version = "wget"
  } else if strings.HasPrefix(agent, "libwww-perl") || strings.HasPrefix(agent, "WWW-Mechanize") || strings.HasPrefix(agent, "LWP::Simple") || strings.HasPrefix(agent, "LWP ") || strings.HasPrefix(agent, "lwp-trivial") {
    version = "perl"
  } else if strings.HasPrefix(agent, "Ruby") || strings.HasPrefix(agent, "feedzirra") || strings.HasPrefix(agent, "Typoeus") {
    version = "ruby"
  } else if strings.HasPrefix(agent, "Python-urllib/") || strings.HasPrefix(agent, "Twisted ") {
    version = "python"
  } else if p.isPHP(agent) {
    version = "php"
  }

  if version == "" {
    return nil, ErrNoMatch
  }

  result, err := p.LookupDataset("HTTPLibrary")
  if err != nil {
    return nil, err
  }
  result = result.Clone()
  result.Version = version
  return result, nil
}

var PHP_PREFIX_PATTERNS []string = []string{
  "PHP",
  "WordPress",
  "CakePHP",
  "PukiWiki",
  "PECL::HTTP",
}

var PEAR_PATTERNS []string = []string {
  "PEAR HTTP_Request",
  "HTTP_Request",
}
func (p *Parser) isPHP(agent string) bool {
  agent_len := len(agent)
  for _, pattern := range PHP_PREFIX_PATTERNS {
    if ! strings.HasPrefix(agent, pattern) {
      continue
    }

    // Either this pattern is followed by "/", " ", or is exactly same
    // as pattern
    pattern_len := len(pattern)
    if pattern_len == agent_len {
      return true
    } else if c := agent[pattern_len]; c == '/' || c == ' ' {
      return true
    }
  }

  // None of the above patterns matched
  for _, pattern := range PEAR_PATTERNS {
    i := strings.Index(agent, pattern)
    if i == -1 {
      continue
    }

    // match. this pattern must be followed by either
    // " class" or "2"
    pattern_len := len(pattern)
    if i + pattern_len + 6 <= agent_len {
      if agent[i + pattern_len:i + pattern_len + 6] == " class" {
        return true
      }
    }

    if i + pattern_len + 1 <= agent_len {
      if agent[i + pattern_len] == '2' {
        return true
      }
    }
  }

  return false
}

func (p *Parser) ChallengeMaybeRssReader(agent string) (*Result, error) {
  if p.isRssReader(agent) || strings.Contains(agent, "headline-reader") || strings.Contains(agent, "cococ/") {
    return p.LookupDataset("VariousRSSReader")
  }

  return nil, ErrNoMatch
}

func (p *Parser) isRssReader(agent string) bool {
  i := strings.Index(agent, "rss")
  if i == -1 {
    return false
  }

  // "rss" must be followed by one of the following
  agent_len := len(agent)
  if i + 3 + 6 <= agent_len {
    if agent[i + 3:i + 3 + 6] == "reader" {
      return true
    }
  }

  if i + 3 + 3 <= agent_len {
    if agent[i + 3:i + 3 + 3] == "bar" {
      return true
    }
  }

  if i + 3 + 1 <= agent_len {
    switch agent[i+3] {
    case '-', '_', '/', ';', '(', ')':
      return true
    }
  }

  // Otherwise, need to be followed by 0 or more " ", "+", followed by "/"
  for x := i + 3; x < agent_len; x++ {
    switch agent[x] {
    case ' ', '+':
      // no op
    case '/':
      return true
    default:
      break
    }
  }

  return false
}
<commit_msg>grr, use regexp<commit_after>package woothee

import (
  "regexp"
  "strings"
)

type Parser struct {
  AgentDataSet DataSet
}

func NewParser() (*Parser) {
  return &Parser { DefaultDataSet }
}

func (p *Parser) LookupDataset(label string) (*Result, error) {
  result := p.AgentDataSet[label]
  if result == nil {
    return nil, ErrNoDataSet
  }

  return result, nil
}

func (p *Parser) TryCrawler(agent string) (result *Result, err error) {
  result, err = p.ChallengeGoogle(agent)
  if err == nil {
    return
  }

  result, err = p.ChallengeCrawlers(agent)
  if err == nil {
    return
  }

  err = ErrNoMatch
  return
}

func (p *Parser) TryMisc(agent string) (result *Result, err error) {
  result, err = p.ChallengeDesktopTools(agent)
  if err == nil {
    return
  }

  err = ErrNoMatch
  return
}

func (p *Parser) TryRareCases(agent string) (result *Result, err error) {
  result, err = p.ChallengeSmartphonePatterns(agent)
  if err == nil {
    return
  }

  result, err = p.ChallengeSleipnir(agent)
  if err == nil {
    return
  }

  result, err = p.ChallengeHttpLibrary(agent)
  if err == nil {
    return
  }

  result, err = p.ChallengeMaybeRssReader(agent)
  if err == nil {
    return
  }

  result, err = p.ChallengeMaybeCrawler(agent)
  if err == nil {
    return
  }

  err = ErrNoMatch
  return
}

func (p *Parser) ChallengeGoogle(agent string) (*Result, error) {
  if ! strings.Contains(agent, "Google") {
    return nil, ErrNoMatch
  }

  if strings.Contains(agent, "compatible; Googlebot") {
    if strings.Contains(agent, "compatible; Googlebot-Mobile") {
      return p.LookupDataset("GoogleBotMobile")
    } else {
      return p.LookupDataset("GoogleBot")
    }
  }

  if strings.Contains(agent, "Googlebot-Image/") {
    return p.LookupDataset("GoogleBot")
  }

  if strings.Contains(agent, "Mediapartners-Google") {
    if strings.Contains(agent, "compatible; Mediapartners-Google") || agent == "Mediapartners-Google" {
      return p.LookupDataset("GoogleMediaPartners")
    }
  }

  if strings.Contains(agent, "Feedfetcher-Google;") {
    return p.LookupDataset("GoogleFeedFetcher")
  }

  if strings.Contains(agent, "AppEngine-Google") {
    return p.LookupDataset("GoogleAppEngine")
  }

  if strings.Contains(agent, "Google Web Preview") {
    return p.LookupDataset("GoogleWebPreview")
  }

  return nil, ErrNoMatch
}

func (p *Parser) ChallengeCrawlers(agent string) (*Result, error) {

  if strings.Contains(agent, "Yahoo") || strings.Contains(agent, "help.yahoo.co.jp/help/jp/") || strings.Contains(agent, "listing.yahoo.co.jp/support/faq/") {
    switch {
    case strings.Contains(agent, "compatible; Yahoo! Slurp"):
      return p.LookupDataset("YahooSlurp")
    case strings.Contains(agent, "YahooFeedSeekerJp") || strings.Contains(agent, "YahooFeedSeekerBetaJp"):
      return p.LookupDataset("YahooJP")
    case strings.Contains(agent, "crawler (http://listing.yahoo.co.jp/support/faq/") || strings.Contains(agent, "crawler (http://help.yahoo.co.jp/help/jp/"):
      return p.LookupDataset("YahooJP")
    case strings.Contains(agent, "Yahoo Pipes"):
      return p.LookupDataset("YahooPipes")
    }
  }

  if strings.Contains(agent, "msnbot") {
    return p.LookupDataset("msnbot")
  }

  if strings.Contains(agent, "bingbot") {
    if strings.Contains(agent, "compatible; bingbot") {
      return p.LookupDataset("bingbot")
    }
  }

  if strings.Contains(agent, "Baidu") {
    if strings.Contains(agent, "compatible; Baiduspider") || strings.Contains(agent, "Baiduspider+") || strings.Contains(agent, "Baiduspider-image+") {
      return p.LookupDataset("Baiduspider")
    }
  }

  if strings.Contains(agent, "Yeti") {
    if strings.Contains(agent, "http://help.naver.com/robots") {
      return p.LookupDataset("Yeti")
    }
  }

  if strings.Contains(agent, "FeedBurner/") {
    return p.LookupDataset("FeedBurner")
  }

  if strings.Contains(agent, "facebookexternalhit") {
    return p.LookupDataset("facebook")
  }

  if strings.Contains(agent, "ichiro") {
    if strings.Contains(agent, "http://help.goo.ne.jp/door/crawler.html") || strings.Contains(agent, "compatible; ichiro/mobile goo;") {
      return p.LookupDataset("goo")
    }
  }

  if strings.Contains(agent, "gooblogsearch/") {
    return p.LookupDataset("goo")
  }

  if strings.Contains(agent, "Apple-PubSub") {
    return p.LookupDataset("ApplePubSub")
  }

  if strings.Contains(agent, "(www.radian6.com/crawler)") {
    return p.LookupDataset("radian6")
  }

  if strings.Contains(agent, "Genieo/") {
    return p.LookupDataset("Genieo")
  }

  if strings.Contains(agent, "labs.topsy.com/butterfly/") {
    return p.LookupDataset("topsyButterfly")
  }

  if strings.Contains(agent, "rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot") {
    return p.LookupDataset("rogerbot")
  }

  if strings.Contains(agent, "compatible; AhrefsBot/") {
    return p.LookupDataset("AhrefsBot")
  }

  if strings.Contains(agent, "livedoor FeedFetcher") || strings.Contains(agent, "Fastladder FeedFetcher") {
    return p.LookupDataset("livedoorFeedFetcher")
  }

  if strings.Contains(agent, "Hatena ") {
    if strings.Contains(agent, "Hatena Antenna") || strings.Contains(agent, "Hatena Pagetitle Agent") || strings.Contains(agent, "Hatena Diary RSS") {
      return p.LookupDataset("Hatena")
    }
  }

  if strings.Contains(agent, "mixi-check") || strings.Contains(agent, "mixi-crawler") || strings.Contains(agent, "mixi-news-crawler") {
    return p.LookupDataset("mixi")
  }

  if strings.Contains(agent, "Indy Library") {
    if strings.Contains(agent, "compatible; Indy Library") {
      return p.LookupDataset("IndyLibrary")
    }
  }

  return nil, ErrNoMatch
}

func (p *Parser) ChallengeDesktopTools(agent string) (*Result, error) {
  if strings.Contains(agent, "AppleSyndication/") {
    return p.LookupDataset("SafariRSSReader")
  }

  if strings.Contains(agent, "compatible; Google Desktop/") {
    return p.LookupDataset("GoogleDesktop")
  }

  if strings.Contains(agent, "Windows-RSS-Platform") {
    return p.LookupDataset("WindowsRSSReader")
  }

  return nil, ErrNoMatch
}

func (p *Parser) ChallengeSmartphonePatterns(agent string) (*Result, error) {
  if strings.Contains(agent, "CFNetwork/") {
    // This is like iPhone, but only Category and Name are filled
    result, err := p.LookupDataset("iOS")
    if err != nil {
      return nil, err
    } else {
      r := EmptyResult.Clone()
      r.Name = result.Name
      r.Category = result.Category
      return r, nil
    }
  }

  return nil, ErrNoMatch
}

func (p *Parser) ChallengeSleipnir (agent string) (*Result, error) {
  start := strings.Index(agent, "Sleipnir/");
  if start < 0 {
    return nil, ErrNoMatch
  }

  // Absolutely refuse to use regexps
  agent_len := len(agent)

  end := start + 9
  for ; end < agent_len; end++ {
    switch agent[end] {
    case '0','1','2','3','4','5','6','7','8','9','.':
      // no op
    default:
      break
    }
  }
  version := agent[start:end]
  if version == "" {
    version = VALUE_UNKNOWN
  }

  result, err := p.LookupDataset("Sleipnir")
  if err != nil {
    return nil, err
  }

  win, err := p.LookupDataset("Win")
  if err != nil {
    return nil, err
  }

  r := result.Clone()
  r.Version = version
  r.Category = win.Category
  r.Os = win.Name

  return r, nil
}

func (p *Parser) ChallengeHttpLibrary (agent string) (*Result, error) {
  var version string

  if strings.HasPrefix(agent, "Apache-HttpClient/") || strings.HasPrefix(agent, "Jakarta Commons-HttpClient/") || strings.HasPrefix(agent, "Java/") {
    version = "Java"
  } else if i := strings.Index(agent, "HttpClient"); i > - 1 {
    // i should be greater than 0 so we can check for either "-" or " "
    // preceding the "HttpClient" string
    if i == 0 {
      return nil, ErrNoMatch
    }

    switch agent[i - 1] {
    case '-', ' ':
      // OK, no op
    default:
      return nil, ErrNoMatch
    }

    agent_len := len(agent)
    if agent_len > i + 10 { // Longer than "...HttpClient"
      if agent[i + 11] == '/' {
        goto MATCH_JAVA_MISC
      }
      return nil, ErrNoMatch
    }

MATCH_JAVA_MISC:
    version = "Java"
  } else if strings.Contains(agent, "Java(TM) 2 Runtime Environment,") {
    version = "Java"
  } else if strings.HasPrefix(agent, "Wget/") {
    version = "wget"
  } else if strings.HasPrefix(agent, "libwww-perl") || strings.HasPrefix(agent, "WWW-Mechanize") || strings.HasPrefix(agent, "LWP::Simple") || strings.HasPrefix(agent, "LWP ") || strings.HasPrefix(agent, "lwp-trivial") {
    version = "perl"
  } else if strings.HasPrefix(agent, "Ruby") || strings.HasPrefix(agent, "feedzirra") || strings.HasPrefix(agent, "Typoeus") {
    version = "ruby"
  } else if strings.HasPrefix(agent, "Python-urllib/") || strings.HasPrefix(agent, "Twisted ") {
    version = "python"
  } else if p.isPHP(agent) {
    version = "php"
  }

  if version == "" {
    return nil, ErrNoMatch
  }

  result, err := p.LookupDataset("HTTPLibrary")
  if err != nil {
    return nil, err
  }
  result = result.Clone()
  result.Version = version
  return result, nil
}

var PHP_PREFIX_PATTERNS []string = []string{
  "PHP",
  "WordPress",
  "CakePHP",
  "PukiWiki",
  "PECL::HTTP",
}

var PEAR_PATTERNS []string = []string {
  "PEAR HTTP_Request",
  "HTTP_Request",
}
func (p *Parser) isPHP(agent string) bool {
  agent_len := len(agent)
  for _, pattern := range PHP_PREFIX_PATTERNS {
    if ! strings.HasPrefix(agent, pattern) {
      continue
    }

    // Either this pattern is followed by "/", " ", or is exactly same
    // as pattern
    pattern_len := len(pattern)
    if pattern_len == agent_len {
      return true
    } else if c := agent[pattern_len]; c == '/' || c == ' ' {
      return true
    }
  }

  // None of the above patterns matched
  for _, pattern := range PEAR_PATTERNS {
    i := strings.Index(agent, pattern)
    if i == -1 {
      continue
    }

    // match. this pattern must be followed by either
    // " class" or "2"
    pattern_len := len(pattern)
    if i + pattern_len + 6 <= agent_len {
      if agent[i + pattern_len:i + pattern_len + 6] == " class" {
        return true
      }
    }

    if i + pattern_len + 1 <= agent_len {
      if agent[i + pattern_len] == '2' {
        return true
      }
    }
  }

  return false
}

var MAYBE_RSS_PATTERN = regexp.MustCompile(
  `(?i)rss(?:reader|bar|[-_ /;()]|[ +]*/)`,
)
func (p *Parser) ChallengeMaybeRssReader(agent string) (*Result, error) {
  if MAYBE_RSS_PATTERN.MatchString(agent) || strings.Contains(agent, "headline-reader") || strings.Contains(agent, "cococ/") {
    return p.LookupDataset("VariousRSSReader")
  }

  return nil, ErrNoMatch
}

var MAYBE_CRAWLER_PATTERN = regexp.MustCompile(
  `(?i)(?:bot|crawler|spider)(?:[-_ ./;@()]|$)`,
)
var MAYBE_FEED_PARSER_PATTERN = regexp.MustCompile(
  `(?i)(?:feed|web) ?parser`,
)
var MAYBE_WATCHDOG_PATTERN = regexp.MustCompile(
  `(?i)watch ?dog`,
)
func (p *Parser) ChallengeMaybeCrawler(agent string) (*Result, error) {
  if MAYBE_CRAWLER_PATTERN.MatchString(agent) || p.hasCrawlerPrefix(agent) || MAYBE_FEED_PARSER_PATTERN.MatchString(agent) || MAYBE_WATCHDOG_PATTERN.MatchString(agent) {
    return p.LookupDataset("VariousCrawler")
  }
  return nil, ErrNoMatch
}

var CRAWLER_PREFIX_PATTERNS []string = []string{
  "Rome Client ",
  "UnwindFetchor/",
  "ia_archiver ",
  "Summify ",
  "PostRank/",
}
func (p *Parser) hasCrawlerPrefix(agent string) bool {
  for _, pattern := range CRAWLER_PREFIX_PATTERNS {
    if strings.HasPrefix(agent, pattern) {
      return true
    }
  }

  if strings.Contains(agent, "ASP-Ranker Feed Crawler") {
    return true
  }

  return false
}
<|endoftext|>


<commit_before>package jo

import (
	"fmt"
)

// Events signal a change in context, for example the start of
// a string literal.
type Event int

const (
	Primitive Event = 1 << (5 + iota)
	Composite
	Start
	End
)

const (
	Continue    Event = iota
	SyntaxError       = iota
	Done              = iota

	ObjectStart = iota | Composite | Start
	ObjectEnd   = iota | Composite | End
	ArrayStart  = iota | Composite | Start
	ArrayEnd    = iota | Composite | End

	KeyStart = iota | Start
	KeyEnd   = iota | End

	StringStart = iota | Primitive | Start
	StringEnd   = iota | Primitive | End
	NumberStart = iota | Primitive | Start
	NumberEnd   = iota | Primitive | End
	BoolStart   = iota | Primitive | Start
	BoolEnd     = iota | Primitive | End
	NullStart   = iota | Primitive | Start
	NullEnd     = iota | Primitive | End
)

// String returns the name of the event.
func (e Event) String() string {
	switch e {
	case Continue:
		return "Continue"
	case SyntaxError:
		return "SyntaxError"
	case Done:
		return "Done"
	case ObjectStart:
		return "ObjectStart"
	case ObjectEnd:
		return "ObjectEnd"
	case KeyStart:
		return "KeyStart"
	case KeyEnd:
		return "KeyEnd"
	case ArrayStart:
		return "ArrayStart"
	case ArrayEnd:
		return "ArrayEnd"
	case StringStart:
		return "StringStart"
	case StringEnd:
		return "StringEnd"
	case NumberStart:
		return "NumberStart"
	case NumberEnd:
		return "NumberEnd"
	case BoolStart:
		return "BoolStart"
	case BoolEnd:
		return "BoolEnd"
	case NullStart:
		return "NullStart"
	case NullEnd:
		return "NullEnd"
	}
	return "<unknown event>"
}

// Parser is the state machine used while parsing a stream of JSON data.
// It requires no initialization before use.
type Parser struct {
	state int
	stack []int
}

const (
	_Value int = iota
	_Done

	_ObjectKeyOrBrace   // {
	_ObjectKeyDone      // {"foo
	_ObjectColon        // {"foo"
	_ObjectCommaOrBrace // {"foo":"bar"
	_ObjectKey          // {"foo":"bar",

	_ArrayValueOrBracket // [
	_ArrayCommaOrBracket // ["any value"

	// leading whitespace must be be consumed before any of
	// the states listed above are processed
	__CONSUME_SPACE__

	_StringUnicode  // "\u
	_StringUnicode2 // "\u1
	_StringUnicode3 // "\u12
	_StringUnicode4 // "\u123
	_String         // "
	_StringDone     // "foo
	_StringEscaped  // "\

	_NumberNegative      // -
	_NumberZero          // 0
	_Number              // 123
	_NumberDotFirstDigit // 123.
	_NumberDotDigit      // 123.4
	_NumberExpSign       // 123e
	_NumberExpFirstDigit // 123e+
	_NumberExpDigit      // 123e+1

	_True  // t
	_True2 // tr
	_True3 // tru

	_False  // f
	_False2 // fa
	_False3 // fal
	_False4 // fals

	_Null  // n
	_Null2 // nu
	_Null3 // nul
)

var expected = map[int]string{
	_Value:               "start of JSON value",
	_Done:                "end of input",
	_ObjectKeyOrBrace:    "object key or '}'",
	_ObjectColon:         "':'",
	_ObjectCommaOrBrace:  "',' or '}'",
	_ObjectKey:           "object key",
	_ArrayValueOrBracket: "array element or ']'",
	_ArrayCommaOrBracket: "',' or ']'",
	_StringUnicode:       "hexadecimal digit",
	_StringUnicode2:      "hexadecimal digit",
	_StringUnicode3:      "hexadecimal digit",
	_StringUnicode4:      "hexadecimal digit",
	_String:              "valid string character or '\"'",
	_StringEscaped:       "'b', 'f', 'n', 'r', 't', 'u', '\\', '/' or '\"'",
	_NumberNegative:      "digit",
	_NumberZero:          "'.', 'e' or 'E'",
	_NumberDotFirstDigit: "digit",
	_NumberExpSign:       "'-', '+' or digit",
	_NumberExpFirstDigit: "digit",
	_True:                "'r' in literal true",
	_True2:               "'u' in literal true",
	_True3:               "'e' in literal true",
	_False:               "'a' in literal false",
	_False2:              "'l' in literal false",
	_False3:              "'s' in literal false",
	_False4:              "'e' in literal false",
	_Null:                "'u' in literal null",
	_Null2:               "'l' in literal null",
	_Null3:               "'l' in literal null",
}

// Next parses a slice of JSON data, signalling the first change in context by
// returning the number of bytes read and an appropriate event. If parsing
// concludes without any change in parser state, the Continue psuedo-event
// will be returned instead.
//
// When the event returned is SyntaxError, the error return value will describe
// why parsing failed.
func (p *Parser) Next(data []byte) (int, Event, error) {
	for i := 0; i < len(data); i++ {
		ev := Continue
		b := data[i]

		// trim insignificant whitespace
		if p.state < __CONSUME_SPACE__ && isSpace(b) {
			continue
		}

		switch p.state {
		case _Value:
			if b == '{' {
				ev = ObjectStart
				p.state = _ObjectKeyOrBrace
			} else if b == '[' {
				ev = ArrayStart
				p.state = _ArrayValueOrBracket
			} else if b == '"' {
				ev = StringStart
				p.state = _String
				p.push(_StringDone)
			} else if b == '-' {
				ev = NumberStart
				p.state = _NumberNegative
			} else if b == '0' {
				ev = NumberStart
				p.state = _NumberZero
			} else if '1' <= b && b <= '9' {
				ev = NumberStart
				p.state = _Number
			} else if b == 't' {
				ev = BoolStart
				p.state = _True
			} else if b == 'f' {
				ev = BoolStart
				p.state = _False
			} else if b == 'n' {
				ev = NullStart
				p.state = _Null
			} else {
				goto abort
			}

		case _ObjectKeyOrBrace:
			if b == '}' {
				ev = ObjectEnd
				p.state = p.pop()
				break
			}

			// if it's not a brace, it must be a key
			p.state = _ObjectKey
			fallthrough

		case _ObjectKey:
			if b == '"' {
				ev = KeyStart
				p.state = _String
				p.push(_ObjectKeyDone)
			} else {
				goto abort
			}

		case _ObjectKeyDone:
			// we wouldn't be here unless b == '"', so we can avoid
			// checking it again
			ev = KeyEnd
			p.state = _ObjectColon

		case _ObjectColon:
			if b == ':' {
				p.state = _Value
				p.push(_ObjectCommaOrBrace)
			} else {
				goto abort
			}

		case _ObjectCommaOrBrace:
			if b == ',' {
				p.state = _ObjectKey
			} else if b == '}' {
				ev = ObjectEnd
				p.state = p.pop()
			} else {
				goto abort
			}

		case _ArrayValueOrBracket:
			if b == ']' {
				ev = ArrayEnd
				p.state = p.pop()
			} else {
				p.state = _Value
				p.push(_ArrayCommaOrBracket)

				// rewind and let _Value parse this byte for us
				i--
			}

		case _ArrayCommaOrBracket:
			if b == ',' {
				p.state = _Value
				p.push(_ArrayCommaOrBracket)
			} else if b == ']' {
				ev = ArrayEnd
				p.state = p.pop()
			} else {
				goto abort
			}

		case _StringUnicode,
			_StringUnicode2,
			_StringUnicode3,
			_StringUnicode4:
			if isHex(b) {
				// move on to the next unicode byte state, or back to
				// `_String` if this was the fourth hexadecimal
				// character after "\u"
				p.state++
			} else {
				goto abort
			}

		case _String:
			if b == '"' {
				// forget we saw the double quote, let the next state
				// "discover" it instead
				i--
				p.state = p.pop()
			} else if b == '\\' {
				p.state = _StringEscaped
			} else if b < 0x20 {
				goto abort
			}

		case _StringDone:
			// we wouldn't be here unless b == '"', so we can avoid
			// checking it again
			ev = StringEnd
			p.state = p.pop()

		case _StringEscaped:
			switch b {
			case 'b', 'f', 'n', 'r', 't', '\\', '/', '"':
				p.state = _String
			case 'u':
				p.state = _StringUnicode
			default:
				goto abort
			}

		case _NumberNegative:
			if b == '0' {
				p.state = _NumberZero
			} else if '1' <= b && b <= '9' {
				p.state = _Number
			} else {
				goto abort
			}

		case _Number:
			if b >= '0' && b <= '9' {
				break
			}
			fallthrough

		case _NumberZero:
			if b == '.' {
				p.state = _NumberDotFirstDigit
			} else if b == 'e' || b == 'E' {
				p.state = _NumberExpSign
			} else {
				ev = NumberEnd
				p.state = p.pop()

				// rewind a byte, because the character we encountered was
				// not part of the number
				i--
			}

		case _NumberDotFirstDigit:
			if b >= '0' && b <= '9' {
				p.state = _NumberDotDigit
			} else {
				goto abort
			}

		case _NumberDotDigit:
			if b == 'e' || b == 'E' {
				p.state = _NumberExpSign
			} else if b < '0' || b > '9' {
				ev = NumberEnd
				p.state = p.pop()

				// rewind a byte, because the character we encountered was
				// not part of the number
				i--
			}

		case _NumberExpSign:
			p.state = _NumberExpFirstDigit
			if b == '+' || b == '-' {
				break
			}
			fallthrough

		case _NumberExpFirstDigit:
			if b < '0' || b > '9' {
				goto abort
			} else {
				p.state++
			}

		case _NumberExpDigit:
			if b < '0' || b > '9' {
				ev = NumberEnd
				p.state = p.pop()

				// rewind a byte, because the character we encountered was
				// not part of the number
				i--
			}

		case _True:
			if b == 'r' {
				p.state = _True2
			} else {
				goto abort
			}

		case _True2:
			if b == 'u' {
				p.state = _True3
			} else {
				goto abort
			}

		case _True3:
			if b == 'e' {
				ev = BoolEnd
				p.state = p.pop()
			} else {
				goto abort
			}

		case _False:
			if b == 'a' {
				p.state = _False2
			} else {
				goto abort
			}

		case _False2:
			if b == 'l' {
				p.state = _False3
			} else {
				goto abort
			}

		case _False3:
			if b == 's' {
				p.state = _False4
			} else {
				goto abort
			}

		case _False4:
			if b == 'e' {
				ev = BoolEnd
				p.state = p.pop()
			} else {
				goto abort
			}

		case _Null:
			if b == 'u' {
				p.state = _Null2
			} else {
				goto abort
			}

		case _Null2:
			if b == 'l' {
				p.state = _Null3
			} else {
				goto abort
			}

		case _Null3:
			if b == 'l' {
				ev = NullEnd
				p.state = p.pop()
			} else {
				goto abort
			}

		case _Done:
			// only whitespace characters are legal after the
			// top-level value
			goto abort

		default:
			panic("invalid state")
		}

		// if this byte didn't yield an event, continue with
		// the next one
		if ev == Continue {
			continue
		}

		return i + 1, ev, nil

	abort:
		return i, SyntaxError, fmt.Errorf("expected %s, found %q",
			expected[p.state], b)
	}

	return len(data), Continue, nil
}

// push puts a new state at the top of the stack.
func (p *Parser) push(state int) {
	p.stack = append(p.stack, state)
}

// pop retrieves the top state in the stack.
func (p *Parser) pop() int {
	length := len(p.stack)

	// if the state queue is empty, the top level value has ended
	if length == 0 {
		return _Done
	}

	state := p.stack[length-1]
	p.stack = p.stack[:length-1]

	return state
}

// End informs the parser not to expect any further input (i.e. EOF).
//
// A SyntaxError event and a relevant error will be returned if the method
// was invoked before the top-level had been completely parsed. Returns either
// a NumberEnd or Done event otherwise.
func (p *Parser) End() (Event, error) {
	switch p.state {
	case _Done:
		return Done, nil
	case _NumberZero, _Number, _NumberDotDigit, _NumberExpDigit:
		if len(p.stack) == 0 {
			p.state = _Done
			return NumberEnd, nil
		}
	}

	return SyntaxError, fmt.Errorf("expected %s, found end of input",
		expected[p.state])
}

// Reset resets the parser struct to its initial state. Convenient when parsing
// a stream of more than one JSON value (simply reset the parser after each Done
// event).
func (p *Parser) Reset() {
	*p = Parser{}
}

func isSpace(b byte) bool {
	return b == ' ' || b == '\t' || b == '\n' || b == '\r'
}

func isHex(b byte) bool {
	return (b >= '0' && b <= '9') ||
		(b >= 'a' && b <= 'f') ||
		(b >= 'A' && b <= 'F')
}
<commit_msg>_X -> stateX<commit_after>package jo

import (
	"fmt"
)

// Events signal a change in context, for example the start of
// a string literal.
type Event int

const (
	Primitive Event = 1 << (5 + iota)
	Composite
	Start
	End
)

const (
	Continue    Event = iota
	SyntaxError       = iota
	Done              = iota

	ObjectStart = iota | Composite | Start
	ObjectEnd   = iota | Composite | End
	ArrayStart  = iota | Composite | Start
	ArrayEnd    = iota | Composite | End

	KeyStart = iota | Start
	KeyEnd   = iota | End

	StringStart = iota | Primitive | Start
	StringEnd   = iota | Primitive | End
	NumberStart = iota | Primitive | Start
	NumberEnd   = iota | Primitive | End
	BoolStart   = iota | Primitive | Start
	BoolEnd     = iota | Primitive | End
	NullStart   = iota | Primitive | Start
	NullEnd     = iota | Primitive | End
)

// String returns the name of the event.
func (e Event) String() string {
	switch e {
	case Continue:
		return "Continue"
	case SyntaxError:
		return "SyntaxError"
	case Done:
		return "Done"
	case ObjectStart:
		return "ObjectStart"
	case ObjectEnd:
		return "ObjectEnd"
	case KeyStart:
		return "KeyStart"
	case KeyEnd:
		return "KeyEnd"
	case ArrayStart:
		return "ArrayStart"
	case ArrayEnd:
		return "ArrayEnd"
	case StringStart:
		return "StringStart"
	case StringEnd:
		return "StringEnd"
	case NumberStart:
		return "NumberStart"
	case NumberEnd:
		return "NumberEnd"
	case BoolStart:
		return "BoolStart"
	case BoolEnd:
		return "BoolEnd"
	case NullStart:
		return "NullStart"
	case NullEnd:
		return "NullEnd"
	}
	return "<unknown event>"
}

// Parser is the state machine used while parsing a stream of JSON data.
// It requires no initialization before use.
type Parser struct {
	state int
	stack []int
}

const (
	stateValue int = iota
	stateDone

	stateObjectKeyOrBrace   // {
	stateObjectKeyDone      // {"foo
	stateObjectColon        // {"foo"
	stateObjectCommaOrBrace // {"foo":"bar"
	stateObjectKey          // {"foo":"bar",

	stateArrayValueOrBracket // [
	stateArrayCommaOrBracket // ["any value"

	// leading whitespace must be be consumed before any of
	// the states listed above are processed
	__CONSUME_SPACE__

	stateStringUnicode  // "\u
	stateStringUnicode2 // "\u1
	stateStringUnicode3 // "\u12
	stateStringUnicode4 // "\u123
	stateString         // "
	stateStringDone     // "foo
	stateStringEscaped  // "\

	stateNumberNegative      // -
	stateNumberZero          // 0
	stateNumber              // 123
	stateNumberDotFirstDigit // 123.
	stateNumberDotDigit      // 123.4
	stateNumberExpSign       // 123e
	stateNumberExpFirstDigit // 123e+
	stateNumberExpDigit      // 123e+1

	stateTrue  // t
	stateTrue2 // tr
	stateTrue3 // tru

	stateFalse  // f
	stateFalse2 // fa
	stateFalse3 // fal
	stateFalse4 // fals

	stateNull  // n
	stateNull2 // nu
	stateNull3 // nul
)

var expected = map[int]string{
	stateValue:               "start of JSON value",
	stateDone:                "end of input",
	stateObjectKeyOrBrace:    "object key or '}'",
	stateObjectColon:         "':'",
	stateObjectCommaOrBrace:  "',' or '}'",
	stateObjectKey:           "object key",
	stateArrayValueOrBracket: "array element or ']'",
	stateArrayCommaOrBracket: "',' or ']'",
	stateStringUnicode:       "hexadecimal digit",
	stateStringUnicode2:      "hexadecimal digit",
	stateStringUnicode3:      "hexadecimal digit",
	stateStringUnicode4:      "hexadecimal digit",
	stateString:              "valid string character or '\"'",
	stateStringEscaped:       "'b', 'f', 'n', 'r', 't', 'u', '\\', '/' or '\"'",
	stateNumberNegative:      "digit",
	stateNumberZero:          "'.', 'e' or 'E'",
	stateNumberDotFirstDigit: "digit",
	stateNumberExpSign:       "'-', '+' or digit",
	stateNumberExpFirstDigit: "digit",
	stateTrue:                "'r' in literal true",
	stateTrue2:               "'u' in literal true",
	stateTrue3:               "'e' in literal true",
	stateFalse:               "'a' in literal false",
	stateFalse2:              "'l' in literal false",
	stateFalse3:              "'s' in literal false",
	stateFalse4:              "'e' in literal false",
	stateNull:                "'u' in literal null",
	stateNull2:               "'l' in literal null",
	stateNull3:               "'l' in literal null",
}

// Next parses a slice of JSON data, signalling the first change in context by
// returning the number of bytes read and an appropriate event. If parsing
// concludes without any change in parser state, the Continue psuedo-event
// will be returned instead.
//
// When the event returned is SyntaxError, the error return value will describe
// why parsing failed.
func (p *Parser) Next(data []byte) (int, Event, error) {
	for i := 0; i < len(data); i++ {
		ev := Continue
		b := data[i]

		// trim insignificant whitespace
		if p.state < __CONSUME_SPACE__ && isSpace(b) {
			continue
		}

		switch p.state {
		case stateValue:
			if b == '{' {
				ev = ObjectStart
				p.state = stateObjectKeyOrBrace
			} else if b == '[' {
				ev = ArrayStart
				p.state = stateArrayValueOrBracket
			} else if b == '"' {
				ev = StringStart
				p.state = stateString
				p.push(stateStringDone)
			} else if b == '-' {
				ev = NumberStart
				p.state = stateNumberNegative
			} else if b == '0' {
				ev = NumberStart
				p.state = stateNumberZero
			} else if '1' <= b && b <= '9' {
				ev = NumberStart
				p.state = stateNumber
			} else if b == 't' {
				ev = BoolStart
				p.state = stateTrue
			} else if b == 'f' {
				ev = BoolStart
				p.state = stateFalse
			} else if b == 'n' {
				ev = NullStart
				p.state = stateNull
			} else {
				goto abort
			}

		case stateObjectKeyOrBrace:
			if b == '}' {
				ev = ObjectEnd
				p.state = p.pop()
				break
			}

			// if it's not a brace, it must be a key
			p.state = stateObjectKey
			fallthrough

		case stateObjectKey:
			if b == '"' {
				ev = KeyStart
				p.state = stateString
				p.push(stateObjectKeyDone)
			} else {
				goto abort
			}

		case stateObjectKeyDone:
			// we wouldn't be here unless b == '"', so we can avoid
			// checking it again
			ev = KeyEnd
			p.state = stateObjectColon

		case stateObjectColon:
			if b == ':' {
				p.state = stateValue
				p.push(stateObjectCommaOrBrace)
			} else {
				goto abort
			}

		case stateObjectCommaOrBrace:
			if b == ',' {
				p.state = stateObjectKey
			} else if b == '}' {
				ev = ObjectEnd
				p.state = p.pop()
			} else {
				goto abort
			}

		case stateArrayValueOrBracket:
			if b == ']' {
				ev = ArrayEnd
				p.state = p.pop()
			} else {
				p.state = stateValue
				p.push(stateArrayCommaOrBracket)

				// rewind and let stateValue parse this byte for us
				i--
			}

		case stateArrayCommaOrBracket:
			if b == ',' {
				p.state = stateValue
				p.push(stateArrayCommaOrBracket)
			} else if b == ']' {
				ev = ArrayEnd
				p.state = p.pop()
			} else {
				goto abort
			}

		case stateStringUnicode,
			stateStringUnicode2,
			stateStringUnicode3,
			stateStringUnicode4:
			if isHex(b) {
				// move on to the next unicode byte state, or back to
				// `stateString` if this was the fourth hexadecimal
				// character after "\u"
				p.state++
			} else {
				goto abort
			}

		case stateString:
			if b == '"' {
				// forget we saw the double quote, let the next state
				// "discover" it instead
				i--
				p.state = p.pop()
			} else if b == '\\' {
				p.state = stateStringEscaped
			} else if b < 0x20 {
				goto abort
			}

		case stateStringDone:
			// we wouldn't be here unless b == '"', so we can avoid
			// checking it again
			ev = StringEnd
			p.state = p.pop()

		case stateStringEscaped:
			switch b {
			case 'b', 'f', 'n', 'r', 't', '\\', '/', '"':
				p.state = stateString
			case 'u':
				p.state = stateStringUnicode
			default:
				goto abort
			}

		case stateNumberNegative:
			if b == '0' {
				p.state = stateNumberZero
			} else if '1' <= b && b <= '9' {
				p.state = stateNumber
			} else {
				goto abort
			}

		case stateNumber:
			if b >= '0' && b <= '9' {
				break
			}
			fallthrough

		case stateNumberZero:
			if b == '.' {
				p.state = stateNumberDotFirstDigit
			} else if b == 'e' || b == 'E' {
				p.state = stateNumberExpSign
			} else {
				ev = NumberEnd
				p.state = p.pop()

				// rewind a byte, because the character we encountered was
				// not part of the number
				i--
			}

		case stateNumberDotFirstDigit:
			if b >= '0' && b <= '9' {
				p.state = stateNumberDotDigit
			} else {
				goto abort
			}

		case stateNumberDotDigit:
			if b == 'e' || b == 'E' {
				p.state = stateNumberExpSign
			} else if b < '0' || b > '9' {
				ev = NumberEnd
				p.state = p.pop()

				// rewind a byte, because the character we encountered was
				// not part of the number
				i--
			}

		case stateNumberExpSign:
			p.state = stateNumberExpFirstDigit
			if b == '+' || b == '-' {
				break
			}
			fallthrough

		case stateNumberExpFirstDigit:
			if b < '0' || b > '9' {
				goto abort
			} else {
				p.state++
			}

		case stateNumberExpDigit:
			if b < '0' || b > '9' {
				ev = NumberEnd
				p.state = p.pop()

				// rewind a byte, because the character we encountered was
				// not part of the number
				i--
			}

		case stateTrue:
			if b == 'r' {
				p.state = stateTrue2
			} else {
				goto abort
			}

		case stateTrue2:
			if b == 'u' {
				p.state = stateTrue3
			} else {
				goto abort
			}

		case stateTrue3:
			if b == 'e' {
				ev = BoolEnd
				p.state = p.pop()
			} else {
				goto abort
			}

		case stateFalse:
			if b == 'a' {
				p.state = stateFalse2
			} else {
				goto abort
			}

		case stateFalse2:
			if b == 'l' {
				p.state = stateFalse3
			} else {
				goto abort
			}

		case stateFalse3:
			if b == 's' {
				p.state = stateFalse4
			} else {
				goto abort
			}

		case stateFalse4:
			if b == 'e' {
				ev = BoolEnd
				p.state = p.pop()
			} else {
				goto abort
			}

		case stateNull:
			if b == 'u' {
				p.state = stateNull2
			} else {
				goto abort
			}

		case stateNull2:
			if b == 'l' {
				p.state = stateNull3
			} else {
				goto abort
			}

		case stateNull3:
			if b == 'l' {
				ev = NullEnd
				p.state = p.pop()
			} else {
				goto abort
			}

		case stateDone:
			// only whitespace characters are legal after the
			// top-level value
			goto abort

		default:
			panic("invalid state")
		}

		// if this byte didn't yield an event, continue with
		// the next one
		if ev == Continue {
			continue
		}

		return i + 1, ev, nil

	abort:
		return i, SyntaxError, fmt.Errorf("expected %s, found %q",
			expected[p.state], b)
	}

	return len(data), Continue, nil
}

// push puts a new state at the top of the stack.
func (p *Parser) push(state int) {
	p.stack = append(p.stack, state)
}

// pop retrieves the top state in the stack.
func (p *Parser) pop() int {
	length := len(p.stack)

	// if the state queue is empty, the top level value has ended
	if length == 0 {
		return stateDone
	}

	state := p.stack[length-1]
	p.stack = p.stack[:length-1]

	return state
}

// End informs the parser not to expect any further input (i.e. EOF).
//
// A SyntaxError event and a relevant error will be returned if the method
// was invoked before the top-level had been completely parsed. Returns either
// a NumberEnd or Done event otherwise.
func (p *Parser) End() (Event, error) {
	switch p.state {
	case stateDone:
		return Done, nil
	case stateNumberZero, stateNumber, stateNumberDotDigit, stateNumberExpDigit:
		if len(p.stack) == 0 {
			p.state = stateDone
			return NumberEnd, nil
		}
	}

	return SyntaxError, fmt.Errorf("expected %s, found end of input",
		expected[p.state])
}

// Reset resets the parser struct to its initial state. Convenient when parsing
// a stream of more than one JSON value (simply reset the parser after each Done
// event).
func (p *Parser) Reset() {
	*p = Parser{}
}

func isSpace(b byte) bool {
	return b == ' ' || b == '\t' || b == '\n' || b == '\r'
}

func isHex(b byte) bool {
	return (b >= '0' && b <= '9') ||
		(b >= 'a' && b <= 'f') ||
		(b >= 'A' && b <= 'F')
}
<|endoftext|>


<commit_before>// Copyright 2017 Tomas Machalek <tomas.machalek@gmail.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vertigo

import (
	"bufio"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"regexp"
	"strings"

	"golang.org/x/text/encoding/charmap"
	"golang.org/x/text/transform"
)

const (
	channelChunkSize = 250000 // changing the value affects performance (10k...300k ~ 15%)
	logProgressEach  = 1000000
	LineTypeToken    = "token"
	LineTypeStruct   = "struct"
	LineTypeIgnored  = "ignored"

	AccumulatorTypeStack = "stack"
	AccumulatorTypeComb  = "comb"
	AccumulatorTypeNil   = "nil"

	CharsetISO8859_2   = "iso-8859-2"
	CharsetWindows1250 = "windows-1250"
	CharsetUTF_8       = "utf-8"
)

var (
	tagSrchRegexp  = regexp.MustCompile("^<([\\w]+)(\\s*[^>]*?|)/?>$")
	attrValRegexp  = regexp.MustCompile("(\\w+)=\"([^\"]+)\"")
	closeTagRegexp = regexp.MustCompile("</([^>]+)\\s*>")
)

// --------------------------------------------------------

// ParserConf contains configuration parameters for
// vertical file parser
type ParserConf struct {

	// Source vertical file (either a plain text file or a gzip one)
	VerticalFilePath string `json:"verticalFilePath"`

	Encoding string `json:"encoding"`

	FilterArgs [][][]string `json:"filterArgs"`

	StructAttrAccumulator string `json:"structAttrAccumulator"`
}

// LoadConfig loads the configuration from a JSON file.
// In case of an error the program exits with panic.
func LoadConfig(path string) *ParserConf {
	rawData, err := ioutil.ReadFile(path)
	if err != nil {
		panic(err)
	}
	var conf ParserConf
	err = json.Unmarshal(rawData, &conf)
	if err != nil {
		panic(err)
	}
	return &conf
}

// --------------------------------------------------------

// Token is a representation of
// a parsed line. It connects both, positional attributes
// and currently accumulated structural attributes.
type Token struct {
	Word        string
	Attrs       []string
	StructAttrs map[string]string
}

func (v *Token) WordLC() string {
	return strings.ToLower(v.Word)
}

// --------------------------------------------------------

type Structure struct {
	Name  string
	Attrs map[string]string
}

// --------------------------------------------------------

type StructureClose struct {
	Name string
}

// --------------------------------------------------------

type LineProcessor interface {
	ProcToken(token *Token)
	ProcStruct(strc *Structure)
	ProcStructClose(strc *StructureClose)
}

// --------------------------------------------------------

// this is quite simplified but it should work for our purposes
func isElement(tagSrc string) bool {
	return strings.HasPrefix(tagSrc, "<") && strings.HasSuffix(tagSrc, ">")
}

func isOpenElement(tagSrc string) bool {
	return isElement(tagSrc) && !strings.HasPrefix(tagSrc, "</") &&
		!strings.HasSuffix(tagSrc, "/>")
}

func isCloseElement(tagSrc string) bool {
	return isElement(tagSrc) && strings.HasPrefix(tagSrc, "</")
}

func isSelfCloseElement(tagSrc string) bool {
	return isElement(tagSrc) && strings.HasSuffix(tagSrc, "/>")
}

func parseAttrVal(src string) map[string]string {
	ans := make(map[string]string)
	srch := attrValRegexp.FindAllStringSubmatch(src, -1)
	for i := 0; i < len(srch); i++ {
		ans[srch[i][1]] = srch[i][2]
	}
	return ans
}

func parseLine(line string, elmStack structAttrAccumulator) interface{} {
	switch {
	case isOpenElement(line):
		srch := tagSrchRegexp.FindStringSubmatch(line)
		meta := &Structure{Name: srch[1], Attrs: parseAttrVal(srch[2])}
		elmStack.Begin(meta)
		return meta
	case isCloseElement(line):
		srch := closeTagRegexp.FindStringSubmatch(line)
		elmStack.End(srch[1])
		return &StructureClose{Name: srch[1]}
	case isSelfCloseElement(line):
		srch := tagSrchRegexp.FindStringSubmatch(line)
		return &Structure{Name: srch[1], Attrs: parseAttrVal(srch[2])}
	default:
		items := strings.Split(line, "\t")
		return &Token{
			Word:        items[0],
			Attrs:       items[1:],
			StructAttrs: elmStack.GetAttrs(),
		}
	}
}

func tokenMatchesFilter(token *Token, filterCNF [][][]string) bool {
	var sub bool
	for _, item := range filterCNF {
		sub = false
		for _, v := range item {
			if v[1] == token.StructAttrs[v[0]] {
				sub = true
				break
			}
		}
		if sub == false {
			return false
		}
	}
	return true
}

func createStructAttrAccumulator(ident string) (structAttrAccumulator, error) {
	switch ident {
	case AccumulatorTypeStack:
		return newStack(), nil
	case AccumulatorTypeComb:
		return newStructAttrs(), nil
	case AccumulatorTypeNil:
		return newNilStructAttrs(), nil
	default:
		return nil, fmt.Errorf("Unknown accumulator type \"%s\"", ident)
	}
}

// SupportedCharsets returns a list of names of
// character sets.
func SupportedCharsets() []string {
	return []string{CharsetISO8859_2, CharsetUTF_8, CharsetWindows1250}
}

func getCharmapByName(name string) (*charmap.Charmap, error) {
	switch strings.ToLower(name) {
	case CharsetISO8859_2:
		return charmap.ISO8859_2, nil
	case CharsetWindows1250:
		return charmap.Windows1250, nil
	case CharsetUTF_8:
		return nil, nil
	default:
		return nil, fmt.Errorf("Unsupported charset '%s'", name)
	}
}

func importString(s string, ch *charmap.Charmap) string {
	if ch == nil {
		return s
	}
	ans, _, _ := transform.String(ch.NewDecoder(), s)
	// TODO handle error
	return ans
}

// ParseVerticalFile processes a corpus vertical file
// line by line and applies a custom LineProcessor on
// them. The processing is parallelized in the sense
// that reading a file into lines and processing of
// the lines runs in different goroutines. To reduce
// overhead, the data are passed between goroutines
// in chunks.
func ParseVerticalFile(conf *ParserConf, lproc LineProcessor) error {
	f, err := os.Open(conf.VerticalFilePath)
	if err != nil {
		return err
	}
	finfo, err := f.Stat()
	if err != nil {
		return err
	}
	if !finfo.Mode().IsRegular() {
		return fmt.Errorf("Path %s is not a regular file", conf.VerticalFilePath)
	}

	var rd io.Reader
	if strings.HasSuffix(conf.VerticalFilePath, ".gz") {
		rd, err = gzip.NewReader(f)
		if err != nil {
			return err
		}

	} else {
		rd = f
	}
	brd := bufio.NewScanner(rd)

	stack, err := createStructAttrAccumulator(conf.StructAttrAccumulator)
	if err != nil {
		return err
	}

	chm, chErr := getCharmapByName(conf.Encoding)
	if chErr != nil {
		return chErr

	} else if chm != nil {
		log.Printf("Configured conversion from charset %s", chm)

	} else {
		log.Printf("Assume encoding is utf-8")
	}
	ch := make(chan []interface{})
	chunk := make([]interface{}, channelChunkSize)
	go func() {
		i := 0
		progress := 0
		for brd.Scan() {
			line := parseLine(importString(brd.Text(), chm), stack)
			chunk[i] = line
			i++
			if i == channelChunkSize {
				i = 0
				ch <- chunk
				chunk = make([]interface{}, channelChunkSize)
			}
			progress++
			if progress%logProgressEach == 0 {
				log.Printf("...processed %d lines.\n", progress)
			}
		}
		if i > 0 {
			ch <- chunk[:i]
		}
		close(ch)
	}()

	for tokens := range ch {
		for _, token := range tokens {
			switch token.(type) {
			case *Token:
				tk := token.(*Token)
				if tokenMatchesFilter(tk, conf.FilterArgs) {
					lproc.ProcToken(tk)
				}
			case *Structure:
				lproc.ProcStruct(token.(*Structure))
			case *StructureClose:
				lproc.ProcStructClose(token.(*StructureClose))
			}
		}
	}
	log.Println("Parsing done. Metadata stack size: ", stack.Size())
	return nil
}

//ParseVerticalFileNoGoRo is just for benchmarking purposes
func ParseVerticalFileNoGoRo(conf *ParserConf, lproc LineProcessor) {
	f, err := os.Open(conf.VerticalFilePath)
	if err != nil {
		panic(err)
	}
	rd := bufio.NewScanner(f)
	stack := newStack()

	for rd.Scan() {
		token := parseLine(rd.Text(), stack)
		switch token.(type) {
		case *Token:
			lproc.ProcToken(token.(*Token))
		}
	}

	log.Println("Parsing done. Metadata stack size: ", stack.Size())
}
<commit_msg>Fix tag analysis regexp<commit_after>// Copyright 2017 Tomas Machalek <tomas.machalek@gmail.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vertigo

import (
	"bufio"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"regexp"
	"strings"

	"golang.org/x/text/encoding/charmap"
	"golang.org/x/text/transform"
)

const (
	channelChunkSize = 250000 // changing the value affects performance (10k...300k ~ 15%)
	logProgressEach  = 1000000
	LineTypeToken    = "token"
	LineTypeStruct   = "struct"
	LineTypeIgnored  = "ignored"

	AccumulatorTypeStack = "stack"
	AccumulatorTypeComb  = "comb"
	AccumulatorTypeNil   = "nil"

	CharsetISO8859_2   = "iso-8859-2"
	CharsetWindows1250 = "windows-1250"
	CharsetUTF_8       = "utf-8"
)

var (
	tagSrchRegexp  = regexp.MustCompile("^<([\\w\\d\\p{Po}]+)(\\s+.*?|)/?>$")
	attrValRegexp  = regexp.MustCompile("(\\w+)=\"([^\"]+)\"")
	closeTagRegexp = regexp.MustCompile("</([^>]+)\\s*>")
)

// --------------------------------------------------------

// ParserConf contains configuration parameters for
// vertical file parser
type ParserConf struct {

	// Source vertical file (either a plain text file or a gzip one)
	VerticalFilePath string `json:"verticalFilePath"`

	Encoding string `json:"encoding"`

	FilterArgs [][][]string `json:"filterArgs"`

	StructAttrAccumulator string `json:"structAttrAccumulator"`
}

// LoadConfig loads the configuration from a JSON file.
// In case of an error the program exits with panic.
func LoadConfig(path string) *ParserConf {
	rawData, err := ioutil.ReadFile(path)
	if err != nil {
		panic(err)
	}
	var conf ParserConf
	err = json.Unmarshal(rawData, &conf)
	if err != nil {
		panic(err)
	}
	return &conf
}

// --------------------------------------------------------

// Token is a representation of
// a parsed line. It connects both, positional attributes
// and currently accumulated structural attributes.
type Token struct {
	Word        string
	Attrs       []string
	StructAttrs map[string]string
}

func (v *Token) WordLC() string {
	return strings.ToLower(v.Word)
}

// --------------------------------------------------------

type Structure struct {
	Name  string
	Attrs map[string]string
}

// --------------------------------------------------------

type StructureClose struct {
	Name string
}

// --------------------------------------------------------

type LineProcessor interface {
	ProcToken(token *Token)
	ProcStruct(strc *Structure)
	ProcStructClose(strc *StructureClose)
}

// --------------------------------------------------------

// this is quite simplified but it should work for our purposes
func isElement(tagSrc string) bool {
	return strings.HasPrefix(tagSrc, "<") && strings.HasSuffix(tagSrc, ">")
}

func isOpenElement(tagSrc string) bool {
	return isElement(tagSrc) && !strings.HasPrefix(tagSrc, "</") &&
		!strings.HasSuffix(tagSrc, "/>")
}

func isCloseElement(tagSrc string) bool {
	return isElement(tagSrc) && strings.HasPrefix(tagSrc, "</")
}

func isSelfCloseElement(tagSrc string) bool {
	return isElement(tagSrc) && strings.HasSuffix(tagSrc, "/>")
}

func parseAttrVal(src string) map[string]string {
	ans := make(map[string]string)
	srch := attrValRegexp.FindAllStringSubmatch(src, -1)
	for i := 0; i < len(srch); i++ {
		ans[srch[i][1]] = srch[i][2]
	}
	return ans
}

func parseLine(line string, elmStack structAttrAccumulator) interface{} {
	switch {
	case isOpenElement(line):
		srch := tagSrchRegexp.FindStringSubmatch(line)
		meta := &Structure{Name: srch[1], Attrs: parseAttrVal(srch[2])}
		elmStack.Begin(meta)
		return meta
	case isCloseElement(line):
		srch := closeTagRegexp.FindStringSubmatch(line)
		elmStack.End(srch[1])
		return &StructureClose{Name: srch[1]}
	case isSelfCloseElement(line):
		srch := tagSrchRegexp.FindStringSubmatch(line)
		return &Structure{Name: srch[1], Attrs: parseAttrVal(srch[2])}
	default:
		items := strings.Split(line, "\t")
		return &Token{
			Word:        items[0],
			Attrs:       items[1:],
			StructAttrs: elmStack.GetAttrs(),
		}
	}
}

func tokenMatchesFilter(token *Token, filterCNF [][][]string) bool {
	var sub bool
	for _, item := range filterCNF {
		sub = false
		for _, v := range item {
			if v[1] == token.StructAttrs[v[0]] {
				sub = true
				break
			}
		}
		if sub == false {
			return false
		}
	}
	return true
}

func createStructAttrAccumulator(ident string) (structAttrAccumulator, error) {
	switch ident {
	case AccumulatorTypeStack:
		return newStack(), nil
	case AccumulatorTypeComb:
		return newStructAttrs(), nil
	case AccumulatorTypeNil:
		return newNilStructAttrs(), nil
	default:
		return nil, fmt.Errorf("Unknown accumulator type \"%s\"", ident)
	}
}

// SupportedCharsets returns a list of names of
// character sets.
func SupportedCharsets() []string {
	return []string{CharsetISO8859_2, CharsetUTF_8, CharsetWindows1250}
}

func getCharmapByName(name string) (*charmap.Charmap, error) {
	switch strings.ToLower(name) {
	case CharsetISO8859_2:
		return charmap.ISO8859_2, nil
	case CharsetWindows1250:
		return charmap.Windows1250, nil
	case CharsetUTF_8:
		return nil, nil
	default:
		return nil, fmt.Errorf("Unsupported charset '%s'", name)
	}
}

func importString(s string, ch *charmap.Charmap) string {
	if ch == nil {
		return s
	}
	ans, _, _ := transform.String(ch.NewDecoder(), s)
	// TODO handle error
	return ans
}

// ParseVerticalFile processes a corpus vertical file
// line by line and applies a custom LineProcessor on
// them. The processing is parallelized in the sense
// that reading a file into lines and processing of
// the lines runs in different goroutines. To reduce
// overhead, the data are passed between goroutines
// in chunks.
func ParseVerticalFile(conf *ParserConf, lproc LineProcessor) error {
	f, err := os.Open(conf.VerticalFilePath)
	if err != nil {
		return err
	}
	finfo, err := f.Stat()
	if err != nil {
		return err
	}
	if !finfo.Mode().IsRegular() {
		return fmt.Errorf("Path %s is not a regular file", conf.VerticalFilePath)
	}

	var rd io.Reader
	if strings.HasSuffix(conf.VerticalFilePath, ".gz") {
		rd, err = gzip.NewReader(f)
		if err != nil {
			return err
		}

	} else {
		rd = f
	}
	brd := bufio.NewScanner(rd)

	stack, err := createStructAttrAccumulator(conf.StructAttrAccumulator)
	if err != nil {
		return err
	}

	chm, chErr := getCharmapByName(conf.Encoding)
	if chErr != nil {
		return chErr

	} else if chm != nil {
		log.Printf("Configured conversion from charset %s", chm)

	} else {
		log.Printf("Assume encoding is utf-8")
	}
	ch := make(chan []interface{})
	chunk := make([]interface{}, channelChunkSize)
	go func() {
		i := 0
		progress := 0
		for brd.Scan() {
			line := parseLine(importString(brd.Text(), chm), stack)
			chunk[i] = line
			i++
			if i == channelChunkSize {
				i = 0
				ch <- chunk
				chunk = make([]interface{}, channelChunkSize)
			}
			progress++
			if progress%logProgressEach == 0 {
				log.Printf("...processed %d lines.\n", progress)
			}
		}
		if i > 0 {
			ch <- chunk[:i]
		}
		close(ch)
	}()

	for tokens := range ch {
		for _, token := range tokens {
			switch token.(type) {
			case *Token:
				tk := token.(*Token)
				if tokenMatchesFilter(tk, conf.FilterArgs) {
					lproc.ProcToken(tk)
				}
			case *Structure:
				lproc.ProcStruct(token.(*Structure))
			case *StructureClose:
				lproc.ProcStructClose(token.(*StructureClose))
			}
		}
	}
	log.Println("Parsing done. Metadata stack size: ", stack.Size())
	return nil
}

//ParseVerticalFileNoGoRo is just for benchmarking purposes
func ParseVerticalFileNoGoRo(conf *ParserConf, lproc LineProcessor) {
	f, err := os.Open(conf.VerticalFilePath)
	if err != nil {
		panic(err)
	}
	rd := bufio.NewScanner(f)
	stack := newStack()

	for rd.Scan() {
		token := parseLine(rd.Text(), stack)
		switch token.(type) {
		case *Token:
			lproc.ProcToken(token.(*Token))
		}
	}

	log.Println("Parsing done. Metadata stack size: ", stack.Size())
}
<|endoftext|>