<commit_before>package bdiscord

import (
	"bytes"
	"errors"
	"fmt"
	"strings"
	"sync"

	"github.com/42wim/matterbridge/bridge"
	"github.com/42wim/matterbridge/bridge/config"
	"github.com/42wim/matterbridge/bridge/helper"
	"github.com/bwmarrin/discordgo"
)

const MessageLength = 1950

type Bdiscord struct {
	*bridge.Config

	c *discordgo.Session

	nick            string
	useChannelID    bool
	guildID         string
	webhookID       string
	webhookToken    string
	canEditWebhooks bool

	channelsMutex  sync.RWMutex
	channels       []*discordgo.Channel
	channelInfoMap map[string]*config.ChannelInfo

	membersMutex  sync.RWMutex
	userMemberMap map[string]*discordgo.Member
	nickMemberMap map[string]*discordgo.Member
}

func New(cfg *bridge.Config) bridge.Bridger {
	b := &Bdiscord{Config: cfg}
	b.userMemberMap = make(map[string]*discordgo.Member)
	b.nickMemberMap = make(map[string]*discordgo.Member)
	b.channelInfoMap = make(map[string]*config.ChannelInfo)
	if b.GetString("WebhookURL") != "" {
		b.Log.Debug("Configuring Discord Incoming Webhook")
		b.webhookID, b.webhookToken = b.splitURL(b.GetString("WebhookURL"))
	}
	return b
}

func (b *Bdiscord) Connect() error {
	var err error
	var token string
	var guildFound bool
	b.Log.Info("Connecting")
	if b.GetString("WebhookURL") == "" {
		b.Log.Info("Connecting using token")
	} else {
		b.Log.Info("Connecting using webhookurl (for posting) and token")
	}
	if !strings.HasPrefix(b.GetString("Token"), "Bot ") {
		token = "Bot " + b.GetString("Token")
	}
	// if we have a User token, remove the `Bot` prefix
	if strings.HasPrefix(b.GetString("Token"),"User ") {
	   token = strings.Replace(token,"Bot ","",-1)
	}

	b.c, err = discordgo.New(token)
	if err != nil {
		return err
	}
	b.Log.Info("Connection succeeded")
	b.c.AddHandler(b.messageCreate)
	b.c.AddHandler(b.memberUpdate)
	b.c.AddHandler(b.messageUpdate)
	b.c.AddHandler(b.messageDelete)
	err = b.c.Open()
	if err != nil {
		return err
	}
	guilds, err := b.c.UserGuilds(100, "", "")
	if err != nil {
		return err
	}
	userinfo, err := b.c.User("@me")
	if err != nil {
		return err
	}
	serverName := strings.Replace(b.GetString("Server"), "ID:", "", -1)
	b.nick = userinfo.Username
	b.channelsMutex.Lock()
	for _, guild := range guilds {
		if guild.Name == serverName || guild.ID == serverName {
			b.channels, err = b.c.GuildChannels(guild.ID)
			b.guildID = guild.ID
			guildFound = true
			if err != nil {
				break
			}
		}
	}
	b.channelsMutex.Unlock()
	if !guildFound {
		msg := fmt.Sprintf("Server \"%s\" not found", b.GetString("Server"))
		err = errors.New(msg)
		b.Log.Error(msg)
		b.Log.Info("Possible values:")
		for _, guild := range guilds {
			b.Log.Infof("Server=\"%s\" # Server name", guild.Name)
			b.Log.Infof("Server=\"%s\" # Server ID", guild.ID)
		}
	}

	if err != nil {
		return err
	}
	b.channelsMutex.RLock()
	if b.GetString("WebhookURL") == "" {
		for _, channel := range b.channels {
			b.Log.Debugf("found channel %#v", channel)
		}
	} else {
		b.canEditWebhooks = true
		for _, channel := range b.channels {
			b.Log.Debugf("found channel %#v; verifying PermissionManageWebhooks", channel)
			perms, permsErr := b.c.State.UserChannelPermissions(userinfo.ID, channel.ID)
			manageWebhooks := discordgo.PermissionManageWebhooks
			if permsErr != nil || perms&manageWebhooks != manageWebhooks {
				b.Log.Warnf("Can't manage webhooks in channel \"%s\"", channel.Name)
				b.canEditWebhooks = false
			}
		}
		if b.canEditWebhooks {
			b.Log.Info("Can manage webhooks; will edit channel for global webhook on send")
		} else {
			b.Log.Warn("Can't manage webhooks; won't edit channel for global webhook on send")
		}
	}
	b.channelsMutex.RUnlock()

	// Obtaining guild members and initializing nickname mapping.
	b.membersMutex.Lock()
	defer b.membersMutex.Unlock()
	members, err := b.c.GuildMembers(b.guildID, "", 1000)
	if err != nil {
		b.Log.Error("Error obtaining server members: ", err)
		return err
	}
	for _, member := range members {
		if member == nil {
			b.Log.Warnf("Skipping missing information for a user.")
			continue
		}
		b.userMemberMap[member.User.ID] = member
		b.nickMemberMap[member.User.Username] = member
		if member.Nick != "" {
			b.nickMemberMap[member.Nick] = member
		}
	}
	return nil
}

func (b *Bdiscord) Disconnect() error {
	return b.c.Close()
}

func (b *Bdiscord) JoinChannel(channel config.ChannelInfo) error {
	b.channelsMutex.Lock()
	defer b.channelsMutex.Unlock()

	b.channelInfoMap[channel.ID] = &channel
	idcheck := strings.Split(channel.Name, "ID:")
	if len(idcheck) > 1 {
		b.useChannelID = true
	}
	return nil
}

func (b *Bdiscord) Send(msg config.Message) (string, error) {
	b.Log.Debugf("=> Receiving %#v", msg)

	channelID := b.getChannelID(msg.Channel)
	if channelID == "" {
		return "", fmt.Errorf("Could not find channelID for %v", msg.Channel)
	}

	// Make a action /me of the message
	if msg.Event == config.EventUserAction {
		msg.Text = "_" + msg.Text + "_"
	}

	// use initial webhook configured for the entire Discord account
	isGlobalWebhook := true
	wID := b.webhookID
	wToken := b.webhookToken

	// check if have a channel specific webhook
	b.channelsMutex.RLock()
	if ci, ok := b.channelInfoMap[msg.Channel+b.Account]; ok {
		if ci.Options.WebhookURL != "" {
			wID, wToken = b.splitURL(ci.Options.WebhookURL)
			isGlobalWebhook = false
		}
	}
	b.channelsMutex.RUnlock()

	// Use webhook to send the message
	if wID != "" {
		// skip events
		if msg.Event != "" && msg.Event != config.EventJoinLeave && msg.Event != config.EventTopicChange {
			return "", nil
		}
		b.Log.Debugf("Broadcasting using Webhook")
		for _, f := range msg.Extra["file"] {
			fi := f.(config.FileInfo)
			if fi.Comment != "" {
				msg.Text += fi.Comment + ": "
			}
			if fi.URL != "" {
				msg.Text = fi.URL
				if fi.Comment != "" {
					msg.Text = fi.Comment + ": " + fi.URL
				}
			}
		}
		// skip empty messages
		if msg.Text == "" {
			return "", nil
		}

		msg.Text = helper.ClipMessage(msg.Text, MessageLength)
		msg.Text = b.replaceUserMentions(msg.Text)
		// discord username must be [0..32] max
		if len(msg.Username) > 32 {
			msg.Username = msg.Username[0:32]
		}
		// if we have a global webhook for this Discord account, and permission
		// to modify webhooks (previously verified), then set its channel to
		// the message channel before using it
		// TODO: this isn't necessary if the last message from this webhook was
		// sent to the current channel
		if isGlobalWebhook && b.canEditWebhooks {
			b.Log.Debugf("Setting webhook channel to \"%s\"", msg.Channel)
			_, err := b.c.WebhookEdit(wID, "", "", channelID)
			if err != nil {
				b.Log.Errorf("Could not set webhook channel: %v", err)
				return "", err
			}
		}
		err := b.c.WebhookExecute(
			wID,
			wToken,
			true,
			&discordgo.WebhookParams{
				Content:   msg.Text,
				Username:  msg.Username,
				AvatarURL: msg.Avatar,
			})
		return "", err
	}

	b.Log.Debugf("Broadcasting using token (API)")

	// Delete message
	if msg.Event == config.EventMsgDelete {
		if msg.ID == "" {
			return "", nil
		}
		err := b.c.ChannelMessageDelete(channelID, msg.ID)
		return "", err
	}

	// Upload a file if it exists
	if msg.Extra != nil {
		for _, rmsg := range helper.HandleExtra(&msg, b.General) {
			rmsg.Text = helper.ClipMessage(rmsg.Text, MessageLength)
			if _, err := b.c.ChannelMessageSend(channelID, rmsg.Username+rmsg.Text); err != nil {
				b.Log.Errorf("Could not send message %#v: %v", rmsg, err)
			}
		}
		// check if we have files to upload (from slack, telegram or mattermost)
		if len(msg.Extra["file"]) > 0 {
			return b.handleUploadFile(&msg, channelID)
		}
	}

	msg.Text = helper.ClipMessage(msg.Text, MessageLength)
	msg.Text = b.replaceUserMentions(msg.Text)

	// Edit message
	if msg.ID != "" {
		_, err := b.c.ChannelMessageEdit(channelID, msg.ID, msg.Username+msg.Text)
		return msg.ID, err
	}

	// Post normal message
	res, err := b.c.ChannelMessageSend(channelID, msg.Username+msg.Text)
	if err != nil {
		return "", err
	}
	return res.ID, err
}

// useWebhook returns true if we have a webhook defined somewhere
func (b *Bdiscord) useWebhook() bool {
	if b.GetString("WebhookURL") != "" {
		return true
	}

	b.channelsMutex.RLock()
	defer b.channelsMutex.RUnlock()

	for _, channel := range b.channelInfoMap {
		if channel.Options.WebhookURL != "" {
			return true
		}
	}
	return false
}

// isWebhookID returns true if the specified id is used in a defined webhook
func (b *Bdiscord) isWebhookID(id string) bool {
	if b.GetString("WebhookURL") != "" {
		wID, _ := b.splitURL(b.GetString("WebhookURL"))
		if wID == id {
			return true
		}
	}

	b.channelsMutex.RLock()
	defer b.channelsMutex.RUnlock()

	for _, channel := range b.channelInfoMap {
		if channel.Options.WebhookURL != "" {
			wID, _ := b.splitURL(channel.Options.WebhookURL)
			if wID == id {
				return true
			}
		}
	}
	return false
}

// handleUploadFile handles native upload of files
func (b *Bdiscord) handleUploadFile(msg *config.Message, channelID string) (string, error) {
	var err error
	for _, f := range msg.Extra["file"] {
		fi := f.(config.FileInfo)
		file := discordgo.File{
			Name:        fi.Name,
			ContentType: "",
			Reader:      bytes.NewReader(*fi.Data),
		}
		m := discordgo.MessageSend{
			Content: msg.Username + fi.Comment,
			Files:   []*discordgo.File{&file},
		}
		_, err = b.c.ChannelMessageSendComplex(channelID, &m)
		if err != nil {
			return "", fmt.Errorf("file upload failed: %#v", err)
		}
	}
	return "", nil
}
<commit_msg>Run go fmt<commit_after>package bdiscord

import (
	"bytes"
	"errors"
	"fmt"
	"strings"
	"sync"

	"github.com/42wim/matterbridge/bridge"
	"github.com/42wim/matterbridge/bridge/config"
	"github.com/42wim/matterbridge/bridge/helper"
	"github.com/bwmarrin/discordgo"
)

const MessageLength = 1950

type Bdiscord struct {
	*bridge.Config

	c *discordgo.Session

	nick            string
	useChannelID    bool
	guildID         string
	webhookID       string
	webhookToken    string
	canEditWebhooks bool

	channelsMutex  sync.RWMutex
	channels       []*discordgo.Channel
	channelInfoMap map[string]*config.ChannelInfo

	membersMutex  sync.RWMutex
	userMemberMap map[string]*discordgo.Member
	nickMemberMap map[string]*discordgo.Member
}

func New(cfg *bridge.Config) bridge.Bridger {
	b := &Bdiscord{Config: cfg}
	b.userMemberMap = make(map[string]*discordgo.Member)
	b.nickMemberMap = make(map[string]*discordgo.Member)
	b.channelInfoMap = make(map[string]*config.ChannelInfo)
	if b.GetString("WebhookURL") != "" {
		b.Log.Debug("Configuring Discord Incoming Webhook")
		b.webhookID, b.webhookToken = b.splitURL(b.GetString("WebhookURL"))
	}
	return b
}

func (b *Bdiscord) Connect() error {
	var err error
	var token string
	var guildFound bool
	b.Log.Info("Connecting")
	if b.GetString("WebhookURL") == "" {
		b.Log.Info("Connecting using token")
	} else {
		b.Log.Info("Connecting using webhookurl (for posting) and token")
	}
	if !strings.HasPrefix(b.GetString("Token"), "Bot ") {
		token = "Bot " + b.GetString("Token")
	}
	// if we have a User token, remove the `Bot` prefix
	if strings.HasPrefix(b.GetString("Token"), "User ") {
		token = strings.Replace(token, "Bot ", "", -1)
	}

	b.c, err = discordgo.New(token)
	if err != nil {
		return err
	}
	b.Log.Info("Connection succeeded")
	b.c.AddHandler(b.messageCreate)
	b.c.AddHandler(b.memberUpdate)
	b.c.AddHandler(b.messageUpdate)
	b.c.AddHandler(b.messageDelete)
	err = b.c.Open()
	if err != nil {
		return err
	}
	guilds, err := b.c.UserGuilds(100, "", "")
	if err != nil {
		return err
	}
	userinfo, err := b.c.User("@me")
	if err != nil {
		return err
	}
	serverName := strings.Replace(b.GetString("Server"), "ID:", "", -1)
	b.nick = userinfo.Username
	b.channelsMutex.Lock()
	for _, guild := range guilds {
		if guild.Name == serverName || guild.ID == serverName {
			b.channels, err = b.c.GuildChannels(guild.ID)
			b.guildID = guild.ID
			guildFound = true
			if err != nil {
				break
			}
		}
	}
	b.channelsMutex.Unlock()
	if !guildFound {
		msg := fmt.Sprintf("Server \"%s\" not found", b.GetString("Server"))
		err = errors.New(msg)
		b.Log.Error(msg)
		b.Log.Info("Possible values:")
		for _, guild := range guilds {
			b.Log.Infof("Server=\"%s\" # Server name", guild.Name)
			b.Log.Infof("Server=\"%s\" # Server ID", guild.ID)
		}
	}

	if err != nil {
		return err
	}
	b.channelsMutex.RLock()
	if b.GetString("WebhookURL") == "" {
		for _, channel := range b.channels {
			b.Log.Debugf("found channel %#v", channel)
		}
	} else {
		b.canEditWebhooks = true
		for _, channel := range b.channels {
			b.Log.Debugf("found channel %#v; verifying PermissionManageWebhooks", channel)
			perms, permsErr := b.c.State.UserChannelPermissions(userinfo.ID, channel.ID)
			manageWebhooks := discordgo.PermissionManageWebhooks
			if permsErr != nil || perms&manageWebhooks != manageWebhooks {
				b.Log.Warnf("Can't manage webhooks in channel \"%s\"", channel.Name)
				b.canEditWebhooks = false
			}
		}
		if b.canEditWebhooks {
			b.Log.Info("Can manage webhooks; will edit channel for global webhook on send")
		} else {
			b.Log.Warn("Can't manage webhooks; won't edit channel for global webhook on send")
		}
	}
	b.channelsMutex.RUnlock()

	// Obtaining guild members and initializing nickname mapping.
	b.membersMutex.Lock()
	defer b.membersMutex.Unlock()
	members, err := b.c.GuildMembers(b.guildID, "", 1000)
	if err != nil {
		b.Log.Error("Error obtaining server members: ", err)
		return err
	}
	for _, member := range members {
		if member == nil {
			b.Log.Warnf("Skipping missing information for a user.")
			continue
		}
		b.userMemberMap[member.User.ID] = member
		b.nickMemberMap[member.User.Username] = member
		if member.Nick != "" {
			b.nickMemberMap[member.Nick] = member
		}
	}
	return nil
}

func (b *Bdiscord) Disconnect() error {
	return b.c.Close()
}

func (b *Bdiscord) JoinChannel(channel config.ChannelInfo) error {
	b.channelsMutex.Lock()
	defer b.channelsMutex.Unlock()

	b.channelInfoMap[channel.ID] = &channel
	idcheck := strings.Split(channel.Name, "ID:")
	if len(idcheck) > 1 {
		b.useChannelID = true
	}
	return nil
}

func (b *Bdiscord) Send(msg config.Message) (string, error) {
	b.Log.Debugf("=> Receiving %#v", msg)

	channelID := b.getChannelID(msg.Channel)
	if channelID == "" {
		return "", fmt.Errorf("Could not find channelID for %v", msg.Channel)
	}

	// Make a action /me of the message
	if msg.Event == config.EventUserAction {
		msg.Text = "_" + msg.Text + "_"
	}

	// use initial webhook configured for the entire Discord account
	isGlobalWebhook := true
	wID := b.webhookID
	wToken := b.webhookToken

	// check if have a channel specific webhook
	b.channelsMutex.RLock()
	if ci, ok := b.channelInfoMap[msg.Channel+b.Account]; ok {
		if ci.Options.WebhookURL != "" {
			wID, wToken = b.splitURL(ci.Options.WebhookURL)
			isGlobalWebhook = false
		}
	}
	b.channelsMutex.RUnlock()

	// Use webhook to send the message
	if wID != "" {
		// skip events
		if msg.Event != "" && msg.Event != config.EventJoinLeave && msg.Event != config.EventTopicChange {
			return "", nil
		}
		b.Log.Debugf("Broadcasting using Webhook")
		for _, f := range msg.Extra["file"] {
			fi := f.(config.FileInfo)
			if fi.Comment != "" {
				msg.Text += fi.Comment + ": "
			}
			if fi.URL != "" {
				msg.Text = fi.URL
				if fi.Comment != "" {
					msg.Text = fi.Comment + ": " + fi.URL
				}
			}
		}
		// skip empty messages
		if msg.Text == "" {
			return "", nil
		}

		msg.Text = helper.ClipMessage(msg.Text, MessageLength)
		msg.Text = b.replaceUserMentions(msg.Text)
		// discord username must be [0..32] max
		if len(msg.Username) > 32 {
			msg.Username = msg.Username[0:32]
		}
		// if we have a global webhook for this Discord account, and permission
		// to modify webhooks (previously verified), then set its channel to
		// the message channel before using it
		// TODO: this isn't necessary if the last message from this webhook was
		// sent to the current channel
		if isGlobalWebhook && b.canEditWebhooks {
			b.Log.Debugf("Setting webhook channel to \"%s\"", msg.Channel)
			_, err := b.c.WebhookEdit(wID, "", "", channelID)
			if err != nil {
				b.Log.Errorf("Could not set webhook channel: %v", err)
				return "", err
			}
		}
		err := b.c.WebhookExecute(
			wID,
			wToken,
			true,
			&discordgo.WebhookParams{
				Content:   msg.Text,
				Username:  msg.Username,
				AvatarURL: msg.Avatar,
			})
		return "", err
	}

	b.Log.Debugf("Broadcasting using token (API)")

	// Delete message
	if msg.Event == config.EventMsgDelete {
		if msg.ID == "" {
			return "", nil
		}
		err := b.c.ChannelMessageDelete(channelID, msg.ID)
		return "", err
	}

	// Upload a file if it exists
	if msg.Extra != nil {
		for _, rmsg := range helper.HandleExtra(&msg, b.General) {
			rmsg.Text = helper.ClipMessage(rmsg.Text, MessageLength)
			if _, err := b.c.ChannelMessageSend(channelID, rmsg.Username+rmsg.Text); err != nil {
				b.Log.Errorf("Could not send message %#v: %v", rmsg, err)
			}
		}
		// check if we have files to upload (from slack, telegram or mattermost)
		if len(msg.Extra["file"]) > 0 {
			return b.handleUploadFile(&msg, channelID)
		}
	}

	msg.Text = helper.ClipMessage(msg.Text, MessageLength)
	msg.Text = b.replaceUserMentions(msg.Text)

	// Edit message
	if msg.ID != "" {
		_, err := b.c.ChannelMessageEdit(channelID, msg.ID, msg.Username+msg.Text)
		return msg.ID, err
	}

	// Post normal message
	res, err := b.c.ChannelMessageSend(channelID, msg.Username+msg.Text)
	if err != nil {
		return "", err
	}
	return res.ID, err
}

// useWebhook returns true if we have a webhook defined somewhere
func (b *Bdiscord) useWebhook() bool {
	if b.GetString("WebhookURL") != "" {
		return true
	}

	b.channelsMutex.RLock()
	defer b.channelsMutex.RUnlock()

	for _, channel := range b.channelInfoMap {
		if channel.Options.WebhookURL != "" {
			return true
		}
	}
	return false
}

// isWebhookID returns true if the specified id is used in a defined webhook
func (b *Bdiscord) isWebhookID(id string) bool {
	if b.GetString("WebhookURL") != "" {
		wID, _ := b.splitURL(b.GetString("WebhookURL"))
		if wID == id {
			return true
		}
	}

	b.channelsMutex.RLock()
	defer b.channelsMutex.RUnlock()

	for _, channel := range b.channelInfoMap {
		if channel.Options.WebhookURL != "" {
			wID, _ := b.splitURL(channel.Options.WebhookURL)
			if wID == id {
				return true
			}
		}
	}
	return false
}

// handleUploadFile handles native upload of files
func (b *Bdiscord) handleUploadFile(msg *config.Message, channelID string) (string, error) {
	var err error
	for _, f := range msg.Extra["file"] {
		fi := f.(config.FileInfo)
		file := discordgo.File{
			Name:        fi.Name,
			ContentType: "",
			Reader:      bytes.NewReader(*fi.Data),
		}
		m := discordgo.MessageSend{
			Content: msg.Username + fi.Comment,
			Files:   []*discordgo.File{&file},
		}
		_, err = b.c.ChannelMessageSendComplex(channelID, &m)
		if err != nil {
			return "", fmt.Errorf("file upload failed: %#v", err)
		}
	}
	return "", nil
}
<|endoftext|>


<commit_before>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package domain

import (
	"math/rand"
	"testing"
	"time"

	"github.com/pingcap/tidb/sessionctx/variable"
	"github.com/pingcap/tidb/util"
	"github.com/stretchr/testify/require"
	"github.com/tikv/client-go/v2/oracle"
	"github.com/tikv/client-go/v2/txnkv/transaction"
)

func TestSchemaValidator(t *testing.T) {
	t.Run("general", subTestSchemaValidatorGeneral)
	t.Run("enqueue", subTestEnqueue)
	t.Run("enqueueActionType", subTestEnqueueActionType)
}

// subTestSchemaValidatorGeneral is batched in TestSchemaValidator
func subTestSchemaValidatorGeneral(t *testing.T) {
	lease := 10 * time.Millisecond
	leaseGrantCh := make(chan leaseGrantItem)
	oracleCh := make(chan uint64)
	exit := make(chan struct{})
	var wg util.WaitGroupWrapper
	wg.Run(func() { serverFunc(lease, leaseGrantCh, oracleCh, exit) })

	validator := NewSchemaValidator(lease, nil).(*schemaValidator)
	require.True(t, validator.IsStarted())

	for i := 0; i < 10; i++ {
		delay := time.Duration(100+rand.Intn(900)) * time.Microsecond
		time.Sleep(delay)
		// Reload can run arbitrarily, at any time.
		item := <-leaseGrantCh
		validator.Update(item.leaseGrantTS, item.oldVer, item.schemaVer, nil)
	}

	// Take a lease, check it's valid.
	item := <-leaseGrantCh
	validator.Update(
		item.leaseGrantTS,
		item.oldVer,
		item.schemaVer,
		&transaction.RelatedSchemaChange{PhyTblIDS: []int64{10}, ActionTypes: []uint64{10}})
	_, valid := validator.Check(item.leaseGrantTS, item.schemaVer, []int64{10})
	require.Equal(t, ResultSucc, valid)

	// Stop the validator, validator's items value is nil.
	validator.Stop()
	require.False(t, validator.IsStarted())
	_, isTablesChanged := validator.isRelatedTablesChanged(item.schemaVer, []int64{10})
	require.True(t, isTablesChanged)
	_, valid = validator.Check(item.leaseGrantTS, item.schemaVer, []int64{10})
	require.Equal(t, ResultUnknown, valid)
	validator.Restart()

	// Increase the current time by 2 leases, check schema is invalid.
	ts := uint64(time.Now().Add(2 * lease).UnixNano()) // Make sure that ts has timed out a lease.
	_, valid = validator.Check(ts, item.schemaVer, []int64{10})
	require.Equalf(t, ResultUnknown, valid, "validator latest schema ver %v, time %v, item schema ver %v, ts %v", validator.latestSchemaVer, validator.latestSchemaExpire, 0, oracle.GetTimeFromTS(ts))

	// Make sure newItem's version is greater than item.schema.
	newItem := getGreaterVersionItem(t, lease, leaseGrantCh, item.schemaVer)
	currVer := newItem.schemaVer
	validator.Update(newItem.leaseGrantTS, newItem.oldVer, currVer, nil)
	_, valid = validator.Check(ts, item.schemaVer, nil)
	require.Equalf(t, ResultFail, valid, "currVer %d, newItem %v", currVer, item)
	_, valid = validator.Check(ts, item.schemaVer, []int64{0})
	require.Equalf(t, ResultFail, valid, "currVer %d, newItem %v", currVer, item)

	// Check the latest schema version must changed.
	require.Less(t, item.schemaVer, validator.latestSchemaVer)

	// Make sure newItem's version is greater than currVer.
	newItem = getGreaterVersionItem(t, lease, leaseGrantCh, currVer)
	// Update current schema version to newItem's version and the delta table IDs is 1, 2, 3.
	validator.Update(ts, currVer, newItem.schemaVer, &transaction.RelatedSchemaChange{PhyTblIDS: []int64{1, 2, 3}, ActionTypes: []uint64{1, 2, 3}})
	// Make sure the updated table IDs don't be covered with the same schema version.
	validator.Update(ts, newItem.schemaVer, newItem.schemaVer, nil)
	_, isTablesChanged = validator.isRelatedTablesChanged(currVer, nil)
	require.False(t, isTablesChanged)
	_, isTablesChanged = validator.isRelatedTablesChanged(currVer, []int64{2})
	require.Truef(t, isTablesChanged, "currVer %d, newItem %v", currVer, newItem)
	// The current schema version is older than the oldest schema version.
	_, isTablesChanged = validator.isRelatedTablesChanged(-1, nil)
	require.Truef(t, isTablesChanged, "currVer %d, newItem %v", currVer, newItem)

	// All schema versions is expired.
	ts = uint64(time.Now().Add(2 * lease).UnixNano())
	_, valid = validator.Check(ts, newItem.schemaVer, nil)
	require.Equal(t, ResultUnknown, valid)

	close(exit)
	wg.Wait()
}

// subTestEnqueue is batched in TestSchemaValidator
func subTestEnqueue(t *testing.T) {
	lease := 10 * time.Millisecond
	originalCnt := variable.GetMaxDeltaSchemaCount()
	defer variable.SetMaxDeltaSchemaCount(originalCnt)

	validator := NewSchemaValidator(lease, nil).(*schemaValidator)
	require.True(t, validator.IsStarted())

	// maxCnt is 0.
	variable.SetMaxDeltaSchemaCount(0)
	validator.enqueue(1, &transaction.RelatedSchemaChange{PhyTblIDS: []int64{11}, ActionTypes: []uint64{11}})
	require.Len(t, validator.deltaSchemaInfos, 0)

	// maxCnt is 10.
	variable.SetMaxDeltaSchemaCount(10)
	ds := []deltaSchemaInfo{
		{0, []int64{1}, []uint64{1}},
		{1, []int64{1}, []uint64{1}},
		{2, []int64{1}, []uint64{1}},
		{3, []int64{2, 2}, []uint64{2, 2}},
		{4, []int64{2}, []uint64{2}},
		{5, []int64{1, 4}, []uint64{1, 4}},
		{6, []int64{1, 4}, []uint64{1, 4}},
		{7, []int64{3, 1, 3}, []uint64{3, 1, 3}},
		{8, []int64{1, 2, 3}, []uint64{1, 2, 3}},
		{9, []int64{1, 2, 3}, []uint64{1, 2, 3}},
	}
	for _, d := range ds {
		validator.enqueue(d.schemaVersion, &transaction.RelatedSchemaChange{PhyTblIDS: d.relatedIDs, ActionTypes: d.relatedActions})
	}
	validator.enqueue(10, &transaction.RelatedSchemaChange{PhyTblIDS: []int64{1}, ActionTypes: []uint64{1}})
	ret := []deltaSchemaInfo{
		{0, []int64{1}, []uint64{1}},
		{2, []int64{1}, []uint64{1}},
		{3, []int64{2, 2}, []uint64{2, 2}},
		{4, []int64{2}, []uint64{2}},
		{6, []int64{1, 4}, []uint64{1, 4}},
		{9, []int64{1, 2, 3}, []uint64{1, 2, 3}},
		{10, []int64{1}, []uint64{1}},
	}
	require.Equal(t, ret, validator.deltaSchemaInfos)
	// The Items' relatedTableIDs have different order.
	validator.enqueue(11, &transaction.RelatedSchemaChange{PhyTblIDS: []int64{1, 2, 3, 4}, ActionTypes: []uint64{1, 2, 3, 4}})
	validator.enqueue(12, &transaction.RelatedSchemaChange{PhyTblIDS: []int64{4, 1, 2, 3, 1}, ActionTypes: []uint64{4, 1, 2, 3, 1}})
	validator.enqueue(13, &transaction.RelatedSchemaChange{PhyTblIDS: []int64{4, 1, 3, 2, 5}, ActionTypes: []uint64{4, 1, 3, 2, 5}})
	ret[len(ret)-1] = deltaSchemaInfo{13, []int64{4, 1, 3, 2, 5}, []uint64{4, 1, 3, 2, 5}}
	require.Equal(t, ret, validator.deltaSchemaInfos)
	// The length of deltaSchemaInfos is greater then maxCnt.
	validator.enqueue(14, &transaction.RelatedSchemaChange{PhyTblIDS: []int64{1}, ActionTypes: []uint64{1}})
	validator.enqueue(15, &transaction.RelatedSchemaChange{PhyTblIDS: []int64{2}, ActionTypes: []uint64{2}})
	validator.enqueue(16, &transaction.RelatedSchemaChange{PhyTblIDS: []int64{3}, ActionTypes: []uint64{3}})
	validator.enqueue(17, &transaction.RelatedSchemaChange{PhyTblIDS: []int64{4}, ActionTypes: []uint64{4}})
	ret = append(ret, deltaSchemaInfo{14, []int64{1}, []uint64{1}})
	ret = append(ret, deltaSchemaInfo{15, []int64{2}, []uint64{2}})
	ret = append(ret, deltaSchemaInfo{16, []int64{3}, []uint64{3}})
	ret = append(ret, deltaSchemaInfo{17, []int64{4}, []uint64{4}})
	require.Equal(t, ret[1:], validator.deltaSchemaInfos)
}

// subTestEnqueueActionType is batched in TestSchemaValidator
func subTestEnqueueActionType(t *testing.T) {
	lease := 10 * time.Millisecond
	originalCnt := variable.GetMaxDeltaSchemaCount()
	defer variable.SetMaxDeltaSchemaCount(originalCnt)

	validator := NewSchemaValidator(lease, nil).(*schemaValidator)
	require.True(t, validator.IsStarted())

	// maxCnt is 0.
	variable.SetMaxDeltaSchemaCount(0)
	validator.enqueue(1, &transaction.RelatedSchemaChange{PhyTblIDS: []int64{11}, ActionTypes: []uint64{11}})
	require.Len(t, validator.deltaSchemaInfos, 0)

	// maxCnt is 10.
	variable.SetMaxDeltaSchemaCount(10)
	ds := []deltaSchemaInfo{
		{0, []int64{1}, []uint64{1}},
		{1, []int64{1}, []uint64{1}},
		{2, []int64{1}, []uint64{1}},
		{3, []int64{2, 2}, []uint64{2, 2}},
		{4, []int64{2}, []uint64{2}},
		{5, []int64{1, 4}, []uint64{1, 4}},
		{6, []int64{1, 4}, []uint64{1, 4}},
		{7, []int64{3, 1, 3}, []uint64{3, 1, 3}},
		{8, []int64{1, 2, 3}, []uint64{1, 2, 3}},
		{9, []int64{1, 2, 3}, []uint64{1, 2, 4}},
	}
	for _, d := range ds {
		validator.enqueue(d.schemaVersion, &transaction.RelatedSchemaChange{PhyTblIDS: d.relatedIDs, ActionTypes: d.relatedActions})
	}
	validator.enqueue(10, &transaction.RelatedSchemaChange{PhyTblIDS: []int64{1}, ActionTypes: []uint64{15}})
	ret := []deltaSchemaInfo{
		{0, []int64{1}, []uint64{1}},
		{2, []int64{1}, []uint64{1}},
		{3, []int64{2, 2}, []uint64{2, 2}},
		{4, []int64{2}, []uint64{2}},
		{6, []int64{1, 4}, []uint64{1, 4}},
		{8, []int64{1, 2, 3}, []uint64{1, 2, 3}},
		{9, []int64{1, 2, 3}, []uint64{1, 2, 4}},
		{10, []int64{1}, []uint64{15}},
	}
	require.Equal(t, ret, validator.deltaSchemaInfos)

	// Check the flag set by schema diff, note tableID = 3 has been set flag 0x3 in schema version 9, and flag 0x4
	// in schema version 10, so the resActions for tableID = 3 should be 0x3 & 0x4 = 0x7.
	relatedChanges, isTablesChanged := validator.isRelatedTablesChanged(5, []int64{1, 2, 3, 4})
	require.True(t, isTablesChanged)
	require.Equal(t, []int64{1, 2, 3, 4}, relatedChanges.PhyTblIDS)
	require.Equal(t, []uint64{15, 2, 7, 4}, relatedChanges.ActionTypes)
}

type leaseGrantItem struct {
	leaseGrantTS uint64
	oldVer       int64
	schemaVer    int64
}

func getGreaterVersionItem(t *testing.T, lease time.Duration, leaseGrantCh chan leaseGrantItem, currVer int64) leaseGrantItem {
	var newItem leaseGrantItem
	for i := 0; i < 10; i++ {
		time.Sleep(lease / 2)
		newItem = <-leaseGrantCh
		if newItem.schemaVer > currVer {
			break
		}
	}
	require.Greaterf(t, newItem.schemaVer, currVer, "currVer %d, newItem %v", currVer, newItem)
	return newItem
}

// serverFunc plays the role as a remote server, runs in a separate goroutine.
// It can grant lease and provide timestamp oracle.
// Caller should communicate with it through channel to mock network.
func serverFunc(lease time.Duration, requireLease chan leaseGrantItem, oracleCh chan uint64, exit chan struct{}) {
	var version int64
	leaseTS := uint64(time.Now().UnixNano())
	ticker := time.NewTicker(lease)
	defer ticker.Stop()
	for {
		select {
		case now := <-ticker.C:
			version++
			leaseTS = uint64(now.UnixNano())
		case requireLease <- leaseGrantItem{
			leaseGrantTS: leaseTS,
			oldVer:       version - 1,
			schemaVer:    version,
		}:
		case oracleCh <- uint64(time.Now().UnixNano()):
		case <-exit:
			return
		}
	}
}
<commit_msg>domain: fix unstable subTestSchemaValidatorGeneral (#37447)<commit_after>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package domain

import (
	"testing"
	"time"

	"github.com/pingcap/tidb/sessionctx/variable"
	"github.com/pingcap/tidb/util"
	"github.com/stretchr/testify/require"
	"github.com/tikv/client-go/v2/oracle"
	"github.com/tikv/client-go/v2/txnkv/transaction"
)

func TestSchemaValidator(t *testing.T) {
	t.Run("general", subTestSchemaValidatorGeneral)
	t.Run("enqueue", subTestEnqueue)
	t.Run("enqueueActionType", subTestEnqueueActionType)
}

// subTestSchemaValidatorGeneral is batched in TestSchemaValidator
func subTestSchemaValidatorGeneral(t *testing.T) {
	lease := 10 * time.Millisecond
	leaseGrantCh := make(chan leaseGrantItem)
	exit := make(chan struct{})
	var wg util.WaitGroupWrapper
	wg.Run(func() { serverFunc(leaseGrantCh, exit) })

	validator := NewSchemaValidator(lease, nil).(*schemaValidator)
	require.True(t, validator.IsStarted())

	for i := 0; i < 3; i++ {
		// Reload can run arbitrarily, at any time.
		item := <-leaseGrantCh
		validator.Update(item.leaseGrantTS, item.oldVer, item.schemaVer, nil)
	}

	// Take a lease, check it's valid.
	item := <-leaseGrantCh
	validator.Update(
		item.leaseGrantTS,
		item.oldVer,
		item.schemaVer,
		&transaction.RelatedSchemaChange{PhyTblIDS: []int64{10}, ActionTypes: []uint64{10}})
	_, valid := validator.Check(item.leaseGrantTS, item.schemaVer, []int64{10})
	require.Equal(t, ResultSucc, valid)

	// Stop the validator, validator's items value is nil.
	validator.Stop()
	require.False(t, validator.IsStarted())
	_, isTablesChanged := validator.isRelatedTablesChanged(item.schemaVer, []int64{10})
	require.True(t, isTablesChanged)
	_, valid = validator.Check(item.leaseGrantTS, item.schemaVer, []int64{10})
	require.Equal(t, ResultUnknown, valid)
	validator.Restart()

	// Increase the current time by 2 leases, check schema is invalid.
	after2LeaseTime := time.Now().Add(2 * lease)
	ts := uint64(after2LeaseTime.UnixNano()) // Make sure that ts has timed out a lease.
	_, valid = validator.Check(ts, item.schemaVer, []int64{10})
	require.Equalf(t, ResultUnknown, valid, "validator latest schema ver %v, time %v, item schema ver %v, ts %v", validator.latestSchemaVer, validator.latestSchemaExpire, 0, oracle.GetTimeFromTS(ts))

	// Make sure newItem's version is greater than item.schema.
	newItem := getGreaterVersionItem(t, leaseGrantCh, item.schemaVer)
	currVer := newItem.schemaVer
	validator.Update(newItem.leaseGrantTS, newItem.oldVer, currVer, nil)
	_, valid = validator.Check(ts, item.schemaVer, nil)
	require.Equalf(t, ResultFail, valid, "currVer %d, newItem %v", currVer, item)
	_, valid = validator.Check(ts, item.schemaVer, []int64{0})
	require.Equalf(t, ResultFail, valid, "currVer %d, newItem %v", currVer, item)

	// Check the latest schema version must changed.
	require.Less(t, item.schemaVer, validator.latestSchemaVer)

	// Make sure newItem's version is greater than currVer.
	newItem = getGreaterVersionItem(t, leaseGrantCh, currVer)
	// Update current schema version to newItem's version and the delta table IDs is 1, 2, 3.
	validator.Update(ts, currVer, newItem.schemaVer, &transaction.RelatedSchemaChange{PhyTblIDS: []int64{1, 2, 3}, ActionTypes: []uint64{1, 2, 3}})
	// Make sure the updated table IDs don't be covered with the same schema version.
	validator.Update(ts, newItem.schemaVer, newItem.schemaVer, nil)
	_, isTablesChanged = validator.isRelatedTablesChanged(currVer, nil)
	require.False(t, isTablesChanged)
	_, isTablesChanged = validator.isRelatedTablesChanged(currVer, []int64{2})
	require.Truef(t, isTablesChanged, "currVer %d, newItem %v", currVer, newItem)
	// The current schema version is older than the oldest schema version.
	_, isTablesChanged = validator.isRelatedTablesChanged(-1, nil)
	require.Truef(t, isTablesChanged, "currVer %d, newItem %v", currVer, newItem)

	// All schema versions is expired.
	ts = uint64(after2LeaseTime.Add(2 * lease).UnixNano())
	_, valid = validator.Check(ts, newItem.schemaVer, nil)
	require.Equal(t, ResultUnknown, valid, "schemaVer %v, validator %#v", newItem.schemaVer, validator)

	close(exit)
	wg.Wait()
}

// subTestEnqueue is batched in TestSchemaValidator
func subTestEnqueue(t *testing.T) {
	lease := 10 * time.Millisecond
	originalCnt := variable.GetMaxDeltaSchemaCount()
	defer variable.SetMaxDeltaSchemaCount(originalCnt)

	validator := NewSchemaValidator(lease, nil).(*schemaValidator)
	require.True(t, validator.IsStarted())

	// maxCnt is 0.
	variable.SetMaxDeltaSchemaCount(0)
	validator.enqueue(1, &transaction.RelatedSchemaChange{PhyTblIDS: []int64{11}, ActionTypes: []uint64{11}})
	require.Len(t, validator.deltaSchemaInfos, 0)

	// maxCnt is 10.
	variable.SetMaxDeltaSchemaCount(10)
	ds := []deltaSchemaInfo{
		{0, []int64{1}, []uint64{1}},
		{1, []int64{1}, []uint64{1}},
		{2, []int64{1}, []uint64{1}},
		{3, []int64{2, 2}, []uint64{2, 2}},
		{4, []int64{2}, []uint64{2}},
		{5, []int64{1, 4}, []uint64{1, 4}},
		{6, []int64{1, 4}, []uint64{1, 4}},
		{7, []int64{3, 1, 3}, []uint64{3, 1, 3}},
		{8, []int64{1, 2, 3}, []uint64{1, 2, 3}},
		{9, []int64{1, 2, 3}, []uint64{1, 2, 3}},
	}
	for _, d := range ds {
		validator.enqueue(d.schemaVersion, &transaction.RelatedSchemaChange{PhyTblIDS: d.relatedIDs, ActionTypes: d.relatedActions})
	}
	validator.enqueue(10, &transaction.RelatedSchemaChange{PhyTblIDS: []int64{1}, ActionTypes: []uint64{1}})
	ret := []deltaSchemaInfo{
		{0, []int64{1}, []uint64{1}},
		{2, []int64{1}, []uint64{1}},
		{3, []int64{2, 2}, []uint64{2, 2}},
		{4, []int64{2}, []uint64{2}},
		{6, []int64{1, 4}, []uint64{1, 4}},
		{9, []int64{1, 2, 3}, []uint64{1, 2, 3}},
		{10, []int64{1}, []uint64{1}},
	}
	require.Equal(t, ret, validator.deltaSchemaInfos)
	// The Items' relatedTableIDs have different order.
	validator.enqueue(11, &transaction.RelatedSchemaChange{PhyTblIDS: []int64{1, 2, 3, 4}, ActionTypes: []uint64{1, 2, 3, 4}})
	validator.enqueue(12, &transaction.RelatedSchemaChange{PhyTblIDS: []int64{4, 1, 2, 3, 1}, ActionTypes: []uint64{4, 1, 2, 3, 1}})
	validator.enqueue(13, &transaction.RelatedSchemaChange{PhyTblIDS: []int64{4, 1, 3, 2, 5}, ActionTypes: []uint64{4, 1, 3, 2, 5}})
	ret[len(ret)-1] = deltaSchemaInfo{13, []int64{4, 1, 3, 2, 5}, []uint64{4, 1, 3, 2, 5}}
	require.Equal(t, ret, validator.deltaSchemaInfos)
	// The length of deltaSchemaInfos is greater then maxCnt.
	validator.enqueue(14, &transaction.RelatedSchemaChange{PhyTblIDS: []int64{1}, ActionTypes: []uint64{1}})
	validator.enqueue(15, &transaction.RelatedSchemaChange{PhyTblIDS: []int64{2}, ActionTypes: []uint64{2}})
	validator.enqueue(16, &transaction.RelatedSchemaChange{PhyTblIDS: []int64{3}, ActionTypes: []uint64{3}})
	validator.enqueue(17, &transaction.RelatedSchemaChange{PhyTblIDS: []int64{4}, ActionTypes: []uint64{4}})
	ret = append(ret, deltaSchemaInfo{14, []int64{1}, []uint64{1}})
	ret = append(ret, deltaSchemaInfo{15, []int64{2}, []uint64{2}})
	ret = append(ret, deltaSchemaInfo{16, []int64{3}, []uint64{3}})
	ret = append(ret, deltaSchemaInfo{17, []int64{4}, []uint64{4}})
	require.Equal(t, ret[1:], validator.deltaSchemaInfos)
}

// subTestEnqueueActionType is batched in TestSchemaValidator
func subTestEnqueueActionType(t *testing.T) {
	lease := 10 * time.Millisecond
	originalCnt := variable.GetMaxDeltaSchemaCount()
	defer variable.SetMaxDeltaSchemaCount(originalCnt)

	validator := NewSchemaValidator(lease, nil).(*schemaValidator)
	require.True(t, validator.IsStarted())

	// maxCnt is 0.
	variable.SetMaxDeltaSchemaCount(0)
	validator.enqueue(1, &transaction.RelatedSchemaChange{PhyTblIDS: []int64{11}, ActionTypes: []uint64{11}})
	require.Len(t, validator.deltaSchemaInfos, 0)

	// maxCnt is 10.
	variable.SetMaxDeltaSchemaCount(10)
	ds := []deltaSchemaInfo{
		{0, []int64{1}, []uint64{1}},
		{1, []int64{1}, []uint64{1}},
		{2, []int64{1}, []uint64{1}},
		{3, []int64{2, 2}, []uint64{2, 2}},
		{4, []int64{2}, []uint64{2}},
		{5, []int64{1, 4}, []uint64{1, 4}},
		{6, []int64{1, 4}, []uint64{1, 4}},
		{7, []int64{3, 1, 3}, []uint64{3, 1, 3}},
		{8, []int64{1, 2, 3}, []uint64{1, 2, 3}},
		{9, []int64{1, 2, 3}, []uint64{1, 2, 4}},
	}
	for _, d := range ds {
		validator.enqueue(d.schemaVersion, &transaction.RelatedSchemaChange{PhyTblIDS: d.relatedIDs, ActionTypes: d.relatedActions})
	}
	validator.enqueue(10, &transaction.RelatedSchemaChange{PhyTblIDS: []int64{1}, ActionTypes: []uint64{15}})
	ret := []deltaSchemaInfo{
		{0, []int64{1}, []uint64{1}},
		{2, []int64{1}, []uint64{1}},
		{3, []int64{2, 2}, []uint64{2, 2}},
		{4, []int64{2}, []uint64{2}},
		{6, []int64{1, 4}, []uint64{1, 4}},
		{8, []int64{1, 2, 3}, []uint64{1, 2, 3}},
		{9, []int64{1, 2, 3}, []uint64{1, 2, 4}},
		{10, []int64{1}, []uint64{15}},
	}
	require.Equal(t, ret, validator.deltaSchemaInfos)

	// Check the flag set by schema diff, note tableID = 3 has been set flag 0x3 in schema version 9, and flag 0x4
	// in schema version 10, so the resActions for tableID = 3 should be 0x3 & 0x4 = 0x7.
	relatedChanges, isTablesChanged := validator.isRelatedTablesChanged(5, []int64{1, 2, 3, 4})
	require.True(t, isTablesChanged)
	require.Equal(t, []int64{1, 2, 3, 4}, relatedChanges.PhyTblIDS)
	require.Equal(t, []uint64{15, 2, 7, 4}, relatedChanges.ActionTypes)
}

type leaseGrantItem struct {
	leaseGrantTS uint64
	oldVer       int64
	schemaVer    int64
}

func getGreaterVersionItem(t *testing.T, leaseGrantCh chan leaseGrantItem, currVer int64) leaseGrantItem {
	newItem := <-leaseGrantCh
	require.Greaterf(t, newItem.schemaVer, currVer, "currVer %d, newItem %v", currVer, newItem)
	return newItem
}

// serverFunc plays the role as a remote server, runs in a separate goroutine.
// It can grant lease and provide timestamp oracle.
// Caller should communicate with it through channel to mock network.
func serverFunc(requireLease chan leaseGrantItem, exit chan struct{}) {
	var version int64
	leaseTS := uint64(time.Now().UnixNano())
	for {
		select {
		case requireLease <- leaseGrantItem{
			leaseGrantTS: leaseTS,
			oldVer:       version - 1,
			schemaVer:    version,
		}:
			version++
			leaseTS = uint64(time.Now().UnixNano())
		case <-exit:
			return
		}
	}
}
<|endoftext|>


<commit_before>package ringbuffer

import (
	"fmt"
	"sync"
	"sync/atomic"
	"time"
)

type RingBuffer struct {
	readIndex  int64      //读序号
	writeIndex int64      //写序号
	buf        []*[]byte  //环形buffer指针数组
	bufSize    int64      //初始化环形buffer指针数组大小
	mask       int64      //初始化环形buffer指针数组大小
	pcond      *sync.Cond //生产者
	ccond      *sync.Cond //消费者
	done       int64      //is done? 1=done; 0=doing
}

func powerOfTwo64(n int64) bool {
	return n != 0 && (n&(n-1)) == 0
}

/**
初始化ringbuffer
参数bufferSize：初始化环形buffer指针数组大小
*/
func NewRingBuffer(size int64) (*RingBuffer, error) {
	if !powerOfTwo64(size) {
		return nil, fmt.Errorf("This size is not able to used")
	}
	buffer := RingBuffer{
		readIndex:  int64(0),
		writeIndex: int64(0),
		buf:        make([]*[]byte, size),
		bufSize:    size,
		mask:       size - int64(1),
		pcond:      sync.NewCond(new(sync.Mutex)),
		ccond:      sync.NewCond(new(sync.Mutex)),
		done:       int64(0),
	}
	for i := int64(0); i < size; i++ {
		buffer.buf[i] = nil
	}
	return &buffer, nil
}

/**
获取当前读序号
*/
func (this *RingBuffer) GetCurrentReadIndex() int64 {
	return atomic.LoadInt64(&this.readIndex)
}

/**
获取当前写序号
*/
func (this *RingBuffer) GetCurrentWriteIndex() int64 {
	return atomic.LoadInt64(&this.writeIndex)
}

/**
读取ringbuffer指定的buffer指针，返回该指针并清空ringbuffer该位置存在的指针内容，以及将读序号加1
*/
func (this *RingBuffer) ReadBuffer() (p *[]byte, ok bool) {
	this.ccond.L.Lock()
	defer func() {
		this.pcond.Signal()
		//this.pcond.Broadcast()
		this.ccond.L.Unlock()
		time.Sleep(5 * time.Millisecond)
	}()
	ok = false
	p = nil
	readIndex := this.GetCurrentReadIndex()
	writeIndex := this.GetCurrentWriteIndex()
	for {
		if this.isDone() {
			return nil, false
		}
		writeIndex = this.GetCurrentWriteIndex()
		if readIndex >= writeIndex {
			this.pcond.Signal()
			//this.pcond.Broadcast()
			this.ccond.Wait()
		} else {
			break
		}
		//time.Sleep(1 * time.Millisecond)
		//time.Sleep(500 * time.Microsecond)
	}
	index := readIndex & this.mask //替代求模
	p = this.buf[index]
	this.buf[index] = nil
	atomic.AddInt64(&this.readIndex, int64(1))
	if p != nil {
		ok = true
	}
	return p, ok
}

/**
写入ringbuffer指针，以及将写序号加1
*/
func (this *RingBuffer) WriteBuffer(in *[]byte) (ok bool) {
	this.pcond.L.Lock()
	defer func() {
		this.ccond.Signal()
		//this.ccond.Broadcast()
		this.pcond.L.Unlock()
		time.Sleep(5 * time.Millisecond)
	}()
	ok = false
	readIndex := this.GetCurrentReadIndex()
	writeIndex := this.GetCurrentWriteIndex()
	for {
		if this.isDone() {
			return false
		}
		readIndex = this.GetCurrentReadIndex()
		if writeIndex >= readIndex && writeIndex-readIndex >= this.bufSize {
			this.ccond.Signal()
			//this.ccond.Broadcast()
			this.pcond.Wait()
		} else {
			break
		}
		//time.Sleep(1 * time.Millisecond)
		//time.Sleep(500 * time.Microsecond)
	}
	index := writeIndex & this.mask //替代求模
	this.buf[index] = in
	atomic.AddInt64(&this.writeIndex, int64(1))
	ok = true
	return ok
}

func (this *RingBuffer) Close() error {
	atomic.StoreInt64(&this.done, 1)

	this.pcond.L.Lock()
	this.ccond.Signal()
	//this.ccond.Broadcast()
	this.pcond.L.Unlock()

	this.ccond.L.Lock()
	this.pcond.Signal()
	//this.pcond.Broadcast()
	this.ccond.L.Unlock()

	return nil
}

func (this *RingBuffer) isDone() bool {
	if atomic.LoadInt64(&this.done) == 1 {
		return true
	}

	return false
}
<commit_msg>修改ringbuffer锁<commit_after>package ringbuffer

import (
	"fmt"
	"sync"
	"sync/atomic"
	"time"
)

type RingBuffer struct {
	readIndex  int64      //读序号
	writeIndex int64      //写序号
	buf        []*[]byte  //环形buffer指针数组
	bufSize    int64      //初始化环形buffer指针数组大小
	mask       int64      //初始化环形buffer指针数组大小
	pcond      *sync.Cond //生产者
	ccond      *sync.Cond //消费者
	done       int64      //is done? 1=done; 0=doing
}

func powerOfTwo64(n int64) bool {
	return n != 0 && (n&(n-1)) == 0
}

/**
初始化ringbuffer
参数bufferSize：初始化环形buffer指针数组大小
*/
func NewRingBuffer(size int64) (*RingBuffer, error) {
	if !powerOfTwo64(size) {
		return nil, fmt.Errorf("This size is not able to used")
	}
	buffer := RingBuffer{
		readIndex:  int64(0),
		writeIndex: int64(0),
		buf:        make([]*[]byte, size),
		bufSize:    size,
		mask:       size - int64(1),
		pcond:      sync.NewCond(new(sync.Mutex)),
		ccond:      sync.NewCond(new(sync.Mutex)),
		done:       int64(0),
	}
	for i := int64(0); i < size; i++ {
		buffer.buf[i] = nil
	}
	return &buffer, nil
}

/**
获取当前读序号
*/
func (this *RingBuffer) GetCurrentReadIndex() int64 {
	return atomic.LoadInt64(&this.readIndex)
}

/**
获取当前写序号
*/
func (this *RingBuffer) GetCurrentWriteIndex() int64 {
	return atomic.LoadInt64(&this.writeIndex)
}

/**
读取ringbuffer指定的buffer指针，返回该指针并清空ringbuffer该位置存在的指针内容，以及将读序号加1
*/
func (this *RingBuffer) ReadBuffer() (p *[]byte, ok bool) {
	this.ccond.L.Lock()
	defer func() {
		this.pcond.Signal()
		//this.pcond.Broadcast()
		this.ccond.L.Unlock()
		time.Sleep(3 * time.Millisecond)
	}()
	ok = false
	p = nil
	readIndex := this.GetCurrentReadIndex()
	writeIndex := this.GetCurrentWriteIndex()
	for {
		if this.isDone() {
			return nil, false
		}
		writeIndex = this.GetCurrentWriteIndex()
		if readIndex >= writeIndex {
			this.pcond.Signal()
			//this.pcond.Broadcast()
			this.ccond.Wait()
		} else {
			break
		}
		//time.Sleep(1 * time.Millisecond)
		//time.Sleep(500 * time.Microsecond)
	}
	index := readIndex & this.mask //替代求模
	p = this.buf[index]
	this.buf[index] = nil
	atomic.AddInt64(&this.readIndex, int64(1))
	if p != nil {
		ok = true
	}
	return p, ok
}

/**
写入ringbuffer指针，以及将写序号加1
*/
func (this *RingBuffer) WriteBuffer(in *[]byte) (ok bool) {
	this.pcond.L.Lock()
	defer func() {
		this.ccond.Signal()
		//this.ccond.Broadcast()
		this.pcond.L.Unlock()
		time.Sleep(3 * time.Millisecond)
	}()
	ok = false
	readIndex := this.GetCurrentReadIndex()
	writeIndex := this.GetCurrentWriteIndex()
	for {
		if this.isDone() {
			return false
		}
		readIndex = this.GetCurrentReadIndex()
		if writeIndex >= readIndex && writeIndex-readIndex >= this.bufSize {
			this.ccond.Signal()
			//this.ccond.Broadcast()
			this.pcond.Wait()
		} else {
			break
		}
		//time.Sleep(1 * time.Millisecond)
		//time.Sleep(500 * time.Microsecond)
	}
	index := writeIndex & this.mask //替代求模
	this.buf[index] = in
	atomic.AddInt64(&this.writeIndex, int64(1))
	ok = true
	return ok
}

func (this *RingBuffer) Close() error {
	atomic.StoreInt64(&this.done, 1)

	this.pcond.L.Lock()
	this.ccond.Signal()
	//this.ccond.Broadcast()
	this.pcond.L.Unlock()

	this.ccond.L.Lock()
	this.pcond.Signal()
	//this.pcond.Broadcast()
	this.ccond.L.Unlock()

	return nil
}

func (this *RingBuffer) isDone() bool {
	if atomic.LoadInt64(&this.done) == 1 {
		return true
	}

	return false
}
<|endoftext|>


<commit_before>package dao

import (
	"errors"
	"fmt"
	"reflect"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

//------------------------------------------------------------
// DAO save methods
//------------------------------------------------------------

// Saves objecst(s). Force ID means generate ID before passing to Mongo.
// Useful if you need this ID right away.
// Method checks if ID already exists and doesn't overwrite it in this case.
// IMPORTANT:
// Force ID mode only supports pointers to structs.
// ObjectId must be a pointer too.
func (dao *DAO) Save(forceId bool, objs ...interface{}) (err error) {
	if forceId {
		err = saveExp(dao.Coll, forceId, objs...)
	} else {
		err = dao.Coll.Insert(objs...)
	}
	return
}

// Experimental implementation of save.
func saveExp(coll *mgo.Collection, forceId bool, objs ...interface{}) (err error) {

	for _, obj := range objs {

		switch reflect.ValueOf(obj).Kind() {
		case reflect.Ptr:
			err = setObjId(obj, forceId)

		case reflect.Struct:
			err = errors.New("Save currently only supports pointers to structs")
			//err = setObjId(&obj, forceId)

		default:
			err = fmt.Errorf("Save does not support objs of type: %v",
				reflect.ValueOf(obj).Kind())
			return
		}
	}

	// Save if all is good
	if coll != nil && err == nil {
		err = coll.Insert(objs...)
	}
	return
}

func setObjId(obj interface{}, forceId bool) (err error) {

	var rval = reflect.ValueOf(obj)
	switch rval.Kind() {

	case reflect.Ptr:
		if !rval.IsValid() {
			err = fmt.Errorf("Element not valid: %v", obj)
			return
		}
		val := rval.Elem()
		typ := rval.Elem().Type()
		fn := findIdField(typ)
		if fn != -1 {
			setIdField(val.Field(fn), forceId)
		} else {
			return fmt.Errorf("Field with tag bson:\"_id\" not found")
		}

	case reflect.Struct:
		typ := reflect.TypeOf(obj)
		fn := findIdField(typ)
		if fn != -1 {
			setIdField(rval.Field(fn), forceId)
		} else {
			return fmt.Errorf("Field with tag bson:\"_id\" not found")
		}
	}

	return
}

func findIdField(typ reflect.Type) int {

	for i := 0; i < typ.NumField(); i++ {
		tag := typ.Field(i).Tag.Get("bson")
		if tag == "_id" || tag == "_id,omitempty" {
			return i
		}
	}
	return -1
}

func setIdField(val reflect.Value, forceId bool) {

	id := bson.NewObjectId()
	switch val.Kind() {

	case reflect.Ptr:
		//fmt.Println("\t\tField is a ptr, isNil =", val.IsNil())
		// Only set if value is nil and force ID creation was requested
		if val.IsNil() && forceId {
			val.Set(reflect.ValueOf(&id))
		}

	case reflect.Struct:
		// Only as an exercise, not really supported
		fmt.Println("\t\tField is a struct")
		val.Set(reflect.ValueOf(id))
	}
}
<commit_msg>update<commit_after>package dao

import (
	"errors"
	"fmt"
	"reflect"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

//------------------------------------------------------------
// DAO save methods
//------------------------------------------------------------

// Saves objecst(s). Force ID means generate ID before passing to Mongo.
// Useful if you need this ID right away.
// Method checks if ID already exists and doesn't overwrite it in this case.
// IMPORTANT:
// Force ID mode only supports pointers to structs.
// ObjectId must be a pointer too.
func (dao *DAO) Save(forceID bool, objs ...interface{}) (err error) {
	if forceID {
		err = saveExp(dao.Coll, forceID, objs...)
	} else {
		err = dao.Coll.Insert(objs...)
	}
	return
}

// Experimental implementation of save.
func saveExp(coll *mgo.Collection, forceID bool, objs ...interface{}) (err error) {

	for _, obj := range objs {

		switch reflect.ValueOf(obj).Kind() {
		case reflect.Ptr:
			err = setObjId(obj, forceID)

		case reflect.Struct:
			err = errors.New("Save currently only supports pointers to structs")
			//err = setObjId(&obj, forceID)

		default:
			err = fmt.Errorf("Save does not support objs of type: %v",
				reflect.ValueOf(obj).Kind())
			return
		}
	}

	// Save if all is good
	if coll != nil && err == nil {
		err = coll.Insert(objs...)
	}
	return
}

func setObjId(obj interface{}, forceID bool) (err error) {

	var rval = reflect.ValueOf(obj)
	switch rval.Kind() {

	case reflect.Ptr:
		if !rval.IsValid() {
			err = fmt.Errorf("Element not valid: %v", obj)
			return
		}
		val := rval.Elem()
		typ := rval.Elem().Type()
		fn := findIdField(typ)
		if fn != -1 {
			setIdField(val.Field(fn), forceID)
		} else {
			return fmt.Errorf("Field with tag bson:\"_id\" not found")
		}

	case reflect.Struct:
		typ := reflect.TypeOf(obj)
		fn := findIdField(typ)
		if fn != -1 {
			setIdField(rval.Field(fn), forceID)
		} else {
			return fmt.Errorf("Field with tag bson:\"_id\" not found")
		}
	}

	return
}

func findIdField(typ reflect.Type) int {

	for i := 0; i < typ.NumField(); i++ {
		tag := typ.Field(i).Tag.Get("bson")
		if tag == "_id" || tag == "_id,omitempty" {
			return i
		}
	}
	return -1
}

func setIdField(val reflect.Value, forceID bool) {

	id := bson.NewObjectId()
	switch val.Kind() {

	case reflect.Ptr:
		//fmt.Println("\t\tField is a ptr, isNil =", val.IsNil())
		// Only set if value is nil and force ID creation was requested
		if val.IsNil() && forceID {
			val.Set(reflect.ValueOf(&id))
		}

	case reflect.Struct:
		// Only as an exercise, not really supported
		fmt.Println("\t\tField is a struct")
		val.Set(reflect.ValueOf(id))
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/tls"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"sync"

	"golang.org/x/net/html"
)

// Results -- struct returned by Crawl() to represent the entire crawl process
type Results struct {
	// Inherit the Resource struct
	Resource

	// Body represents a string implementation of the byte array returned by
	// http.Response
	Body string

	// Slice of Resource structs containing the needed resources for the given URL
	Resources []*Resource

	// ResourceTime shows how long it took to fetch all resources
	ResourceTime *TimerResult

	// TotalTime represents the time it took to crawl the site
	TotalTime *TimerResult
}

// Resource represents a single entity of many within a given crawl. These should
// only be of type css, js, jpg, png, etc (static resources).
type Resource struct {
	// connURL is the initial URL received by input
	connURL string

	// connIP is the initial IP address received by input
	connIP string

	// connHostname represents the original requested hostname for the resource
	connHostname string

	// URL represents the resulting static URL derived by the original result page
	URL string

	// Hostname represents the resulting hostname derived by the original returned
	// resource
	Hostname string

	// Remote represents if the resulting resource is remote to the original domain
	Remote bool

	// Error represents any errors that may have occurred when fetching the resource
	Error error

	// Code represents the numeric HTTP based status code
	Code int

	// Proto represents the end protocol used to fetch the page. For example, HTTP/2.0
	Proto string

	// Scheme represents the end scheme used to fetch the page. For example, https
	Scheme string

	// ContentLength represents the number of bytes in the body of the response
	ContentLength int64

	// TLS represents the SSL/TLS handshake/session if the resource was loaded over
	// SSL.
	TLS *tls.ConnectionState

	// Time represents the time it took to complete the request
	Time *TimerResult
}

var resourcePool sync.WaitGroup

// getSrc crawls the body of the Results page, yielding all img/script/link resources
// so they can later be fetched.
func getSrc(b io.ReadCloser, req *http.Request) (urls []string) {
	urls = []string{}

	z := html.NewTokenizer(b)

	for {
		// loop through all tokens in the html body response
		tt := z.Next()

		switch {
		case tt == html.ErrorToken:
			// this assumes that there are no further tokens -- end of document
			return
		case tt == html.StartTagToken:
			t := z.Token()

			// the tokens that we are pulling resources from, and the attribute we are
			// pulling from
			allowed := map[string]string{
				"link":   "href",
				"script": "src",
				"img":    "src",
			}
			var isInAllowed bool
			var checkType string
			var src string

			// loop through all allowed elements, and see if the current element is
			// allowed
			for key := range allowed {
				if t.Data == key {
					isInAllowed = true
					checkType = allowed[key]
					break
				}
			}

			if !isInAllowed {
				continue
			}

			for _, a := range t.Attr {
				if a.Key == checkType {
					src = a.Val
					break
				}
			}

			// this assumes that the resource is something along the lines of:
			//   http://something.com/ -- which we don't care about
			if len(src) == 0 || strings.HasSuffix(src, "/") {
				continue
			}

			// add trailing slash to the end of the path
			if len(req.URL.Path) == 0 {
				req.URL.Path = "/"
			}

			// site was developed using relative paths. E.g:
			//  - url: http://domain.com/sub/path and resource: ./something/main.js
			//    would equal http://domain.com/sub/path/something/main.js
			if strings.HasPrefix(src, "./") {
				src = req.URL.Scheme + "://" + req.URL.Host + req.URL.Path + strings.SplitN(src, "./", 2)[1]
			}

			// site is loading resources from a remote location that supports both
			// http and https. browsers should natively tack on the current sites
			// protocol to the url. E.g:
			//  - url: http://domain.com/ and resource: //other.com/some-resource.js
			//    generates: http://other.com/some-resource.js
			//  - url: https://domain.com/ and resource: //other.com/some-resource.js
			//    generates: https://other.com/some-resource.js
			if strings.HasPrefix(src, "//") {
				src = req.URL.Scheme + ":" + src
			}

			// non-host-absolute resource. E.g. resource is loaded based on the docroot
			// of the domain. E.g:
			//  - url: http://domain.com/ and resource: /some-resource.js
			//    generates: http://domain.com/some-resource.js
			//  - url: https://domain.com/sub/resource and resource: /some-resource.js
			//    generates: https://domain.com/some-resource.js
			if strings.HasPrefix(src, "/") {
				src = req.URL.Scheme + "://" + req.URL.Host + src
			}

			// ignore anything else that isn't http based. E.g. ftp://, and other svg-like
			// data urls, as we really can't fetch those.
			if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
				continue
			}

			urls = append(urls, src)
		}
	}
}

func connHostname(URL string) (host string, err error) {
	tmp, err := url.Parse(URL)

	if err != nil {
		return
	}

	host = tmp.Host
	return
}

// FetchResource fetches a singular resource from a page, returning a *Resource struct.
// As we don't care much about the body of the resource, that can safely be ignored. We
// must still close the body object, however.
func (rsrc *Resource) FetchResource() {
	var err error

	defer resourcePool.Done()

	// calculate the time it takes to fetch the request
	timer := NewTimer()
	resp, err := Get(rsrc.connURL, rsrc.connIP)
	rsrc.Time = timer.End()
	resp.Body.Close()

	if err != nil {
		rsrc.Error = err
		return
	}

	rsrc.connHostname, err = connHostname(rsrc.connURL)
	if err != nil {
		rsrc.Error = err
		return
	}

	rsrc.Hostname = resp.Request.Host
	rsrc.URL = resp.Request.URL.String()
	rsrc.Code = resp.StatusCode
	rsrc.Proto = resp.Proto
	rsrc.Scheme = resp.Request.URL.Scheme
	rsrc.ContentLength = resp.ContentLength
	rsrc.TLS = resp.TLS

	if rsrc.Hostname != rsrc.connHostname {
		rsrc.Remote = true
	}

	fmt.Printf("[%d] [%s] %s\n", rsrc.Code, rsrc.Proto, rsrc.URL)

	return
}

// Crawl manages the fetching of the main resource, as well as all child resources,
// providing a Results struct containing the entire crawl data needed
func Crawl(URL string, IP string) (res *Results) {
	res = &Results{}

	crawlTimer := NewTimer()
	reqTimer := NewTimer()

	// actually fetch the request
	resp, err := Get(URL, IP)

	res.Time = reqTimer.End()

	if err != nil {
		res.Error = err
		return
	}

	defer resp.Body.Close()

	res.connHostname, err = connHostname(URL)
	if err != nil {
		res.Error = err
		return
	}

	res.connURL = URL
	res.connIP = IP
	res.Hostname = resp.Request.Host
	res.URL = URL
	res.Code = resp.StatusCode
	res.Proto = resp.Proto
	res.Scheme = resp.Request.URL.Scheme
	res.ContentLength = resp.ContentLength
	res.TLS = resp.TLS

	if res.Hostname != res.connHostname {
		res.Remote = true
	}

	buf, _ := ioutil.ReadAll(resp.Body)
	b := ioutil.NopCloser(bytes.NewReader(buf))
	defer b.Close()

	bbytes, err := ioutil.ReadAll(bytes.NewBuffer(buf))
	if len(bbytes) != 0 {
		res.Body = string(bbytes[:])
	}

	urls := getSrc(b, resp.Request)

	fmt.Printf("[%d] [%s] %s\n", res.Code, res.Proto, res.URL)

	resourceTime := NewTimer()

	for i := range urls {
		resourcePool.Add(1)

		rsrc := &Resource{connURL: urls[i], connIP: ""}
		res.Resources = append(res.Resources, rsrc)
		go res.Resources[i].FetchResource()
	}

	resourcePool.Wait()

	res.ResourceTime = resourceTime.End()
	res.TotalTime = crawlTimer.End()

	return
}
<commit_msg>also catch self closing tags<commit_after>package main

import (
	"bytes"
	"crypto/tls"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"sync"

	"golang.org/x/net/html"
)

// Results -- struct returned by Crawl() to represent the entire crawl process
type Results struct {
	// Inherit the Resource struct
	Resource

	// Body represents a string implementation of the byte array returned by
	// http.Response
	Body string

	// Slice of Resource structs containing the needed resources for the given URL
	Resources []*Resource

	// ResourceTime shows how long it took to fetch all resources
	ResourceTime *TimerResult

	// TotalTime represents the time it took to crawl the site
	TotalTime *TimerResult
}

// Resource represents a single entity of many within a given crawl. These should
// only be of type css, js, jpg, png, etc (static resources).
type Resource struct {
	// connURL is the initial URL received by input
	connURL string

	// connIP is the initial IP address received by input
	connIP string

	// connHostname represents the original requested hostname for the resource
	connHostname string

	// URL represents the resulting static URL derived by the original result page
	URL string

	// Hostname represents the resulting hostname derived by the original returned
	// resource
	Hostname string

	// Remote represents if the resulting resource is remote to the original domain
	Remote bool

	// Error represents any errors that may have occurred when fetching the resource
	Error error

	// Code represents the numeric HTTP based status code
	Code int

	// Proto represents the end protocol used to fetch the page. For example, HTTP/2.0
	Proto string

	// Scheme represents the end scheme used to fetch the page. For example, https
	Scheme string

	// ContentLength represents the number of bytes in the body of the response
	ContentLength int64

	// TLS represents the SSL/TLS handshake/session if the resource was loaded over
	// SSL.
	TLS *tls.ConnectionState

	// Time represents the time it took to complete the request
	Time *TimerResult
}

var resourcePool sync.WaitGroup

// getSrc crawls the body of the Results page, yielding all img/script/link resources
// so they can later be fetched.
func getSrc(b io.ReadCloser, req *http.Request) (urls []string) {
	urls = []string{}

	z := html.NewTokenizer(b)

	for {
		// loop through all tokens in the html body response
		tt := z.Next()

		switch {
		case tt == html.ErrorToken:
			// this assumes that there are no further tokens -- end of document
			return
		case tt == html.StartTagToken || tt == html.SelfClosingTagToken:
			t := z.Token()

			// the tokens that we are pulling resources from, and the attribute we are
			// pulling from
			allowed := map[string]string{
				"link":   "href",
				"script": "src",
				"img":    "src",
			}
			var isInAllowed bool
			var checkType string
			var src string

			// loop through all allowed elements, and see if the current element is
			// allowed
			for key := range allowed {
				if t.Data == key {
					isInAllowed = true
					checkType = allowed[key]
					break
				}
			}

			if !isInAllowed {
				continue
			}

			for _, a := range t.Attr {
				if a.Key == checkType {
					src = a.Val
					break
				}
			}

			// this assumes that the resource is something along the lines of:
			//   http://something.com/ -- which we don't care about
			if len(src) == 0 || strings.HasSuffix(src, "/") {
				continue
			}

			// add trailing slash to the end of the path
			if len(req.URL.Path) == 0 {
				req.URL.Path = "/"
			}

			// site was developed using relative paths. E.g:
			//  - url: http://domain.com/sub/path and resource: ./something/main.js
			//    would equal http://domain.com/sub/path/something/main.js
			if strings.HasPrefix(src, "./") {
				src = req.URL.Scheme + "://" + req.URL.Host + req.URL.Path + strings.SplitN(src, "./", 2)[1]
			}

			// site is loading resources from a remote location that supports both
			// http and https. browsers should natively tack on the current sites
			// protocol to the url. E.g:
			//  - url: http://domain.com/ and resource: //other.com/some-resource.js
			//    generates: http://other.com/some-resource.js
			//  - url: https://domain.com/ and resource: //other.com/some-resource.js
			//    generates: https://other.com/some-resource.js
			if strings.HasPrefix(src, "//") {
				src = req.URL.Scheme + ":" + src
			}

			// non-host-absolute resource. E.g. resource is loaded based on the docroot
			// of the domain. E.g:
			//  - url: http://domain.com/ and resource: /some-resource.js
			//    generates: http://domain.com/some-resource.js
			//  - url: https://domain.com/sub/resource and resource: /some-resource.js
			//    generates: https://domain.com/some-resource.js
			if strings.HasPrefix(src, "/") {
				src = req.URL.Scheme + "://" + req.URL.Host + src
			}

			// ignore anything else that isn't http based. E.g. ftp://, and other svg-like
			// data urls, as we really can't fetch those.
			if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
				continue
			}

			urls = append(urls, src)
		}
	}
}

func connHostname(URL string) (host string, err error) {
	tmp, err := url.Parse(URL)

	if err != nil {
		return
	}

	host = tmp.Host
	return
}

// FetchResource fetches a singular resource from a page, returning a *Resource struct.
// As we don't care much about the body of the resource, that can safely be ignored. We
// must still close the body object, however.
func (rsrc *Resource) FetchResource() {
	var err error

	defer resourcePool.Done()

	// calculate the time it takes to fetch the request
	timer := NewTimer()
	resp, err := Get(rsrc.connURL, rsrc.connIP)
	rsrc.Time = timer.End()
	resp.Body.Close()

	if err != nil {
		rsrc.Error = err
		return
	}

	rsrc.connHostname, err = connHostname(rsrc.connURL)
	if err != nil {
		rsrc.Error = err
		return
	}

	rsrc.Hostname = resp.Request.Host
	rsrc.URL = resp.Request.URL.String()
	rsrc.Code = resp.StatusCode
	rsrc.Proto = resp.Proto
	rsrc.Scheme = resp.Request.URL.Scheme
	rsrc.ContentLength = resp.ContentLength
	rsrc.TLS = resp.TLS

	if rsrc.Hostname != rsrc.connHostname {
		rsrc.Remote = true
	}

	fmt.Printf("[%d] [%s] %s\n", rsrc.Code, rsrc.Proto, rsrc.URL)

	return
}

// Crawl manages the fetching of the main resource, as well as all child resources,
// providing a Results struct containing the entire crawl data needed
func Crawl(URL string, IP string) (res *Results) {
	res = &Results{}

	crawlTimer := NewTimer()
	reqTimer := NewTimer()

	// actually fetch the request
	resp, err := Get(URL, IP)

	res.Time = reqTimer.End()

	if err != nil {
		res.Error = err
		return
	}

	defer resp.Body.Close()

	res.connHostname, err = connHostname(URL)
	if err != nil {
		res.Error = err
		return
	}

	res.connURL = URL
	res.connIP = IP
	res.Hostname = resp.Request.Host
	res.URL = URL
	res.Code = resp.StatusCode
	res.Proto = resp.Proto
	res.Scheme = resp.Request.URL.Scheme
	res.ContentLength = resp.ContentLength
	res.TLS = resp.TLS

	if res.Hostname != res.connHostname {
		res.Remote = true
	}

	buf, _ := ioutil.ReadAll(resp.Body)
	b := ioutil.NopCloser(bytes.NewReader(buf))
	defer b.Close()

	bbytes, err := ioutil.ReadAll(bytes.NewBuffer(buf))
	if len(bbytes) != 0 {
		res.Body = string(bbytes[:])
	}

	urls := getSrc(b, resp.Request)

	fmt.Printf("[%d] [%s] %s\n", res.Code, res.Proto, res.URL)

	resourceTime := NewTimer()

	for i := range urls {
		resourcePool.Add(1)

		rsrc := &Resource{connURL: urls[i], connIP: ""}
		res.Resources = append(res.Resources, rsrc)
		go res.Resources[i].FetchResource()
	}

	resourcePool.Wait()

	res.ResourceTime = resourceTime.End()
	res.TotalTime = crawlTimer.End()

	return
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"path"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

// A Post is reflective of the JSON used in the tumblr API.
// It contains a PhotoURL, and, optionally, an array of photos.
// If Photos isn't empty, it typically contains at least one URL which matches PhotoURL.
type Post struct {
	ID            string
	Type          string
	PhotoURL      string `json:"photo-url-1280"`
	Photos        []Post `json:"photos,omitempty"`
	UnixTimestamp int64  `json:"unix-timestamp"`
	PhotoCaption  string `json:"photo-caption"`

	// for regular posts
	RegularBody string `json:"regular-body"`

	// for answer posts
	Answer string

	// for videos
	Video        string `json:"video-player"`
	VideoCaption string `json:"video-caption"` // For links to outside sites.
}

// A Blog is the outer container for Posts. It is necessary for easier JSON deserialization,
// even though it's useless in and of itself.
type Blog struct {
	Posts []Post `json:"posts"`
}

var (
	inlineSearch   = regexp.MustCompile(`(http:\/\/\d{2}\.media\.tumblr\.com\/\w{32}\/tumblr_inline_\w+\.\w+)`) // FIXME: Possibly buggy/unoptimized.
	videoSearch    = regexp.MustCompile(`"hdUrl":"(.*\/tumblr_\w+)"`)                                           // fuck it
	altVideoSearch = regexp.MustCompile(`source src="(.*)\/\d+" type`)
	gfycatSearch   = regexp.MustCompile(`href="https?:\/\/(?:www\.)?gfycat\.com\/(\w+)`)
)

// a href=\"http://www.gfycat.com/AcademicEveryBlackbear\"

func scrape(user *blog, limiter <-chan time.Time) <-chan Image {
	var wg sync.WaitGroup
	highestID := "0"
	var IDMutex sync.Mutex

	var once sync.Once
	imageChannel := make(chan Image)

	go func() {

		done := make(chan struct{})
		closeDone := func() { close(done) }

		defer updateDatabase(user.name, &highestID)
		defer close(imageChannel)
		defer wg.Wait()
		defer fmt.Println("Done scraping for", user.name)

		for i := 1; ; i++ {
			select {
			case <-done:
				return
			default:
				select {
				case <-done:
					return
				case <-limiter:
					// We get a value from limiter, and proceed to scrape a page.
				}
			}

			base := fmt.Sprintf("http://%s.tumblr.com/api/read/json", user.name)

			tumblrURL, err := url.Parse(base)
			if err != nil {
				log.Fatal("tumblrURL: ", err)
			}

			vals := url.Values{}
			vals.Set("num", "50")
			vals.Add("start", strconv.Itoa((i-1)*50))
			// vals.Add("type", "photo")

			if user.tag != "" {
				vals.Add("tagged", user.tag)
			}

			tumblrURL.RawQuery = vals.Encode()

			fmt.Println(user.name, "is on page", i)
			resp, err := http.Get(tumblrURL.String())

			// XXX: Ugly as shit. This could probably be done better.
			if err != nil {
				i--
				log.Println(user, err)
				continue
			}
			defer resp.Body.Close()

			contents, _ := ioutil.ReadAll(resp.Body)

			// This is returned as pure javascript. We need to filter out the variable and the ending semicolon.
			contents = []byte(strings.Replace(string(contents), "var tumblr_api_read = ", "", 1))
			contents = []byte(strings.Replace(string(contents), ";", "", -1))

			var blog Blog
			json.Unmarshal(contents, &blog)

			if len(blog.Posts) == 0 {
				break
			}

			wg.Add(1)

			go func() {
				defer wg.Done()
				lastPostIDint, err := strconv.Atoi(user.lastPostID)
				if err != nil {
					log.Fatal("parse1", err)
				}
				for _, post := range blog.Posts {
					postIDint, _ := strconv.Atoi(post.ID)

					IDMutex.Lock()
					highestIDint, _ := strconv.Atoi(highestID)
					if postIDint >= highestIDint {
						highestID = post.ID
					}
					IDMutex.Unlock()

					if (postIDint <= lastPostIDint) && updateMode {
						once.Do(closeDone)
						break
					}

					var URLs []string

					switch post.Type { // TODO: Refactor and clean this up. This is messy and has repeated code.
					case "photo":
						if !ignorePhotos {
							if len(post.Photos) == 0 {
								URLs = append(URLs, post.PhotoURL)
							} else {
								for _, photo := range post.Photos {
									URLs = append(URLs, photo.PhotoURL)
								}
							}
						}

						if !ignoreVideos {
							regexResult := gfycatSearch.FindStringSubmatch(post.PhotoCaption)
							if regexResult != nil {
								for _, v := range regexResult[1:] {
									URLs = append(URLs, GetGfycatURL(v))
								}
							}
						}

					case "answer":
						if !ignorePhotos {
							URLs = inlineSearch.FindAllString(post.Answer, -1)
						}
					case "regular":
						if !ignorePhotos {
							URLs = inlineSearch.FindAllString(post.RegularBody, -1)
						}
					case "video":
						if !ignoreVideos {
							regextest := videoSearch.FindStringSubmatch(post.Video)
							if regextest == nil { // hdUrl is false. We have to get the other URL.
								regextest = altVideoSearch.FindStringSubmatch(post.Video)
							}

							// If it's still nil, it means it's another embedded video type, like Youtube, Vine or Pornhub.
							// In that case, ignore it and move on. Not my problem.
							if regextest == nil {
								continue
							}
							videoURL := strings.Replace(regextest[1], `\`, ``, -1)

							// If there are problems with downloading video, the below part may be the cause.
							// videoURL = strings.Replace(videoURL, `/480`, ``, -1)
							videoURL += ".mp4"

							URLs = append(URLs, videoURL)

							// Here, we get the GfyCat urls from the post.
							regextest = gfycatSearch.FindStringSubmatch(post.VideoCaption)
							if regextest != nil {
								for _, v := range regextest[1:] {
									URLs = append(URLs, GetGfycatURL(v))
								}
							}
						}

					default:
						continue
					}

					// fmt.Println(URLs)

					for _, URL := range URLs {
						i := Image{
							User:          user.name,
							URL:           URL,
							UnixTimestamp: post.UnixTimestamp,
						}

						filename := path.Base(i.URL)
						pathname := fmt.Sprintf("%s/%s", user.name, filename)

						// If there is a file that exists, we skip adding it and move on to the next one.
						// Or, if update mode is enabled, then we can simply stop searching.
						_, err := os.Stat(pathname)
						if err == nil {
							atomic.AddUint64(&alreadyExists, 1)
							continue

						}

						atomic.AddUint64(&totalFound, 1)
						imageChannel <- i
					}

				}

			}()

		}

	}()
	return imageChannel
}
<commit_msg>Fix video detection regex<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"path"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

// A Post is reflective of the JSON used in the tumblr API.
// It contains a PhotoURL, and, optionally, an array of photos.
// If Photos isn't empty, it typically contains at least one URL which matches PhotoURL.
type Post struct {
	ID            string
	Type          string
	PhotoURL      string `json:"photo-url-1280"`
	Photos        []Post `json:"photos,omitempty"`
	UnixTimestamp int64  `json:"unix-timestamp"`
	PhotoCaption  string `json:"photo-caption"`

	// for regular posts
	RegularBody string `json:"regular-body"`

	// for answer posts
	Answer string

	// for videos
	Video        string `json:"video-player"`
	VideoCaption string `json:"video-caption"` // For links to outside sites.
}

// A Blog is the outer container for Posts. It is necessary for easier JSON deserialization,
// even though it's useless in and of itself.
type Blog struct {
	Posts []Post `json:"posts"`
}

var (
	inlineSearch   = regexp.MustCompile(`(http:\/\/\d{2}\.media\.tumblr\.com\/\w{32}\/tumblr_inline_\w+\.\w+)`) // FIXME: Possibly buggy/unoptimized.
	videoSearch    = regexp.MustCompile(`"hdUrl":"(.*\/tumblr_\w+)"`)                                           // fuck it
	altVideoSearch = regexp.MustCompile(`source src="(.*tumblr_\w+)(?:\/\d+)?" type`)
	gfycatSearch   = regexp.MustCompile(`href="https?:\/\/(?:www\.)?gfycat\.com\/(\w+)`)
)

// a href=\"http://www.gfycat.com/AcademicEveryBlackbear\"

func scrape(user *blog, limiter <-chan time.Time) <-chan Image {
	var wg sync.WaitGroup
	highestID := "0"
	var IDMutex sync.Mutex

	var once sync.Once
	imageChannel := make(chan Image)

	go func() {

		done := make(chan struct{})
		closeDone := func() { close(done) }

		defer updateDatabase(user.name, &highestID)
		defer close(imageChannel)
		defer wg.Wait()
		defer fmt.Println("Done scraping for", user.name)

		for i := 1; ; i++ {
			select {
			case <-done:
				return
			default:
				select {
				case <-done:
					return
				case <-limiter:
					// We get a value from limiter, and proceed to scrape a page.
				}
			}

			base := fmt.Sprintf("http://%s.tumblr.com/api/read/json", user.name)

			tumblrURL, err := url.Parse(base)
			if err != nil {
				log.Fatal("tumblrURL: ", err)
			}

			vals := url.Values{}
			vals.Set("num", "50")
			vals.Add("start", strconv.Itoa((i-1)*50))
			// vals.Add("type", "photo")

			if user.tag != "" {
				vals.Add("tagged", user.tag)
			}

			tumblrURL.RawQuery = vals.Encode()

			fmt.Println(user.name, "is on page", i)
			resp, err := http.Get(tumblrURL.String())

			// XXX: Ugly as shit. This could probably be done better.
			if err != nil {
				i--
				log.Println(user, err)
				continue
			}
			defer resp.Body.Close()

			contents, _ := ioutil.ReadAll(resp.Body)

			// This is returned as pure javascript. We need to filter out the variable and the ending semicolon.
			contents = []byte(strings.Replace(string(contents), "var tumblr_api_read = ", "", 1))
			contents = []byte(strings.Replace(string(contents), ";", "", -1))

			var blog Blog
			json.Unmarshal(contents, &blog)

			if len(blog.Posts) == 0 {
				break
			}

			wg.Add(1)

			go func() {
				defer wg.Done()
				lastPostIDint, err := strconv.Atoi(user.lastPostID)
				if err != nil {
					log.Fatal("parse1", err)
				}
				for _, post := range blog.Posts {
					postIDint, _ := strconv.Atoi(post.ID)

					IDMutex.Lock()
					highestIDint, _ := strconv.Atoi(highestID)
					if postIDint >= highestIDint {
						highestID = post.ID
					}
					IDMutex.Unlock()

					if (postIDint <= lastPostIDint) && updateMode {
						once.Do(closeDone)
						break
					}

					var URLs []string

					switch post.Type { // TODO: Refactor and clean this up. This is messy and has repeated code.
					case "photo":
						if !ignorePhotos {
							if len(post.Photos) == 0 {
								URLs = append(URLs, post.PhotoURL)
							} else {
								for _, photo := range post.Photos {
									URLs = append(URLs, photo.PhotoURL)
								}
							}
						}

						if !ignoreVideos {
							regexResult := gfycatSearch.FindStringSubmatch(post.PhotoCaption)
							if regexResult != nil {
								for _, v := range regexResult[1:] {
									URLs = append(URLs, GetGfycatURL(v))
								}
							}
						}

					case "answer":
						if !ignorePhotos {
							URLs = inlineSearch.FindAllString(post.Answer, -1)
						}
					case "regular":
						if !ignorePhotos {
							URLs = inlineSearch.FindAllString(post.RegularBody, -1)
						}
					case "video":
						if !ignoreVideos {
							regextest := videoSearch.FindStringSubmatch(post.Video)
							if regextest == nil { // hdUrl is false. We have to get the other URL.
								regextest = altVideoSearch.FindStringSubmatch(post.Video)
							}

							// If it's still nil, it means it's another embedded video type, like Youtube, Vine or Pornhub.
							// In that case, ignore it and move on. Not my problem.
							if regextest == nil {
								continue
							}
							videoURL := strings.Replace(regextest[1], `\`, ``, -1)

							// If there are problems with downloading video, the below part may be the cause.
							// videoURL = strings.Replace(videoURL, `/480`, ``, -1)
							videoURL += ".mp4"

							URLs = append(URLs, videoURL)

							// Here, we get the GfyCat urls from the post.
							regextest = gfycatSearch.FindStringSubmatch(post.VideoCaption)
							if regextest != nil {
								for _, v := range regextest[1:] {
									URLs = append(URLs, GetGfycatURL(v))
								}
							}
						}

					default:
						continue
					}

					// fmt.Println(URLs)

					for _, URL := range URLs {
						i := Image{
							User:          user.name,
							URL:           URL,
							UnixTimestamp: post.UnixTimestamp,
						}

						filename := path.Base(i.URL)
						pathname := fmt.Sprintf("%s/%s", user.name, filename)

						// If there is a file that exists, we skip adding it and move on to the next one.
						// Or, if update mode is enabled, then we can simply stop searching.
						_, err := os.Stat(pathname)
						if err == nil {
							atomic.AddUint64(&alreadyExists, 1)
							continue

						}

						atomic.AddUint64(&totalFound, 1)
						imageChannel <- i
					}

				}

			}()

		}

	}()
	return imageChannel
}
<|endoftext|>


<commit_before>// Code generated by "mapstructure-to-hcl2 -type AmiFilterOptions,SecurityGroupFilterOptions,SubnetFilterOptions,VpcFilterOptions,PolicyDocument,Statement"; DO NOT EDIT.
package common

import (
	"github.com/hashicorp/hcl/v2/hcldec"
	"github.com/hashicorp/packer/hcl2template"
	"github.com/zclconf/go-cty/cty"
)

// FlatAmiFilterOptions is an auto-generated flat version of AmiFilterOptions.
// Where the contents of a field with a `mapstructure:,squash` tag are bubbled up.
type FlatAmiFilterOptions struct {
	Filters    map[string]string           `cty:"filters" hcl:"filters"`
	Filter     []hcl2template.FlatKeyValue `cty:"filter" hcl:"filter"`
	Owners     []string                    `cty:"owners" hcl:"owners"`
	MostRecent *bool                       `mapstructure:"most_recent" cty:"most_recent" hcl:"most_recent"`
}

// FlatMapstructure returns a new FlatAmiFilterOptions.
// FlatAmiFilterOptions is an auto-generated flat version of AmiFilterOptions.
// Where the contents a fields with a `mapstructure:,squash` tag are bubbled up.
func (*AmiFilterOptions) FlatMapstructure() interface{ HCL2Spec() map[string]hcldec.Spec } {
	return new(FlatAmiFilterOptions)
}

// HCL2Spec returns the hcl spec of a AmiFilterOptions.
// This spec is used by HCL to read the fields of AmiFilterOptions.
// The decoded values from this spec will then be applied to a FlatAmiFilterOptions.
func (*FlatAmiFilterOptions) HCL2Spec() map[string]hcldec.Spec {
	s := map[string]hcldec.Spec{
		"filters":     &hcldec.AttrSpec{Name: "filters", Type: cty.Map(cty.String), Required: false},
		"filter":      &hcldec.BlockListSpec{TypeName: "filter", Nested: hcldec.ObjectSpec((*hcl2template.FlatKeyValue)(nil).HCL2Spec())},
		"owners":      &hcldec.AttrSpec{Name: "owners", Type: cty.List(cty.String), Required: false},
		"most_recent": &hcldec.AttrSpec{Name: "most_recent", Type: cty.Bool, Required: false},
	}
	return s
}

// FlatPolicyDocument is an auto-generated flat version of PolicyDocument.
// Where the contents of a field with a `mapstructure:,squash` tag are bubbled up.
type FlatPolicyDocument struct {
	Version   *string         `cty:"version" hcl:"version"`
	Statement []FlatStatement `cty:"statement" hcl:"statement"`
}

// FlatMapstructure returns a new FlatPolicyDocument.
// FlatPolicyDocument is an auto-generated flat version of PolicyDocument.
// Where the contents a fields with a `mapstructure:,squash` tag are bubbled up.
func (*PolicyDocument) FlatMapstructure() interface{ HCL2Spec() map[string]hcldec.Spec } {
	return new(FlatPolicyDocument)
}

// HCL2Spec returns the hcl spec of a PolicyDocument.
// This spec is used by HCL to read the fields of PolicyDocument.
// The decoded values from this spec will then be applied to a FlatPolicyDocument.
func (*FlatPolicyDocument) HCL2Spec() map[string]hcldec.Spec {
	s := map[string]hcldec.Spec{
		"version":   &hcldec.AttrSpec{Name: "version", Type: cty.String, Required: false},
		"statement": &hcldec.BlockListSpec{TypeName: "statement", Nested: hcldec.ObjectSpec((*FlatStatement)(nil).HCL2Spec())},
	}
	return s
}

// FlatSecurityGroupFilterOptions is an auto-generated flat version of SecurityGroupFilterOptions.
// Where the contents of a field with a `mapstructure:,squash` tag are bubbled up.
type FlatSecurityGroupFilterOptions struct {
	Filters map[string]string            `cty:"filters" hcl:"filters"`
	Filter  []hcl2template.FlatNameValue `cty:"filter" hcl:"filter"`
}

// FlatMapstructure returns a new FlatSecurityGroupFilterOptions.
// FlatSecurityGroupFilterOptions is an auto-generated flat version of SecurityGroupFilterOptions.
// Where the contents a fields with a `mapstructure:,squash` tag are bubbled up.
func (*SecurityGroupFilterOptions) FlatMapstructure() interface{ HCL2Spec() map[string]hcldec.Spec } {
	return new(FlatSecurityGroupFilterOptions)
}

// HCL2Spec returns the hcl spec of a SecurityGroupFilterOptions.
// This spec is used by HCL to read the fields of SecurityGroupFilterOptions.
// The decoded values from this spec will then be applied to a FlatSecurityGroupFilterOptions.
func (*FlatSecurityGroupFilterOptions) HCL2Spec() map[string]hcldec.Spec {
	s := map[string]hcldec.Spec{
		"filters": &hcldec.AttrSpec{Name: "filters", Type: cty.Map(cty.String), Required: false},
		"filter":  &hcldec.BlockListSpec{TypeName: "filter", Nested: hcldec.ObjectSpec((*hcl2template.FlatNameValue)(nil).HCL2Spec())},
	}
	return s
}

// FlatStatement is an auto-generated flat version of Statement.
// Where the contents of a field with a `mapstructure:,squash` tag are bubbled up.
type FlatStatement struct {
	Effect   *string  `cty:"effect" hcl:"effect"`
	Action   []string `cty:"action" hcl:"action"`
	Resource *string  `cty:"resource" hcl:"resource"`
}

// FlatMapstructure returns a new FlatStatement.
// FlatStatement is an auto-generated flat version of Statement.
// Where the contents a fields with a `mapstructure:,squash` tag are bubbled up.
func (*Statement) FlatMapstructure() interface{ HCL2Spec() map[string]hcldec.Spec } {
	return new(FlatStatement)
}

// HCL2Spec returns the hcl spec of a Statement.
// This spec is used by HCL to read the fields of Statement.
// The decoded values from this spec will then be applied to a FlatStatement.
func (*FlatStatement) HCL2Spec() map[string]hcldec.Spec {
	s := map[string]hcldec.Spec{
		"effect":   &hcldec.AttrSpec{Name: "effect", Type: cty.String, Required: false},
		"action":   &hcldec.AttrSpec{Name: "action", Type: cty.List(cty.String), Required: false},
		"resource": &hcldec.AttrSpec{Name: "resource", Type: cty.String, Required: false},
	}
	return s
}

// FlatSubnetFilterOptions is an auto-generated flat version of SubnetFilterOptions.
// Where the contents of a field with a `mapstructure:,squash` tag are bubbled up.
type FlatSubnetFilterOptions struct {
	Filters  map[string]string            `cty:"filters" hcl:"filters"`
	Filter   []hcl2template.FlatNameValue `cty:"filter" hcl:"filter"`
	MostFree *bool                        `mapstructure:"most_free" cty:"most_free" hcl:"most_free"`
	Random   *bool                        `mapstructure:"random" cty:"random" hcl:"random"`
}

// FlatMapstructure returns a new FlatSubnetFilterOptions.
// FlatSubnetFilterOptions is an auto-generated flat version of SubnetFilterOptions.
// Where the contents a fields with a `mapstructure:,squash` tag are bubbled up.
func (*SubnetFilterOptions) FlatMapstructure() interface{ HCL2Spec() map[string]hcldec.Spec } {
	return new(FlatSubnetFilterOptions)
}

// HCL2Spec returns the hcl spec of a SubnetFilterOptions.
// This spec is used by HCL to read the fields of SubnetFilterOptions.
// The decoded values from this spec will then be applied to a FlatSubnetFilterOptions.
func (*FlatSubnetFilterOptions) HCL2Spec() map[string]hcldec.Spec {
	s := map[string]hcldec.Spec{
		"filters":   &hcldec.AttrSpec{Name: "filters", Type: cty.Map(cty.String), Required: false},
		"filter":    &hcldec.BlockListSpec{TypeName: "filter", Nested: hcldec.ObjectSpec((*hcl2template.FlatNameValue)(nil).HCL2Spec())},
		"most_free": &hcldec.AttrSpec{Name: "most_free", Type: cty.Bool, Required: false},
		"random":    &hcldec.AttrSpec{Name: "random", Type: cty.Bool, Required: false},
	}
	return s
}

// FlatVpcFilterOptions is an auto-generated flat version of VpcFilterOptions.
// Where the contents of a field with a `mapstructure:,squash` tag are bubbled up.
type FlatVpcFilterOptions struct {
	Filters map[string]string            `cty:"filters" hcl:"filters"`
	Filter  []hcl2template.FlatNameValue `cty:"filter" hcl:"filter"`
}

// FlatMapstructure returns a new FlatVpcFilterOptions.
// FlatVpcFilterOptions is an auto-generated flat version of VpcFilterOptions.
// Where the contents a fields with a `mapstructure:,squash` tag are bubbled up.
func (*VpcFilterOptions) FlatMapstructure() interface{ HCL2Spec() map[string]hcldec.Spec } {
	return new(FlatVpcFilterOptions)
}

// HCL2Spec returns the hcl spec of a VpcFilterOptions.
// This spec is used by HCL to read the fields of VpcFilterOptions.
// The decoded values from this spec will then be applied to a FlatVpcFilterOptions.
func (*FlatVpcFilterOptions) HCL2Spec() map[string]hcldec.Spec {
	s := map[string]hcldec.Spec{
		"filters": &hcldec.AttrSpec{Name: "filters", Type: cty.Map(cty.String), Required: false},
		"filter":  &hcldec.BlockListSpec{TypeName: "filter", Nested: hcldec.ObjectSpec((*hcl2template.FlatNameValue)(nil).HCL2Spec())},
	}
	return s
}
<commit_msg>generated code<commit_after>// Code generated by "mapstructure-to-hcl2 -type AmiFilterOptions,SecurityGroupFilterOptions,SubnetFilterOptions,VpcFilterOptions,PolicyDocument,Statement"; DO NOT EDIT.
package common

import (
	"github.com/hashicorp/hcl/v2/hcldec"
	"github.com/hashicorp/packer/hcl2template"
	"github.com/zclconf/go-cty/cty"
)

// FlatAmiFilterOptions is an auto-generated flat version of AmiFilterOptions.
// Where the contents of a field with a `mapstructure:,squash` tag are bubbled up.
type FlatAmiFilterOptions struct {
	Filters    map[string]string           `cty:"filters" hcl:"filters"`
	Filter     []hcl2template.FlatKeyValue `cty:"filter" hcl:"filter"`
	Owners     []string                    `cty:"owners" hcl:"owners"`
	MostRecent *bool                       `mapstructure:"most_recent" cty:"most_recent" hcl:"most_recent"`
}

// FlatMapstructure returns a new FlatAmiFilterOptions.
// FlatAmiFilterOptions is an auto-generated flat version of AmiFilterOptions.
// Where the contents a fields with a `mapstructure:,squash` tag are bubbled up.
func (*AmiFilterOptions) FlatMapstructure() interface{ HCL2Spec() map[string]hcldec.Spec } {
	return new(FlatAmiFilterOptions)
}

// HCL2Spec returns the hcl spec of a AmiFilterOptions.
// This spec is used by HCL to read the fields of AmiFilterOptions.
// The decoded values from this spec will then be applied to a FlatAmiFilterOptions.
func (*FlatAmiFilterOptions) HCL2Spec() map[string]hcldec.Spec {
	s := map[string]hcldec.Spec{
		"filters":     &hcldec.AttrSpec{Name: "filters", Type: cty.Map(cty.String), Required: false},
		"filter":      &hcldec.BlockListSpec{TypeName: "filter", Nested: hcldec.ObjectSpec((*hcl2template.FlatKeyValue)(nil).HCL2Spec())},
		"owners":      &hcldec.AttrSpec{Name: "owners", Type: cty.List(cty.String), Required: false},
		"most_recent": &hcldec.AttrSpec{Name: "most_recent", Type: cty.Bool, Required: false},
	}
	return s
}

// FlatPolicyDocument is an auto-generated flat version of PolicyDocument.
// Where the contents of a field with a `mapstructure:,squash` tag are bubbled up.
type FlatPolicyDocument struct {
	Version   *string         `cty:"version" hcl:"version"`
	Statement []FlatStatement `cty:"statement" hcl:"statement"`
}

// FlatMapstructure returns a new FlatPolicyDocument.
// FlatPolicyDocument is an auto-generated flat version of PolicyDocument.
// Where the contents a fields with a `mapstructure:,squash` tag are bubbled up.
func (*PolicyDocument) FlatMapstructure() interface{ HCL2Spec() map[string]hcldec.Spec } {
	return new(FlatPolicyDocument)
}

// HCL2Spec returns the hcl spec of a PolicyDocument.
// This spec is used by HCL to read the fields of PolicyDocument.
// The decoded values from this spec will then be applied to a FlatPolicyDocument.
func (*FlatPolicyDocument) HCL2Spec() map[string]hcldec.Spec {
	s := map[string]hcldec.Spec{
		"version":   &hcldec.AttrSpec{Name: "version", Type: cty.String, Required: false},
		"statement": &hcldec.BlockListSpec{TypeName: "statement", Nested: hcldec.ObjectSpec((*FlatStatement)(nil).HCL2Spec())},
	}
	return s
}

// FlatSecurityGroupFilterOptions is an auto-generated flat version of SecurityGroupFilterOptions.
// Where the contents of a field with a `mapstructure:,squash` tag are bubbled up.
type FlatSecurityGroupFilterOptions struct {
	Filters map[string]string            `cty:"filters" hcl:"filters"`
	Filter  []hcl2template.FlatNameValue `cty:"filter" hcl:"filter"`
}

// FlatMapstructure returns a new FlatSecurityGroupFilterOptions.
// FlatSecurityGroupFilterOptions is an auto-generated flat version of SecurityGroupFilterOptions.
// Where the contents a fields with a `mapstructure:,squash` tag are bubbled up.
func (*SecurityGroupFilterOptions) FlatMapstructure() interface{ HCL2Spec() map[string]hcldec.Spec } {
	return new(FlatSecurityGroupFilterOptions)
}

// HCL2Spec returns the hcl spec of a SecurityGroupFilterOptions.
// This spec is used by HCL to read the fields of SecurityGroupFilterOptions.
// The decoded values from this spec will then be applied to a FlatSecurityGroupFilterOptions.
func (*FlatSecurityGroupFilterOptions) HCL2Spec() map[string]hcldec.Spec {
	s := map[string]hcldec.Spec{
		"filters": &hcldec.AttrSpec{Name: "filters", Type: cty.Map(cty.String), Required: false},
		"filter":  &hcldec.BlockListSpec{TypeName: "filter", Nested: hcldec.ObjectSpec((*hcl2template.FlatNameValue)(nil).HCL2Spec())},
	}
	return s
}

// FlatStatement is an auto-generated flat version of Statement.
// Where the contents of a field with a `mapstructure:,squash` tag are bubbled up.
type FlatStatement struct {
	Effect   *string  `cty:"effect" hcl:"effect"`
	Action   []string `cty:"action" hcl:"action"`
	Resource []string `cty:"resource" hcl:"resource"`
}

// FlatMapstructure returns a new FlatStatement.
// FlatStatement is an auto-generated flat version of Statement.
// Where the contents a fields with a `mapstructure:,squash` tag are bubbled up.
func (*Statement) FlatMapstructure() interface{ HCL2Spec() map[string]hcldec.Spec } {
	return new(FlatStatement)
}

// HCL2Spec returns the hcl spec of a Statement.
// This spec is used by HCL to read the fields of Statement.
// The decoded values from this spec will then be applied to a FlatStatement.
func (*FlatStatement) HCL2Spec() map[string]hcldec.Spec {
	s := map[string]hcldec.Spec{
		"effect":   &hcldec.AttrSpec{Name: "effect", Type: cty.String, Required: false},
		"action":   &hcldec.AttrSpec{Name: "action", Type: cty.List(cty.String), Required: false},
		"resource": &hcldec.AttrSpec{Name: "resource", Type: cty.List(cty.String), Required: false},
	}
	return s
}

// FlatSubnetFilterOptions is an auto-generated flat version of SubnetFilterOptions.
// Where the contents of a field with a `mapstructure:,squash` tag are bubbled up.
type FlatSubnetFilterOptions struct {
	Filters  map[string]string            `cty:"filters" hcl:"filters"`
	Filter   []hcl2template.FlatNameValue `cty:"filter" hcl:"filter"`
	MostFree *bool                        `mapstructure:"most_free" cty:"most_free" hcl:"most_free"`
	Random   *bool                        `mapstructure:"random" cty:"random" hcl:"random"`
}

// FlatMapstructure returns a new FlatSubnetFilterOptions.
// FlatSubnetFilterOptions is an auto-generated flat version of SubnetFilterOptions.
// Where the contents a fields with a `mapstructure:,squash` tag are bubbled up.
func (*SubnetFilterOptions) FlatMapstructure() interface{ HCL2Spec() map[string]hcldec.Spec } {
	return new(FlatSubnetFilterOptions)
}

// HCL2Spec returns the hcl spec of a SubnetFilterOptions.
// This spec is used by HCL to read the fields of SubnetFilterOptions.
// The decoded values from this spec will then be applied to a FlatSubnetFilterOptions.
func (*FlatSubnetFilterOptions) HCL2Spec() map[string]hcldec.Spec {
	s := map[string]hcldec.Spec{
		"filters":   &hcldec.AttrSpec{Name: "filters", Type: cty.Map(cty.String), Required: false},
		"filter":    &hcldec.BlockListSpec{TypeName: "filter", Nested: hcldec.ObjectSpec((*hcl2template.FlatNameValue)(nil).HCL2Spec())},
		"most_free": &hcldec.AttrSpec{Name: "most_free", Type: cty.Bool, Required: false},
		"random":    &hcldec.AttrSpec{Name: "random", Type: cty.Bool, Required: false},
	}
	return s
}

// FlatVpcFilterOptions is an auto-generated flat version of VpcFilterOptions.
// Where the contents of a field with a `mapstructure:,squash` tag are bubbled up.
type FlatVpcFilterOptions struct {
	Filters map[string]string            `cty:"filters" hcl:"filters"`
	Filter  []hcl2template.FlatNameValue `cty:"filter" hcl:"filter"`
}

// FlatMapstructure returns a new FlatVpcFilterOptions.
// FlatVpcFilterOptions is an auto-generated flat version of VpcFilterOptions.
// Where the contents a fields with a `mapstructure:,squash` tag are bubbled up.
func (*VpcFilterOptions) FlatMapstructure() interface{ HCL2Spec() map[string]hcldec.Spec } {
	return new(FlatVpcFilterOptions)
}

// HCL2Spec returns the hcl spec of a VpcFilterOptions.
// This spec is used by HCL to read the fields of VpcFilterOptions.
// The decoded values from this spec will then be applied to a FlatVpcFilterOptions.
func (*FlatVpcFilterOptions) HCL2Spec() map[string]hcldec.Spec {
	s := map[string]hcldec.Spec{
		"filters": &hcldec.AttrSpec{Name: "filters", Type: cty.Map(cty.String), Required: false},
		"filter":  &hcldec.BlockListSpec{TypeName: "filter", Nested: hcldec.ObjectSpec((*hcl2template.FlatNameValue)(nil).HCL2Spec())},
	}
	return s
}
<|endoftext|>