<commit_before>// Copyright (c) 2017 The btcsuite developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package indexers

import (
	"errors"

	"github.com/btcsuite/btcd/blockchain"
	"github.com/btcsuite/btcd/chaincfg"
	"github.com/btcsuite/btcd/chaincfg/chainhash"
	"github.com/btcsuite/btcd/database"
	"github.com/btcsuite/btcutil"
	"github.com/btcsuite/btcutil/gcs"
	"github.com/btcsuite/btcutil/gcs/builder"
	"github.com/btcsuite/fastsha256"
)

const (
	// cfIndexName is the human-readable name for the index.
	cfIndexName = "committed filter index"
)

// Committed filters come in two flavours: basic and extended. They are
// generated and dropped in pairs, and both are indexed by a block's hash.
// Besides holding different content, they also live in different buckets.
var (
	// cfIndexParentBucketKey is the name of the parent bucket used to house
	// the index. The rest of the buckets live below this bucket.
	cfIndexParentBucketKey = []byte("cfindexparentbucket")
	// cfBasicIndexKey is the name of the db bucket used to house the
	// block hash -> basic cf index (cf#0).
	cfBasicIndexKey = []byte("cf0byhashidx")
	// cfBasicHeaderKey is the name of the db bucket used to house the
	// block hash -> basic cf header index (cf#0).
	cfBasicHeaderKey = []byte("cf0headerbyhashidx")
	// cfExtendedIndexKey is the name of the db bucket used to house the
	// block hash -> extended cf index (cf#1).
	cfExtendedIndexKey = []byte("cf1byhashidx")
	// cfExtendedHeaderKey is the name of the db bucket used to house the
	// block hash -> extended cf header index (cf#1).
	cfExtendedHeaderKey = []byte("cf1headerbyhashidx")
)

// dbFetchFilter retrieves a block's basic or extended filter. A filter's
// absence is not considered an error.
func dbFetchFilter(dbTx database.Tx, key []byte, h *chainhash.Hash) ([]byte, error) {
	idx := dbTx.Metadata().Bucket(cfIndexParentBucketKey).Bucket(key)
	return idx.Get(h[:]), nil
}

// dbFetchFilterHeader retrieves a block's basic or extended filter header.
// A filter's absence is not considered an error.
func dbFetchFilterHeader(dbTx database.Tx, key []byte, h *chainhash.Hash) ([]byte, error) {
	idx := dbTx.Metadata().Bucket(cfIndexParentBucketKey).Bucket(key)
	fh := idx.Get(h[:])
	if len(fh) != fastsha256.Size {
		return nil, errors.New("invalid filter header length")
	}
	return fh, nil
}

// dbStoreFilter stores a block's basic or extended filter.
func dbStoreFilter(dbTx database.Tx, key []byte, h *chainhash.Hash, f []byte) error {
	idx := dbTx.Metadata().Bucket(cfIndexParentBucketKey).Bucket(key)
	return idx.Put(h[:], f)
}

// dbStoreFilterHeader stores a block's basic or extended filter header.
func dbStoreFilterHeader(dbTx database.Tx, key []byte, h *chainhash.Hash, fh []byte) error {
	if len(fh) != fastsha256.Size {
		return errors.New("invalid filter header length")
	}
	idx := dbTx.Metadata().Bucket(cfIndexParentBucketKey).Bucket(key)
	return idx.Put(h[:], fh)
}

// dbDeleteFilter deletes a filter's basic or extended filter.
func dbDeleteFilter(dbTx database.Tx, key []byte, h *chainhash.Hash) error {
	idx := dbTx.Metadata().Bucket(cfIndexParentBucketKey).Bucket(key)
	return idx.Delete(h[:])
}

// dbDeleteFilterHeader deletes a filter's basic or extended filter header.
func dbDeleteFilterHeader(dbTx database.Tx, key []byte, h *chainhash.Hash) error {
	idx := dbTx.Metadata().Bucket(cfIndexParentBucketKey).Bucket(key)
	return idx.Delete(h[:])
}

// CfIndex implements a committed filter (cf) by hash index.
type CfIndex struct {
	db          database.DB
	chainParams *chaincfg.Params
}

// Ensure the CfIndex type implements the Indexer interface.
var _ Indexer = (*CfIndex)(nil)

// Init initializes the hash-based cf index. This is part of the Indexer
// interface.
func (idx *CfIndex) Init() error {
	return nil // Nothing to do.
}

// Key returns the database key to use for the index as a byte slice. This is
// part of the Indexer interface.
func (idx *CfIndex) Key() []byte {
	return cfIndexParentBucketKey
}

// Name returns the human-readable name of the index. This is part of the
// Indexer interface.
func (idx *CfIndex) Name() string {
	return cfIndexName
}

// Create is invoked when the indexer manager determines the index needs to
// be created for the first time. It creates buckets for the two hash-based cf
// indexes (simple, extended).
func (idx *CfIndex) Create(dbTx database.Tx) error {
	meta := dbTx.Metadata()
	cfIndexParentBucket, err := meta.CreateBucket(cfIndexParentBucketKey)
	if err != nil {
		return err
	}
	_, err = cfIndexParentBucket.CreateBucket(cfBasicIndexKey)
	if err != nil {
		return err
	}
	_, err = cfIndexParentBucket.CreateBucket(cfBasicHeaderKey)
	if err != nil {
		return err
	}
	_, err = cfIndexParentBucket.CreateBucket(cfExtendedIndexKey)
	if err != nil {
		return err
	}
	_, err = cfIndexParentBucket.CreateBucket(cfExtendedHeaderKey)
	if err != nil {
		return err
	}
	firstHeader := make([]byte, chainhash.HashSize)
	err = dbStoreFilterHeader(
		dbTx,
		cfBasicHeaderKey,
		&idx.chainParams.GenesisBlock.Header.PrevBlock,
		firstHeader,
	)
	if err != nil {
		return err
	}
	err = dbStoreFilterHeader(
		dbTx,
		cfExtendedHeaderKey,
		&idx.chainParams.GenesisBlock.Header.PrevBlock,
		firstHeader,
	)
	return err
}

// storeFilter stores a given filter, and performs the steps needed to
// generate the filter's header.
func storeFilter(dbTx database.Tx, block *btcutil.Block, f *gcs.Filter,
	extended bool) error {
	// Figure out which buckets to use.
	fkey := cfBasicIndexKey
	hkey := cfBasicHeaderKey
	if extended {
		fkey = cfExtendedIndexKey
		hkey = cfExtendedHeaderKey
	}
	// Start by storing the filter.
	h := block.Hash()
	err := dbStoreFilter(dbTx, fkey, h, f.NBytes())
	if err != nil {
		return err
	}
	// Then fetch the previous block's filter header.
	ph := &block.MsgBlock().Header.PrevBlock
	pfh, err := dbFetchFilterHeader(dbTx, hkey, ph)
	if err != nil {
		return err
	}
	// Construct the new block's filter header, and store it.
	prevHeader, err := chainhash.NewHash(pfh)
	if err != nil {
		return err
	}
	fh := builder.MakeHeaderForFilter(f, *prevHeader)
	return dbStoreFilterHeader(dbTx, hkey, h, fh[:])
}

// ConnectBlock is invoked by the index manager when a new block has been
// connected to the main chain. This indexer adds a hash-to-cf mapping for
// every passed block. This is part of the Indexer interface.
func (idx *CfIndex) ConnectBlock(dbTx database.Tx, block *btcutil.Block,
	view *blockchain.UtxoViewpoint) error {
	f, err := builder.BuildBasicFilter(block.MsgBlock())
	if err != nil {
		return err
	}
	err = storeFilter(dbTx, block, f, false)
	if err != nil {
		return err
	}
	f, err = builder.BuildExtFilter(block.MsgBlock())
	if err != nil {
		return err
	}
	return storeFilter(dbTx, block, f, true)
}

// DisconnectBlock is invoked by the index manager when a block has been
// disconnected from the main chain.  This indexer removes the hash-to-cf
// mapping for every passed block. This is part of the Indexer interface.
func (idx *CfIndex) DisconnectBlock(dbTx database.Tx, block *btcutil.Block,
	view *blockchain.UtxoViewpoint) error {
	err := dbDeleteFilter(dbTx, cfBasicIndexKey, block.Hash())
	if err != nil {
		return err
	}
	return dbDeleteFilter(dbTx, cfExtendedIndexKey, block.Hash())
}

// FilterByBlockHash returns the serialized contents of a block's basic or
// extended committed filter.
func (idx *CfIndex) FilterByBlockHash(h *chainhash.Hash, extended bool) ([]byte, error) {
	var f []byte
	err := idx.db.View(func(dbTx database.Tx) error {
		var err error
		key := cfBasicIndexKey
		if extended {
			key = cfExtendedIndexKey
		}
		f, err = dbFetchFilter(dbTx, key, h)
		return err
	})
	return f, err
}

// FilterHeaderByBlockHash returns the serialized contents of a block's basic
// or extended committed filter header.
func (idx *CfIndex) FilterHeaderByBlockHash(h *chainhash.Hash, extended bool) ([]byte, error) {
	var fh []byte
	err := idx.db.View(func(dbTx database.Tx) error {
		var err error
		key := cfBasicHeaderKey
		if extended {
			key = cfExtendedHeaderKey
		}
		fh, err = dbFetchFilterHeader(dbTx, key, h)
		return err
	})
	return fh, err
}

// NewCfIndex returns a new instance of an indexer that is used to create a
// mapping of the hashes of all blocks in the blockchain to their respective
// committed filters.
//
// It implements the Indexer interface which plugs into the IndexManager that in
// turn is used by the blockchain package. This allows the index to be
// seamlessly maintained along with the chain.
func NewCfIndex(db database.DB, chainParams *chaincfg.Params) *CfIndex {
	return &CfIndex{db: db, chainParams: chainParams}
}

// DropCfIndex drops the CF index from the provided database if exists.
func DropCfIndex(db database.DB) error {
	return dropIndex(db, cfIndexParentBucketKey, cfIndexName)
}
<commit_msg>blockchain/indexers: add a bit more line spacing to cfindex.go<commit_after>// Copyright (c) 2017 The btcsuite developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package indexers

import (
	"errors"

	"github.com/btcsuite/btcd/blockchain"
	"github.com/btcsuite/btcd/chaincfg"
	"github.com/btcsuite/btcd/chaincfg/chainhash"
	"github.com/btcsuite/btcd/database"
	"github.com/btcsuite/btcutil"
	"github.com/btcsuite/btcutil/gcs"
	"github.com/btcsuite/btcutil/gcs/builder"
	"github.com/btcsuite/fastsha256"
)

const (
	// cfIndexName is the human-readable name for the index.
	cfIndexName = "committed filter index"
)

// Committed filters come in two flavours: basic and extended. They are
// generated and dropped in pairs, and both are indexed by a block's hash.
// Besides holding different content, they also live in different buckets.
var (
	// cfIndexParentBucketKey is the name of the parent bucket used to house
	// the index. The rest of the buckets live below this bucket.
	cfIndexParentBucketKey = []byte("cfindexparentbucket")

	// cfBasicIndexKey is the name of the db bucket used to house the
	// block hash -> basic cf index (cf#0).
	cfBasicIndexKey = []byte("cf0byhashidx")

	// cfBasicHeaderKey is the name of the db bucket used to house the
	// block hash -> basic cf header index (cf#0).
	cfBasicHeaderKey = []byte("cf0headerbyhashidx")

	// cfExtendedIndexKey is the name of the db bucket used to house the
	// block hash -> extended cf index (cf#1).
	cfExtendedIndexKey = []byte("cf1byhashidx")

	// cfExtendedHeaderKey is the name of the db bucket used to house the
	// block hash -> extended cf header index (cf#1).
	cfExtendedHeaderKey = []byte("cf1headerbyhashidx")
)

// dbFetchFilter retrieves a block's basic or extended filter. A filter's
// absence is not considered an error.
func dbFetchFilter(dbTx database.Tx, key []byte, h *chainhash.Hash) ([]byte, error) {
	idx := dbTx.Metadata().Bucket(cfIndexParentBucketKey).Bucket(key)
	return idx.Get(h[:]), nil
}

// dbFetchFilterHeader retrieves a block's basic or extended filter header.
// A filter's absence is not considered an error.
func dbFetchFilterHeader(dbTx database.Tx, key []byte, h *chainhash.Hash) ([]byte, error) {
	idx := dbTx.Metadata().Bucket(cfIndexParentBucketKey).Bucket(key)

	fh := idx.Get(h[:])
	if len(fh) != fastsha256.Size {
		return nil, errors.New("invalid filter header length")
	}

	return fh, nil
}

// dbStoreFilter stores a block's basic or extended filter.
func dbStoreFilter(dbTx database.Tx, key []byte, h *chainhash.Hash, f []byte) error {
	idx := dbTx.Metadata().Bucket(cfIndexParentBucketKey).Bucket(key)
	return idx.Put(h[:], f)
}

// dbStoreFilterHeader stores a block's basic or extended filter header.
func dbStoreFilterHeader(dbTx database.Tx, key []byte, h *chainhash.Hash, fh []byte) error {
	if len(fh) != fastsha256.Size {
		return errors.New("invalid filter header length")
	}
	idx := dbTx.Metadata().Bucket(cfIndexParentBucketKey).Bucket(key)
	return idx.Put(h[:], fh)
}

// dbDeleteFilter deletes a filter's basic or extended filter.
func dbDeleteFilter(dbTx database.Tx, key []byte, h *chainhash.Hash) error {
	idx := dbTx.Metadata().Bucket(cfIndexParentBucketKey).Bucket(key)
	return idx.Delete(h[:])
}

// dbDeleteFilterHeader deletes a filter's basic or extended filter header.
func dbDeleteFilterHeader(dbTx database.Tx, key []byte, h *chainhash.Hash) error {
	idx := dbTx.Metadata().Bucket(cfIndexParentBucketKey).Bucket(key)
	return idx.Delete(h[:])
}

// CfIndex implements a committed filter (cf) by hash index.
type CfIndex struct {
	db          database.DB
	chainParams *chaincfg.Params
}

// Ensure the CfIndex type implements the Indexer interface.
var _ Indexer = (*CfIndex)(nil)

// Init initializes the hash-based cf index. This is part of the Indexer
// interface.
func (idx *CfIndex) Init() error {
	return nil // Nothing to do.
}

// Key returns the database key to use for the index as a byte slice. This is
// part of the Indexer interface.
func (idx *CfIndex) Key() []byte {
	return cfIndexParentBucketKey
}

// Name returns the human-readable name of the index. This is part of the
// Indexer interface.
func (idx *CfIndex) Name() string {
	return cfIndexName
}

// Create is invoked when the indexer manager determines the index needs to
// be created for the first time. It creates buckets for the two hash-based cf
// indexes (simple, extended).
func (idx *CfIndex) Create(dbTx database.Tx) error {
	meta := dbTx.Metadata()

	cfIndexParentBucket, err := meta.CreateBucket(cfIndexParentBucketKey)
	if err != nil {
		return err
	}
	_, err = cfIndexParentBucket.CreateBucket(cfBasicIndexKey)
	if err != nil {
		return err
	}
	_, err = cfIndexParentBucket.CreateBucket(cfBasicHeaderKey)
	if err != nil {
		return err
	}
	_, err = cfIndexParentBucket.CreateBucket(cfExtendedIndexKey)
	if err != nil {
		return err
	}
	_, err = cfIndexParentBucket.CreateBucket(cfExtendedHeaderKey)
	if err != nil {
		return err
	}

	firstHeader := make([]byte, chainhash.HashSize)
	err = dbStoreFilterHeader(
		dbTx,
		cfBasicHeaderKey,
		&idx.chainParams.GenesisBlock.Header.PrevBlock,
		firstHeader,
	)
	if err != nil {
		return err
	}

	return dbStoreFilterHeader(
		dbTx,
		cfExtendedHeaderKey,
		&idx.chainParams.GenesisBlock.Header.PrevBlock,
		firstHeader,
	)
}

// storeFilter stores a given filter, and performs the steps needed to
// generate the filter's header.
func storeFilter(dbTx database.Tx, block *btcutil.Block, f *gcs.Filter,
	extended bool) error {

	// Figure out which buckets to use.
	fkey := cfBasicIndexKey
	hkey := cfBasicHeaderKey
	if extended {
		fkey = cfExtendedIndexKey
		hkey = cfExtendedHeaderKey
	}

	// Start by storing the filter.
	h := block.Hash()
	err := dbStoreFilter(dbTx, fkey, h, f.NBytes())
	if err != nil {
		return err
	}

	// Then fetch the previous block's filter header.
	ph := &block.MsgBlock().Header.PrevBlock
	pfh, err := dbFetchFilterHeader(dbTx, hkey, ph)
	if err != nil {
		return err
	}

	// Construct the new block's filter header, and store it.
	prevHeader, err := chainhash.NewHash(pfh)
	if err != nil {
		return err
	}
	fh := builder.MakeHeaderForFilter(f, *prevHeader)
	return dbStoreFilterHeader(dbTx, hkey, h, fh[:])
}

// ConnectBlock is invoked by the index manager when a new block has been
// connected to the main chain. This indexer adds a hash-to-cf mapping for
// every passed block. This is part of the Indexer interface.
func (idx *CfIndex) ConnectBlock(dbTx database.Tx, block *btcutil.Block,
	view *blockchain.UtxoViewpoint) error {

	f, err := builder.BuildBasicFilter(block.MsgBlock())
	if err != nil {
		return err
	}

	if err := storeFilter(dbTx, block, f, false); err != nil {
		return err
	}

	f, err = builder.BuildExtFilter(block.MsgBlock())
	if err != nil {
		return err
	}

	return storeFilter(dbTx, block, f, true)
}

// DisconnectBlock is invoked by the index manager when a block has been
// disconnected from the main chain.  This indexer removes the hash-to-cf
// mapping for every passed block. This is part of the Indexer interface.
func (idx *CfIndex) DisconnectBlock(dbTx database.Tx, block *btcutil.Block,
	view *blockchain.UtxoViewpoint) error {

	err := dbDeleteFilter(dbTx, cfBasicIndexKey, block.Hash())
	if err != nil {
		return err
	}

	return dbDeleteFilter(dbTx, cfExtendedIndexKey, block.Hash())
}

// FilterByBlockHash returns the serialized contents of a block's basic or
// extended committed filter.
func (idx *CfIndex) FilterByBlockHash(h *chainhash.Hash, extended bool) ([]byte, error) {
	var f []byte
	err := idx.db.View(func(dbTx database.Tx) error {
		var err error
		key := cfBasicIndexKey
		if extended {
			key = cfExtendedIndexKey
		}

		f, err = dbFetchFilter(dbTx, key, h)
		return err
	})
	return f, err
}

// FilterHeaderByBlockHash returns the serialized contents of a block's basic
// or extended committed filter header.
func (idx *CfIndex) FilterHeaderByBlockHash(h *chainhash.Hash, extended bool) ([]byte, error) {
	var fh []byte
	err := idx.db.View(func(dbTx database.Tx) error {
		var err error
		key := cfBasicHeaderKey
		if extended {
			key = cfExtendedHeaderKey
		}

		fh, err = dbFetchFilterHeader(dbTx, key, h)
		return err
	})
	return fh, err
}

// NewCfIndex returns a new instance of an indexer that is used to create a
// mapping of the hashes of all blocks in the blockchain to their respective
// committed filters.
//
// It implements the Indexer interface which plugs into the IndexManager that
// in turn is used by the blockchain package. This allows the index to be
// seamlessly maintained along with the chain.
func NewCfIndex(db database.DB, chainParams *chaincfg.Params) *CfIndex {
	return &CfIndex{db: db, chainParams: chainParams}
}

// DropCfIndex drops the CF index from the provided database if exists.
func DropCfIndex(db database.DB) error {
	return dropIndex(db, cfIndexParentBucketKey, cfIndexName)
}
<|endoftext|>


<commit_before>//----------------------------------------
//
// Copyright © ying32. All Rights Reserved.
//
// Licensed under Apache License 2.0
//
//----------------------------------------

/*
   这里主要是窗口资源用来反射关联事件，就可以简化相关代码。

   事件名命名的规则：
     On + 组件名 + 事件，TForm特殊性，固定为名称为Form。
   例如窗口建完后：
     func (f *TForm1) OnFormCreate(sender vcl.IObject)
   又如按钮：
     func (f *TForm1) OnButton1Click(sender vcl.IObject)

   原理：
     首次会收集Form以On开头的事件，然后根据 组件名称提取出事件的类型，再通过事件类型查找某个组件中的 SetOn + eventType方法。

   多个组件共享同一个事件：

   type TMainForm struct {
       *vcl.TForm
       Button1 *vcl.TButton
       Button2 *vcl.TButton `events:"OnButton1Click"`
       Button3 *vcl.TButton `events:"OnButton1Click,OnButton1Resize"`
   }

   // 这样只自动关联了Button1的事件，但此时我想将此事件关联到Button2, Button3上
   // 常规的做法就是 Button2.SetOnClick(f.OnButton1Click)
   // 现在提供一种新的方式，这种方式应对于res2go转换后不自动共享事件问题。

   func (f *TMainForm) OnButton1Click(sender vcl.IObject) {

   }

*/

package vcl

import (
	"fmt"
	"reflect"
	"strings"

	"github.com/ying32/govcl/vcl/api"
)

type eventMethod struct {
	Method  reflect.Value
	FuncPtr uintptr
}

// autoBindEvents 自动关联事件。
func autoBindEvents(vForm reflect.Value, root IComponent, subComponentsEvent, afterBindSubComponentsEvents bool) {
	defer func() {
		if err := recover(); err != nil {
			fmt.Println("Calling autoBindEvents exception:", err)
		}
	}()

	// OnFormCreate or OnFrameCreate
	var doCreate reflect.Value

	vt := vForm.Type()

	// 提取所有符合规则的事件
	eventMethods := make(map[string]eventMethod, 0)
	// 遍历当前结构的方法
	for i := 0; i < vt.NumMethod(); i++ {
		m := vt.Method(i)
		// 保存窗口创建事件
		if m.Name == "OnFormCreate" || m.Name == "OnFrameCreate" {
			doCreate = vForm.Method(i)
			continue
		}
		if strings.HasPrefix(m.Name, "On") {
			eventMethods[m.Name] = eventMethod{Method: vForm.Method(i), FuncPtr: m.Func.Pointer()}
		}
	}

	type eventItem struct {
		Type   string
		Method eventMethod
	}

	// 临时方法表
	tempEventTypes := make(map[string]eventItem, 0)

	// 遍历结构中的字段
	//for i := 0; i < vt.Elem().NumField(); i++ {
	//	field := vt.Elem().Field(i)
	//	fmt.Println("field.Name:", field.Name)
	//
	//}

	// 用于之后显示提示的
	formName := root.Name()

	// 设置事件
	setEvent := func(component IComponent) {
		name1 := component.Name()
		name2 := name1
		if component.Equals(root) {
			name1 = "Form"
			name2 = "TForm"
			if root.ClassName() == "TFrame" {
				name1 = "Frame"
				name2 = "TFrame"
			}
		} else if component.Equals(Application) {
			name1 = "Application"
			name2 = name1
		}
		// 前缀 On + 组件名
		prefix := "On" + name1

		for mName, method := range eventMethods {
			if !strings.HasPrefix(mName, prefix) {
				continue
			}
			eventType := mName[len(prefix):]
			// 将事件名与事件类型对应，之后会用到的
			tempEventTypes[mName] = eventItem{eventType, method}

			if component.Equals(Application) {
				addApplicationNotifyEvent(eventType, method)
			} else {
				addComponentNotifyEvent(vForm, name2, method, eventType, formName)
			}
		}
	}

	// 设置Root事件
	setEvent(root)

	// 子组件事件
	bindSubComponentsEvents := func() {
		var i int32
		for i = 0; i < root.ComponentCount(); i++ {
			setEvent(root.Components(i))
		}

		// 提取字段中的事件关联
		for i := 0; i < vt.Elem().NumField(); i++ {
			field := vt.Elem().Field(i)
			eventsTag := field.Tag.Get("events")
			if eventsTag == "" {
				continue
			}
			eventArr := strings.Split(eventsTag, ",")
			for _, event := range eventArr {
				event = strings.TrimSpace(event)
				item, ok := tempEventTypes[event]
				if !ok {
					continue
				}
				if vCtl := vForm.Elem().Field(i); vCtl.IsValid() {
					findAndSetEvent(vCtl, field.Name, item.Type, item.Method, formName)
				}
			}
		}
	}

	// 设置子组件事件
	if subComponentsEvent {
		bindSubComponentsEvents()
	}

	// 设置Application事件
	setEvent(Application)

	// 最后调用OnCreate
	callEvent(doCreate, []reflect.Value{vForm})

	// 设定了之后绑定子组件事件并且之前没有指定要绑定子组件事件
	if afterBindSubComponentsEvents && !subComponentsEvent {
		// 因为手动创建的组件没有名称，所以这里设置下，名称在当前TForm必须是唯一的
		for i := 0; i < vt.Elem().NumField(); i++ {
			field := vt.Elem().Field(i)
			if field.Type.Kind() != reflect.Ptr || field.Anonymous ||
				!strings.Contains(field.Type.String(), ".T") {
				//!strings.HasPrefix(field.Type.String(), "*vcl.") {
				continue
			}
			if vCtl := vForm.Elem().Field(i); vCtl.IsValid() {
				findAndSetComponentName(vCtl, field.Name)
			}
		}
		bindSubComponentsEvents()
	}
}

// callEvent 调用事件。
func callEvent(event reflect.Value, params []reflect.Value) {
	defer func() {
		if err := recover(); err != nil {
			fmt.Println("Calling callEvent exception:", err)
		}
	}()
	if !event.IsValid() {
		return
	}
	event.Call(params)
}

// findAndSetEvent 公用的call SetOnXXXX方法
func findAndSetEvent(v reflect.Value, name, eventType string, method eventMethod, rootName string) {
	defer func() {
		if err := recover(); err != nil {
			fmt.Println("Calling findAndSetEvent exception:", err, ", eventType:", eventType, ", rootName:", rootName)
		}
	}()
	if event := v.MethodByName("SetOn" + eventType); event.IsValid() {
		// 设置EventId
		api.BeginAddEvent()
		defer api.EndAddEvent()
		api.SetCurrentEventId(method.FuncPtr)

		event.Call([]reflect.Value{method.Method})
	} else {
		if len(eventType) > 0 {
			// 也许分析错误，所不打印错误消息。
			s := eventType[0]
			switch {
			case s >= '0' && s <= '9' || s == '_':
				return
			}
		}
		fmt.Printf("%s.%s does not support the %s event.\n", rootName, name, eventType)
	}
}

// findAndSetComponentName 查找并设置组件名称
func findAndSetComponentName(v reflect.Value, name string) {
	defer func() {
		if err := recover(); err != nil {
			fmt.Println("Calling findAndSetComponentName exception:", err)
		}
	}()
	if v.Pointer() == 0 {
		return
	}
	if setName := v.MethodByName("SetName"); setName.IsValid() {
		setName.Call([]reflect.Value{reflect.ValueOf(name)})
	}
}

// addComponentNotifyEvent
func addComponentNotifyEvent(vForm reflect.Value, compName string, method eventMethod, eventType, rootName string) {
	if vCtl := vForm.Elem().FieldByName(compName); vCtl.IsValid() {
		findAndSetEvent(vCtl, compName, eventType, method, rootName)
	}
}

// addApplicationNotifyEvent
// 添加Application的关联事件，在一个程序内，application中的事件只有最后一次设置的才会生效。
// 因为Application是单例存在，推荐在主窗口内处理就行了。
func addApplicationNotifyEvent(eventType string, method eventMethod) {
	if app := reflect.ValueOf(Application); app.IsValid() {
		findAndSetEvent(app, "Application", eventType, method, "Application")
	}
}
<commit_msg>Auto Bind Event: Filter fields that do not start with "A-Z".<commit_after>//----------------------------------------
//
// Copyright © ying32. All Rights Reserved.
//
// Licensed under Apache License 2.0
//
//----------------------------------------

/*
   这里主要是窗口资源用来反射关联事件，就可以简化相关代码。

   事件名命名的规则：
     On + 组件名 + 事件，TForm特殊性，固定为名称为Form。
   例如窗口建完后：
     func (f *TForm1) OnFormCreate(sender vcl.IObject)
   又如按钮：
     func (f *TForm1) OnButton1Click(sender vcl.IObject)

   原理：
     首次会收集Form以On开头的事件，然后根据 组件名称提取出事件的类型，再通过事件类型查找某个组件中的 SetOn + eventType方法。

   多个组件共享同一个事件：

   type TMainForm struct {
       *vcl.TForm
       Button1 *vcl.TButton
       Button2 *vcl.TButton `events:"OnButton1Click"`
       Button3 *vcl.TButton `events:"OnButton1Click,OnButton1Resize"`
   }

   // 这样只自动关联了Button1的事件，但此时我想将此事件关联到Button2, Button3上
   // 常规的做法就是 Button2.SetOnClick(f.OnButton1Click)
   // 现在提供一种新的方式，这种方式应对于res2go转换后不自动共享事件问题。

   func (f *TMainForm) OnButton1Click(sender vcl.IObject) {

   }

*/

package vcl

import (
	"fmt"
	"reflect"
	"strings"

	"github.com/ying32/govcl/vcl/api"
)

type eventMethod struct {
	Method  reflect.Value
	FuncPtr uintptr
}

// autoBindEvents 自动关联事件。
func autoBindEvents(vForm reflect.Value, root IComponent, subComponentsEvent, afterBindSubComponentsEvents bool) {
	defer func() {
		if err := recover(); err != nil {
			fmt.Println("Calling autoBindEvents exception:", err)
		}
	}()

	// OnFormCreate or OnFrameCreate
	var doCreate reflect.Value

	vt := vForm.Type()

	// 提取所有符合规则的事件
	eventMethods := make(map[string]eventMethod, 0)
	// 遍历当前结构的方法
	for i := 0; i < vt.NumMethod(); i++ {
		m := vt.Method(i)
		// 保存窗口创建事件
		if m.Name == "OnFormCreate" || m.Name == "OnFrameCreate" {
			doCreate = vForm.Method(i)
			continue
		}
		if strings.HasPrefix(m.Name, "On") {
			eventMethods[m.Name] = eventMethod{Method: vForm.Method(i), FuncPtr: m.Func.Pointer()}
		}
	}

	type eventItem struct {
		Type   string
		Method eventMethod
	}

	// 临时方法表
	tempEventTypes := make(map[string]eventItem, 0)

	// 遍历结构中的字段
	//for i := 0; i < vt.Elem().NumField(); i++ {
	//	field := vt.Elem().Field(i)
	//	fmt.Println("field.Name:", field.Name)
	//
	//}

	// 用于之后显示提示的
	formName := root.Name()

	// 设置事件
	setEvent := func(component IComponent) {
		name1 := component.Name()
		name2 := name1
		if component.Equals(root) {
			name1 = "Form"
			name2 = "TForm"
			if root.ClassName() == "TFrame" {
				name1 = "Frame"
				name2 = "TFrame"
			}
		} else if component.Equals(Application) {
			name1 = "Application"
			name2 = name1
		}
		// 前缀 On + 组件名
		prefix := "On" + name1

		for mName, method := range eventMethods {
			if !strings.HasPrefix(mName, prefix) {
				continue
			}
			eventType := mName[len(prefix):]
			// 将事件名与事件类型对应，之后会用到的
			tempEventTypes[mName] = eventItem{eventType, method}

			if component.Equals(Application) {
				addApplicationNotifyEvent(eventType, method)
			} else {
				addComponentNotifyEvent(vForm, name2, method, eventType, formName)
			}
		}
	}

	// 设置Root事件
	setEvent(root)

	// 子组件事件
	bindSubComponentsEvents := func() {
		var i int32
		for i = 0; i < root.ComponentCount(); i++ {
			setEvent(root.Components(i))
		}

		// 提取字段中的事件关联
		for i := 0; i < vt.Elem().NumField(); i++ {
			field := vt.Elem().Field(i)
			eventsTag := field.Tag.Get("events")
			if eventsTag == "" {
				continue
			}
			eventArr := strings.Split(eventsTag, ",")
			for _, event := range eventArr {
				event = strings.TrimSpace(event)
				item, ok := tempEventTypes[event]
				if !ok {
					continue
				}
				if vCtl := vForm.Elem().Field(i); vCtl.IsValid() {
					findAndSetEvent(vCtl, field.Name, item.Type, item.Method, formName)
				}
			}
		}
	}

	// 设置子组件事件
	if subComponentsEvent {
		bindSubComponentsEvents()
	}

	// 设置Application事件
	setEvent(Application)

	// 最后调用OnCreate
	callEvent(doCreate, []reflect.Value{vForm})

	// 设定了之后绑定子组件事件并且之前没有指定要绑定子组件事件
	if afterBindSubComponentsEvents && !subComponentsEvent {
		// 因为手动创建的组件没有名称，所以这里设置下，名称在当前TForm必须是唯一的
		for i := 0; i < vt.Elem().NumField(); i++ {
			field := vt.Elem().Field(i)
			if field.Type.Kind() != reflect.Ptr || field.Anonymous ||
				!strings.Contains(field.Type.String(), ".T") {
				continue
			}
			// 检测首字母是否大写
			if len(field.Name) >= 1 {
				// 首字母不为A-Z之间的则排除。
				if c := field.Name[0]; !(c >= 'A' && c <= 'Z') {
					continue
				}
			}
			if vCtl := vForm.Elem().Field(i); vCtl.IsValid() {
				findAndSetComponentName(vCtl, field.Name)
			}
		}
		bindSubComponentsEvents()
	}
}

// callEvent 调用事件。
func callEvent(event reflect.Value, params []reflect.Value) {
	defer func() {
		if err := recover(); err != nil {
			fmt.Println("Calling callEvent exception:", err)
		}
	}()
	if !event.IsValid() {
		return
	}
	event.Call(params)
}

// findAndSetEvent 公用的call SetOnXXXX方法
func findAndSetEvent(v reflect.Value, name, eventType string, method eventMethod, rootName string) {
	defer func() {
		if err := recover(); err != nil {
			fmt.Println("Calling findAndSetEvent exception:", err, ", eventType:", eventType, ", rootName:", rootName)
		}
	}()
	if event := v.MethodByName("SetOn" + eventType); event.IsValid() {
		// 设置EventId
		api.BeginAddEvent()
		defer api.EndAddEvent()
		api.SetCurrentEventId(method.FuncPtr)

		event.Call([]reflect.Value{method.Method})
	} else {
		if len(eventType) > 0 {
			// 也许分析错误，所不打印错误消息。
			s := eventType[0]
			switch {
			case s >= '0' && s <= '9' || s == '_':
				return
			}
		}
		fmt.Printf("%s.%s does not support the %s event.\n", rootName, name, eventType)
	}
}

// findAndSetComponentName 查找并设置组件名称
func findAndSetComponentName(v reflect.Value, name string) {
	defer func() {
		if err := recover(); err != nil {
			fmt.Println("Calling findAndSetComponentName exception:", err)
		}
	}()
	if v.Pointer() == 0 {
		return
	}
	if setName := v.MethodByName("SetName"); setName.IsValid() {
		setName.Call([]reflect.Value{reflect.ValueOf(name)})
	}
}

// addComponentNotifyEvent
func addComponentNotifyEvent(vForm reflect.Value, compName string, method eventMethod, eventType, rootName string) {
	if vCtl := vForm.Elem().FieldByName(compName); vCtl.IsValid() {
		findAndSetEvent(vCtl, compName, eventType, method, rootName)
	}
}

// addApplicationNotifyEvent
// 添加Application的关联事件，在一个程序内，application中的事件只有最后一次设置的才会生效。
// 因为Application是单例存在，推荐在主窗口内处理就行了。
func addApplicationNotifyEvent(eventType string, method eventMethod) {
	if app := reflect.ValueOf(Application); app.IsValid() {
		findAndSetEvent(app, "Application", eventType, method, "Application")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/GeertJohan/go.rice"
	"github.com/dfjones/riprdio/config"
	"github.com/dfjones/riprdio/importer"
	"github.com/dfjones/riprdio/token"
	"github.com/labstack/echo"
	mw "github.com/labstack/echo/middleware"
	"github.com/labstack/gommon/log"
	"net/http"
	"net/url"
)

type AuthData struct {
	AccessToken  string `json:"access_token"`
	TokenType    string `json:"token_type"`
	ExpiresIn    int    `json:"expires_in"`
	RefreshToken string `json:"refresh_token"`
}

var (
	conf config.Config
)

const (
	scope           = "user-read-private user-read-email playlist-read-private playlist-modify-public playlist-modify-private user-library-read user-library-modify playlist-read-collaborative"
	spotifyTokenUrl = "https://accounts.spotify.com/api/token"
	redirectUri     = "http://localhost:3000/callback"
	stateCookieKey  = "spotify_auth_state"
)

func main() {
	config.LoadConfig("config.json")
	conf = config.GetConfig()

	e := echo.New()

	e.Use(mw.Logger())
	e.Use(mw.Recover())

	assetHandler := http.FileServer(rice.MustFindBox("public").HTTPBox())
	e.Get("/", func(c *echo.Context) error {
		assetHandler.ServeHTTP(c.Response().Writer(), c.Request())
		return nil
	})

	e.Get("/static*", func(c *echo.Context) error {
		http.StripPrefix("/static/", assetHandler).
			ServeHTTP(c.Response().Writer(), c.Request())
		return nil
	})

	e.Get("/login", func(c *echo.Context) error {
		state := token.RandString(16)
		log.Info("state %s", state)
		resp := c.Response()
		http.SetCookie(resp.Writer(), &http.Cookie{Name: stateCookieKey, Value: state})
		v := url.Values{}
		v.Set("response_type", "code")
		v.Set("client_id", conf.ClientID)
		v.Set("scope", scope)
		ruri := conf.OAuthCallback
		if ruri == "" {
			ruri = redirectUri
		}
		v.Set("redirect_uri", ruri)
		v.Set("state", state)
		rUri := "https://accounts.spotify.com/authorize?" + v.Encode()
		return c.Redirect(http.StatusFound, rUri)
	})

	e.Get("/callback", func(c *echo.Context) error {
		req := c.Request()
		code := c.Query("code")
		state := c.Query("state")
		log.Info("code %s state %s", code, state)
		storedState, err := req.Cookie(stateCookieKey)
		if err != nil {
			return err
		}
		if state == "" || state != storedState.Value {
			v := url.Values{}
			v.Set("error", "state_mismatch")
			return c.Redirect(http.StatusFound, "/#"+v.Encode())
		}

		storedState.Value = ""
		http.SetCookie(c.Response().Writer(), storedState)

		v := url.Values{}
		v.Set("code", code)
		v.Set("redirect_uri", redirectUri)
		v.Set("grant_type", "authorization_code")
		v.Set("client_id", conf.ClientID)
		v.Set("client_secret", conf.ClientSecret)

		authResp, err := http.PostForm(spotifyTokenUrl, v)
		if err != nil {
			log.Error("Error requesting token", err)
			return err
		}
		defer authResp.Body.Close()
		log.Info("status = %s", authResp.Status)

		var authData AuthData
		err = json.NewDecoder(authResp.Body).Decode(&authData)
		if err != nil {
			log.Error("err decoding json", err)
			return err
		}
		log.Info("data %+v", authData)

		resp := c.Response()
		http.SetCookie(resp.Writer(), &http.Cookie{Name: config.AccessToken, Value: authData.AccessToken})
		http.SetCookie(resp.Writer(), &http.Cookie{Name: config.RefreshToken, Value: authData.RefreshToken})
		rv := url.Values{}
		rv.Set(config.AccessToken, authData.AccessToken)
		rv.Set(config.RefreshToken, authData.RefreshToken)
		return c.Redirect(http.StatusFound, "/#"+rv.Encode())
	})

	e.Get("/refresh_token", func(c *echo.Context) error {
		refreshToken := c.Query("refresh_token")
		v := url.Values{}
		v.Set("client_id", conf.ClientID)
		v.Set("client_secret", conf.ClientSecret)
		v.Set("grant_type", "refresh_token")
		v.Set("refresh_token", refreshToken)

		resp, err := http.PostForm(spotifyTokenUrl, v)
		if err != nil {
			return err
		}
		var authData AuthData
		err = json.NewDecoder(resp.Body).Decode(&authData)
		if err != nil {
			return err
		}
		http.SetCookie(c.Response().Writer(), &http.Cookie{Name: config.AccessToken, Value: authData.AccessToken})
		return c.JSON(http.StatusOK, authData)
	})

	e.Post("/upload", func(c *echo.Context) error {
		mr, err := c.Request().MultipartReader()
		if err != nil {
			return err
		}
		part, err := mr.NextPart()
		if err != nil {
			return err
		}
		defer part.Close()
		log.Info("%+v", part)
		state, err := importer.RunImportPipeline(c, part)
		if err != nil {
			return err
		}
		accessToken, err := c.Request().Cookie(config.AccessToken)
		if err != nil {
			return err
		}
		refreshToken, err := c.Request().Cookie(config.RefreshToken)
		if err != nil {
			return err
		}
		v := url.Values{}
		v.Set("pipeline_id", state.Id)
		v.Set(config.AccessToken, accessToken.Value)
		v.Set(config.RefreshToken, refreshToken.Value)
		return c.Redirect(http.StatusFound, "#"+v.Encode())
	})

	e.Get("/progress/:id", func(c *echo.Context) error {
		writer := c.Response().Writer()
		flusher, ok := c.Response().Writer().(http.Flusher)
		if !ok {
			return errors.New("Streaming unsupported")
		}
		header := c.Response().Header()
		header.Set("Content-Type", "text/event-stream")
		header.Set("Cache-Control", "no-cache")
		header.Set("Connection", "keep-alive")

		id := c.Param("id")
		log.Info("Looking up pipeline %s", id)
		pipeline := importer.GetRunningPipeline(id)
		if pipeline == nil {
			return c.NoContent(http.StatusNotFound)
		}

		defer log.Info("progress method return for id %s", id)

		sub := pipeline.CreateSubscriber()
		defer pipeline.RemoveSubscriber(sub)

		for message := range sub {
			statsJson, err := json.Marshal(message.Stats)
			if err != nil {
				return err
			}
			_, err = fmt.Fprintf(writer, "event: progress\ndata: %s\n\n", statsJson)
			if err != nil {
				return err
			}
			if message.NotFoundSong != nil {
				songJson, err := json.Marshal(message.NotFoundSong)
				if err != nil {
					return err
				}
				_, err = fmt.Fprintf(writer, "event: notfound\ndata: %s\n\n", songJson)
				if err != nil {
					return err
				}
			}
			flusher.Flush()
		}

		_, err := fmt.Fprintf(writer, "event: eof\n\n")
		if err != nil {
			return err
		}
		flusher.Flush()

		return nil
	})

	e.Run(":3030")
}
<commit_msg>After login redirect fixed.<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/GeertJohan/go.rice"
	"github.com/dfjones/riprdio/config"
	"github.com/dfjones/riprdio/importer"
	"github.com/dfjones/riprdio/token"
	"github.com/labstack/echo"
	mw "github.com/labstack/echo/middleware"
	"github.com/labstack/gommon/log"
	"net/http"
	"net/url"
)

type AuthData struct {
	AccessToken  string `json:"access_token"`
	TokenType    string `json:"token_type"`
	ExpiresIn    int    `json:"expires_in"`
	RefreshToken string `json:"refresh_token"`
}

var (
	conf config.Config
)

const (
	scope           = "user-read-private user-read-email playlist-read-private playlist-modify-public playlist-modify-private user-library-read user-library-modify playlist-read-collaborative"
	spotifyTokenUrl = "https://accounts.spotify.com/api/token"
	redirectUri     = "http://localhost:3030/callback"
	stateCookieKey  = "spotify_auth_state"
)

func main() {
	config.LoadConfig("config.json")
	conf = config.GetConfig()

	e := echo.New()

	e.Use(mw.Logger())
	e.Use(mw.Recover())

	assetHandler := http.FileServer(rice.MustFindBox("public").HTTPBox())
	e.Get("/", func(c *echo.Context) error {
		assetHandler.ServeHTTP(c.Response().Writer(), c.Request())
		return nil
	})

	e.Get("/static*", func(c *echo.Context) error {
		http.StripPrefix("/static/", assetHandler).
			ServeHTTP(c.Response().Writer(), c.Request())
		return nil
	})

	e.Get("/login", func(c *echo.Context) error {
		state := token.RandString(16)
		log.Info("state %s", state)
		resp := c.Response()
		http.SetCookie(resp.Writer(), &http.Cookie{Name: stateCookieKey, Value: state})
		v := url.Values{}
		v.Set("response_type", "code")
		v.Set("client_id", conf.ClientID)
		v.Set("scope", scope)
		ruri := conf.OAuthCallback
		if ruri == "" {
			ruri = redirectUri
		}
		v.Set("redirect_uri", ruri)
		v.Set("state", state)
		rUri := "https://accounts.spotify.com/authorize?" + v.Encode()
		return c.Redirect(http.StatusFound, rUri)
	})

	e.Get("/callback", func(c *echo.Context) error {
		req := c.Request()
		code := c.Query("code")
		state := c.Query("state")
		log.Info("code %s state %s", code, state)
		storedState, err := req.Cookie(stateCookieKey)
		if err != nil {
			return err
		}
		if state == "" || state != storedState.Value {
			v := url.Values{}
			v.Set("error", "state_mismatch")
			return c.Redirect(http.StatusFound, "/#"+v.Encode())
		}

		storedState.Value = ""
		http.SetCookie(c.Response().Writer(), storedState)

		v := url.Values{}
		v.Set("code", code)
		v.Set("redirect_uri", redirectUri)
		v.Set("grant_type", "authorization_code")
		v.Set("client_id", conf.ClientID)
		v.Set("client_secret", conf.ClientSecret)

		authResp, err := http.PostForm(spotifyTokenUrl, v)
		if err != nil {
			log.Error("Error requesting token", err)
			return err
		}
		defer authResp.Body.Close()
		log.Info("status = %s", authResp.Status)

		var authData AuthData
		err = json.NewDecoder(authResp.Body).Decode(&authData)
		if err != nil {
			log.Error("err decoding json", err)
			return err
		}
		log.Info("data %+v", authData)

		resp := c.Response()
		http.SetCookie(resp.Writer(), &http.Cookie{Name: config.AccessToken, Value: authData.AccessToken})
		http.SetCookie(resp.Writer(), &http.Cookie{Name: config.RefreshToken, Value: authData.RefreshToken})
		rv := url.Values{}
		rv.Set(config.AccessToken, authData.AccessToken)
		rv.Set(config.RefreshToken, authData.RefreshToken)
		return c.Redirect(http.StatusFound, "../#"+rv.Encode())
	})

	e.Get("/refresh_token", func(c *echo.Context) error {
		refreshToken := c.Query("refresh_token")
		v := url.Values{}
		v.Set("client_id", conf.ClientID)
		v.Set("client_secret", conf.ClientSecret)
		v.Set("grant_type", "refresh_token")
		v.Set("refresh_token", refreshToken)

		resp, err := http.PostForm(spotifyTokenUrl, v)
		if err != nil {
			return err
		}
		var authData AuthData
		err = json.NewDecoder(resp.Body).Decode(&authData)
		if err != nil {
			return err
		}
		http.SetCookie(c.Response().Writer(), &http.Cookie{Name: config.AccessToken, Value: authData.AccessToken})
		return c.JSON(http.StatusOK, authData)
	})

	e.Post("/upload", func(c *echo.Context) error {
		mr, err := c.Request().MultipartReader()
		if err != nil {
			return err
		}
		part, err := mr.NextPart()
		if err != nil {
			return err
		}
		defer part.Close()
		log.Info("%+v", part)
		state, err := importer.RunImportPipeline(c, part)
		if err != nil {
			return err
		}
		accessToken, err := c.Request().Cookie(config.AccessToken)
		if err != nil {
			return err
		}
		refreshToken, err := c.Request().Cookie(config.RefreshToken)
		if err != nil {
			return err
		}
		v := url.Values{}
		v.Set("pipeline_id", state.Id)
		v.Set(config.AccessToken, accessToken.Value)
		v.Set(config.RefreshToken, refreshToken.Value)
		return c.Redirect(http.StatusFound, "#"+v.Encode())
	})

	e.Get("/progress/:id", func(c *echo.Context) error {
		writer := c.Response().Writer()
		flusher, ok := c.Response().Writer().(http.Flusher)
		if !ok {
			return errors.New("Streaming unsupported")
		}
		header := c.Response().Header()
		header.Set("Content-Type", "text/event-stream")
		header.Set("Cache-Control", "no-cache")
		header.Set("Connection", "keep-alive")

		id := c.Param("id")
		log.Info("Looking up pipeline %s", id)
		pipeline := importer.GetRunningPipeline(id)
		if pipeline == nil {
			return c.NoContent(http.StatusNotFound)
		}

		defer log.Info("progress method return for id %s", id)

		sub := pipeline.CreateSubscriber()
		defer pipeline.RemoveSubscriber(sub)

		for message := range sub {
			statsJson, err := json.Marshal(message.Stats)
			if err != nil {
				return err
			}
			_, err = fmt.Fprintf(writer, "event: progress\ndata: %s\n\n", statsJson)
			if err != nil {
				return err
			}
			if message.NotFoundSong != nil {
				songJson, err := json.Marshal(message.NotFoundSong)
				if err != nil {
					return err
				}
				_, err = fmt.Fprintf(writer, "event: notfound\ndata: %s\n\n", songJson)
				if err != nil {
					return err
				}
			}
			flusher.Flush()
		}

		_, err := fmt.Fprintf(writer, "event: eof\n\n")
		if err != nil {
			return err
		}
		flusher.Flush()

		return nil
	})

	e.Run(":3030")
}
<|endoftext|>


<commit_before>package nagiosplugin

import (
	"math"
	"testing"
)

func TestPerfdata(t *testing.T) {
	expected := "badness=9003.4ms;4000;9000;10;"
	pd, err := NewPerfDatum("badness", "ms", 9003.4, 10.0, math.Inf(1), 4000.0, 9000.0)
	if err != nil {
		t.Errorf("Could not render perfdata: %v", err)
	}
	if pd.String() != expected {
		t.Errorf("Perfdata rendering error: expected %s, got %v", expected, pd)
	}
}

func TestRenderPerfdata(t *testing.T) {
	expected := " | goodness=3.141592653589793kb;;;3;34.55751918948773 goodness=6.283185307179586kb;;;3;34.55751918948773 goodness=9.42477796076938kb;;;3;34.55751918948773 goodness=12.566370614359172kb;;;3;34.55751918948773 goodness=15.707963267948966kb;;;3;34.55751918948773 goodness=18.84955592153876kb;;;3;34.55751918948773 goodness=21.991148575128552kb;;;3;34.55751918948773 goodness=25.132741228718345kb;;;3;34.55751918948773 goodness=28.274333882308138kb;;;3;34.55751918948773 goodness=31.41592653589793kb;;;3;34.55751918948773"
	pd := make([]PerfDatum, 0)
	for i := 0; i < 10; i++ {
		datum, err := NewPerfDatum("goodness", "kb", math.Pi*float64(i+1), 3.0, math.Pi*11)
		if err != nil {
			t.Errorf("Could not create perfdata: %v", err)
		}
		pd = append(pd, *datum)
	}
	result := RenderPerfdata(pd)
	if result != expected {
		t.Errorf("Perfdata rendering error: expected %s, got %v", expected, result)
	}
}
<commit_msg>Add test for omitted thresholds<commit_after>package nagiosplugin

import (
	"math"
	"testing"
)

func TestPerfdata(t *testing.T) {
	expected := "badness=9003.4ms;4000;9000;10;"
	pd, err := NewPerfDatum("badness", "ms", 9003.4, 10.0, math.Inf(1), 4000.0, 9000.0)
	if err != nil {
		t.Errorf("Could not render perfdata: %v", err)
	}
	if pd.String() != expected {
		t.Errorf("Perfdata rendering error: expected %s, got %v", expected, pd)
	}
}

func TestRenderPerfdata(t *testing.T) {
	expected := " | goodness=3.141592653589793kb;;;3;34.55751918948773 goodness=6.283185307179586kb;;;3;34.55751918948773 goodness=9.42477796076938kb;;;3;34.55751918948773 goodness=12.566370614359172kb;;;3;34.55751918948773 goodness=15.707963267948966kb;;;3;34.55751918948773 goodness=18.84955592153876kb;;;3;34.55751918948773 goodness=21.991148575128552kb;;;3;34.55751918948773 goodness=25.132741228718345kb;;;3;34.55751918948773 goodness=28.274333882308138kb;;;3;34.55751918948773 goodness=31.41592653589793kb;;;3;34.55751918948773"
	pd := make([]PerfDatum, 0)
	for i := 0; i < 10; i++ {
		datum, err := NewPerfDatum("goodness", "kb", math.Pi*float64(i+1), 3.0, math.Pi*11)
		if err != nil {
			t.Errorf("Could not create perfdata: %v", err)
		}
		pd = append(pd, *datum)
	}
	result := RenderPerfdata(pd)
	if result != expected {
		t.Errorf("Perfdata rendering error: expected %s, got %v", expected, result)
	}
}

func TestRenderPerfdataWithOmissions(t *testing.T) {
	pd := make([]PerfDatum, 0)
	datum, err := NewPerfDatum(
		"age",       // label
		"s",         // UOM
		0.123,       // value
		0.0,         // min
		math.Inf(1), // max: +Inf -> omit
		math.NaN(),  // warn: NaN -> omit
		0.5)         // crit
	if err != nil {
		t.Errorf("Could not create perfdata: %v", err)
	}
	pd = append(pd, *datum)

	// 'label'=value[UOM];[warn];[crit];[min];[max]
	expected := " | age=0.123s;;0.5;0;"
	result := RenderPerfdata(pd)
	if result != expected {
		t.Errorf("Perfdata rendering error: expected %s, got %v", expected, result)
	}
}
<|endoftext|>


<commit_before>// Copyright 2012 Google, Inc. All rights reserved.
// Copyright 2009-2011 Andreas Krennmair. All rights reserved.
//
// Use of this source code is governed by a BSD-style license
// that can be found in the LICENSE file in the root of the source
// tree.

package pfring

/*
#cgo LDFLAGS: -lpfring -lpcap
#include <stdlib.h>
#include <pfring.h>
#include <linux/pf_ring.h>
*/
import "C"

// NOTE:  If you install PF_RING with non-standard options, you may also need
// to use LDFLAGS -lnuma and/or -lrt.  Both have been reported necessary if
// PF_RING is configured with --disable-bpf.

import (
	"fmt"
	"github.com/google/gopacket"
	"os"
	"strconv"
	"sync"
	"time"
	"unsafe"
)

const errorBufferSize = 256

// Ring provides a handle to a pf_ring.
type Ring struct {
	// cptr is the handle for the actual pcap C object.
	cptr    *C.pfring
	snaplen int

	mu sync.Mutex
	// Since pointers to these objects are passed into a C function, if
	// they're declared locally then the Go compiler thinks they may have
	// escaped into C-land, so it allocates them on the heap.  This causes a
	// huge memory hit, so to handle that we store them here instead.
	pkthdr  C.struct_pfring_pkthdr
	buf_ptr *C.u_char
}

type Flag uint32

const (
	FlagReentrant       Flag = C.PF_RING_REENTRANT
	FlagLongHeader      Flag = C.PF_RING_LONG_HEADER
	FlagPromisc         Flag = C.PF_RING_PROMISC
	FlagDNASymmetricRSS Flag = C.PF_RING_DNA_SYMMETRIC_RSS
	FlagTimestamp       Flag = C.PF_RING_TIMESTAMP
	FlagHWTimestamp     Flag = C.PF_RING_HW_TIMESTAMP
)

// NewRing creates a new PFRing.  Note that when the ring is initially created,
// it is disabled.  The caller must call Enable to start receiving packets.
// The caller should call Close on the given ring when finished with it.
func NewRing(device string, snaplen uint32, flags Flag) (ring *Ring, _ error) {
	dev := C.CString(device)
	defer C.free(unsafe.Pointer(dev))

	cptr, err := C.pfring_open(dev, C.u_int32_t(snaplen), C.u_int32_t(flags))
	if cptr == nil || err != nil {
		return nil, fmt.Errorf("pfring NewRing error: %v", err)
	}
	ring = &Ring{cptr: cptr, snaplen: int(snaplen)}
	ring.SetApplicationName(os.Args[0])
	return
}

// Close closes the given Ring.  After this call, the Ring should no longer be
// used.
func (r *Ring) Close() {
	C.pfring_close(r.cptr)
}

// NextResult is the return code from a call to Next.
type NextResult int32

const (
	NextNoPacketNonblocking NextResult = 0
	NextError               NextResult = -1
	NextOk                  NextResult = 1
	NextNotEnabled          NextResult = -7
)

// NextResult implements the error interface.
func (n NextResult) Error() string {
	switch n {
	case NextNoPacketNonblocking:
		return "No packet available, nonblocking socket"
	case NextError:
		return "Generic error"
	case NextOk:
		return "Success (not an error)"
	case NextNotEnabled:
		return "Ring not enabled"
	}
	return strconv.Itoa(int(n))
}

// ReadPacketDataTo reads packet data into a user-supplied buffer.
// This function ignores snaplen and instead reads up to the length of the
// passed-in slice.
// The number of bytes read into data will be returned in ci.CaptureLength.
func (r *Ring) ReadPacketDataTo(data []byte) (ci gopacket.CaptureInfo, err error) {
	// This tricky buf_ptr points to the start of our slice data, so pfring_recv
	// will actually write directly into our Go slice.  Nice!
	r.mu.Lock()
	r.buf_ptr = (*C.u_char)(unsafe.Pointer(&data[0]))
	result := NextResult(C.pfring_recv(r.cptr, &r.buf_ptr, C.u_int(len(data)), &r.pkthdr, 1))
	if result != NextOk {
		err = result
		r.mu.Unlock()
		return
	}
	ci.Timestamp = time.Unix(int64(r.pkthdr.ts.tv_sec),
		int64(r.pkthdr.ts.tv_usec)*1000) // convert micros to nanos
	ci.CaptureLength = int(r.pkthdr.caplen)
	ci.Length = int(r.pkthdr.len)
	r.mu.Unlock()
	return
}

// ReadPacketData returns the next packet read from the pcap handle, along with an error
// code associated with that packet.  If the packet is read successfully, the
// returned error is nil.
func (r *Ring) ReadPacketData() (data []byte, ci gopacket.CaptureInfo, err error) {
	data = make([]byte, r.snaplen)
	ci, err = r.ReadPacketDataTo(data)
	if err != nil {
		data = nil
		return
	}
	data = data[:ci.CaptureLength]
	return
}

type ClusterType C.cluster_type

const (
	// ClusterPerFlow clusters by <src ip, src port, dst ip, dst port, proto,
	// vlan>
	ClusterPerFlow ClusterType = C.cluster_per_flow
	// ClusterRoundRobin round-robins packets between applications, ignoring
	// packet information.
	ClusterRoundRobin ClusterType = C.cluster_round_robin
	// ClusterPerFlow2Tuple clusters by <src ip, dst ip>
	ClusterPerFlow2Tuple ClusterType = C.cluster_per_flow_2_tuple
	// ClusterPerFlow4Tuple clusters by <src ip, src port, dst ip, dst port>
	ClusterPerFlow4Tuple ClusterType = C.cluster_per_flow_4_tuple
	// ClusterPerFlow5Tuple clusters by <src ip, src port, dst ip, dst port,
	// proto>
	ClusterPerFlow5Tuple ClusterType = C.cluster_per_flow_5_tuple
	// ClusterPerFlowTCP5Tuple acts like ClusterPerFlow5Tuple for TCP packets and
	// like ClusterPerFlow2Tuple for all other packets.
	ClusterPerFlowTCP5Tuple ClusterType = C.cluster_per_flow_tcp_5_tuple
)

// SetCluster sets which cluster the ring should be part of, and the cluster
// type to use.
func (r *Ring) SetCluster(cluster int, typ ClusterType) error {
	if rv := C.pfring_set_cluster(r.cptr, C.u_int(cluster), C.cluster_type(typ)); rv != 0 {
		return fmt.Errorf("Unable to set cluster, got error code %d", rv)
	}
	return nil
}

// RemoveFromCluster removes the ring from the cluster it was put in with
// SetCluster.
func (r *Ring) RemoveFromCluster() error {
	if rv := C.pfring_remove_from_cluster(r.cptr); rv != 0 {
		return fmt.Errorf("Unable to remove from cluster, got error code %d", rv)
	}
	return nil
}

// SetSamplingRate sets the sampling rate to 1/<rate>.
func (r *Ring) SetSamplingRate(rate int) error {
	if rv := C.pfring_set_sampling_rate(r.cptr, C.u_int32_t(rate)); rv != 0 {
		return fmt.Errorf("Unable to set sampling rate, got error code %d", rv)
	}
	return nil
}

// SetBPFFilter sets the BPF filter for the ring.
func (r *Ring) SetBPFFilter(bpf_filter string) error {
	filter := C.CString(bpf_filter)
	defer C.free(unsafe.Pointer(filter))
	if rv := C.pfring_set_bpf_filter(r.cptr, filter); rv != 0 {
		return fmt.Errorf("Unable to set BPF filter, got error code %d", rv)
	}
	return nil
}

// RemoveBPFFilter removes the BPF filter from the ring.
func (r *Ring) RemoveBPFFilter() error {
	if rv := C.pfring_remove_bpf_filter(r.cptr); rv != 0 {
		return fmt.Errorf("Unable to remove BPF filter, got error code %d", rv)
	}
	return nil
}

// WritePacketData uses the ring to send raw packet data to the interface.
func (r *Ring) WritePacketData(data []byte) error {
	buf := (*C.char)(unsafe.Pointer(&data[0]))
	if rv := C.pfring_send(r.cptr, buf, C.u_int(len(data)), 1); rv != 0 {
		return fmt.Errorf("Unable to send packet data, got error code %d", rv)
	}
	return nil
}

// Enable enables the given ring.  This function MUST be called on each new
// ring after it has been set up, or that ring will NOT receive packets.
func (r *Ring) Enable() error {
	if rv := C.pfring_enable_ring(r.cptr); rv != 0 {
		return fmt.Errorf("Unable to enable ring, got error code %d", rv)
	}
	return nil
}

// Disable disables the given ring.  After this call, it will no longer receive
// packets.
func (r *Ring) Disable() error {
	if rv := C.pfring_disable_ring(r.cptr); rv != 0 {
		return fmt.Errorf("Unable to disable ring, got error code %d", rv)
	}
	return nil
}

type Stats struct {
	Received, Dropped uint64
}

// Stats returns statistsics for the ring.
func (r *Ring) Stats() (s Stats, err error) {
	var stats C.pfring_stat
	if rv := C.pfring_stats(r.cptr, &stats); rv != 0 {
		err = fmt.Errorf("Unable to get ring stats, got error code %d", rv)
		return
	}
	s.Received = uint64(stats.recv)
	s.Dropped = uint64(stats.drop)
	return
}

type Direction C.packet_direction

const (
	// TransmitOnly will only capture packets transmitted by the ring's
	// interface(s).
	TransmitOnly Direction = C.tx_only_direction
	// ReceiveOnly will only capture packets received by the ring's
	// interface(s).
	ReceiveOnly Direction = C.rx_only_direction
	// ReceiveAndTransmit will capture both received and transmitted packets on
	// the ring's interface(s).
	ReceiveAndTransmit Direction = C.rx_and_tx_direction
)

// SetDirection sets which packets should be captured by the ring.
func (r *Ring) SetDirection(d Direction) error {
	if rv := C.pfring_set_direction(r.cptr, C.packet_direction(d)); rv != 0 {
		return fmt.Errorf("Unable to set ring direction, got error code %d", rv)
	}
	return nil
}

type SocketMode C.socket_mode

const (
	// WriteOnly sets up the ring to only send packets (Inject), not read them.
	WriteOnly SocketMode = C.send_only_mode
	// ReadOnly sets up the ring to only receive packets (ReadPacketData), not
	// send them.
	ReadOnly SocketMode = C.recv_only_mode
	// WriteAndRead sets up the ring to both send and receive packets.
	WriteAndRead SocketMode = C.send_and_recv_mode
)

// SetSocketMode sets the mode of the ring socket to send, receive, or both.
func (r *Ring) SetSocketMode(s SocketMode) error {
	if rv := C.pfring_set_socket_mode(r.cptr, C.socket_mode(s)); rv != 0 {
		return fmt.Errorf("Unable to set socket mode, got error code %d", rv)
	}
	return nil
}

// SetApplicationName sets a string name to the ring.  This name is available in
// /proc stats for pf_ring.  By default, NewRing automatically calls this with
// argv[0].
func (r *Ring) SetApplicationName(name string) error {
	buf := C.CString(name)
	defer C.free(unsafe.Pointer(buf))
	if rv := C.pfring_set_application_name(r.cptr, buf); rv != 0 {
		return fmt.Errorf("Unable to set ring application name, got error code %d", rv)
	}
	return nil
}
<commit_msg>Fix PF_RING WritePacketData rv error<commit_after>// Copyright 2012 Google, Inc. All rights reserved.
// Copyright 2009-2011 Andreas Krennmair. All rights reserved.
//
// Use of this source code is governed by a BSD-style license
// that can be found in the LICENSE file in the root of the source
// tree.

package pfring

/*
#cgo LDFLAGS: -lpfring -lpcap
#include <stdlib.h>
#include <pfring.h>
#include <linux/pf_ring.h>
*/
import "C"

// NOTE:  If you install PF_RING with non-standard options, you may also need
// to use LDFLAGS -lnuma and/or -lrt.  Both have been reported necessary if
// PF_RING is configured with --disable-bpf.

import (
	"fmt"
	"github.com/google/gopacket"
	"os"
	"strconv"
	"sync"
	"time"
	"unsafe"
)

const errorBufferSize = 256

// Ring provides a handle to a pf_ring.
type Ring struct {
	// cptr is the handle for the actual pcap C object.
	cptr    *C.pfring
	snaplen int

	mu sync.Mutex
	// Since pointers to these objects are passed into a C function, if
	// they're declared locally then the Go compiler thinks they may have
	// escaped into C-land, so it allocates them on the heap.  This causes a
	// huge memory hit, so to handle that we store them here instead.
	pkthdr  C.struct_pfring_pkthdr
	buf_ptr *C.u_char
}

type Flag uint32

const (
	FlagReentrant       Flag = C.PF_RING_REENTRANT
	FlagLongHeader      Flag = C.PF_RING_LONG_HEADER
	FlagPromisc         Flag = C.PF_RING_PROMISC
	FlagDNASymmetricRSS Flag = C.PF_RING_DNA_SYMMETRIC_RSS
	FlagTimestamp       Flag = C.PF_RING_TIMESTAMP
	FlagHWTimestamp     Flag = C.PF_RING_HW_TIMESTAMP
)

// NewRing creates a new PFRing.  Note that when the ring is initially created,
// it is disabled.  The caller must call Enable to start receiving packets.
// The caller should call Close on the given ring when finished with it.
func NewRing(device string, snaplen uint32, flags Flag) (ring *Ring, _ error) {
	dev := C.CString(device)
	defer C.free(unsafe.Pointer(dev))

	cptr, err := C.pfring_open(dev, C.u_int32_t(snaplen), C.u_int32_t(flags))
	if cptr == nil || err != nil {
		return nil, fmt.Errorf("pfring NewRing error: %v", err)
	}
	ring = &Ring{cptr: cptr, snaplen: int(snaplen)}
	ring.SetApplicationName(os.Args[0])
	return
}

// Close closes the given Ring.  After this call, the Ring should no longer be
// used.
func (r *Ring) Close() {
	C.pfring_close(r.cptr)
}

// NextResult is the return code from a call to Next.
type NextResult int32

const (
	NextNoPacketNonblocking NextResult = 0
	NextError               NextResult = -1
	NextOk                  NextResult = 1
	NextNotEnabled          NextResult = -7
)

// NextResult implements the error interface.
func (n NextResult) Error() string {
	switch n {
	case NextNoPacketNonblocking:
		return "No packet available, nonblocking socket"
	case NextError:
		return "Generic error"
	case NextOk:
		return "Success (not an error)"
	case NextNotEnabled:
		return "Ring not enabled"
	}
	return strconv.Itoa(int(n))
}

// ReadPacketDataTo reads packet data into a user-supplied buffer.
// This function ignores snaplen and instead reads up to the length of the
// passed-in slice.
// The number of bytes read into data will be returned in ci.CaptureLength.
func (r *Ring) ReadPacketDataTo(data []byte) (ci gopacket.CaptureInfo, err error) {
	// This tricky buf_ptr points to the start of our slice data, so pfring_recv
	// will actually write directly into our Go slice.  Nice!
	r.mu.Lock()
	r.buf_ptr = (*C.u_char)(unsafe.Pointer(&data[0]))
	result := NextResult(C.pfring_recv(r.cptr, &r.buf_ptr, C.u_int(len(data)), &r.pkthdr, 1))
	if result != NextOk {
		err = result
		r.mu.Unlock()
		return
	}
	ci.Timestamp = time.Unix(int64(r.pkthdr.ts.tv_sec),
		int64(r.pkthdr.ts.tv_usec)*1000) // convert micros to nanos
	ci.CaptureLength = int(r.pkthdr.caplen)
	ci.Length = int(r.pkthdr.len)
	r.mu.Unlock()
	return
}

// ReadPacketData returns the next packet read from the pcap handle, along with an error
// code associated with that packet.  If the packet is read successfully, the
// returned error is nil.
func (r *Ring) ReadPacketData() (data []byte, ci gopacket.CaptureInfo, err error) {
	data = make([]byte, r.snaplen)
	ci, err = r.ReadPacketDataTo(data)
	if err != nil {
		data = nil
		return
	}
	data = data[:ci.CaptureLength]
	return
}

type ClusterType C.cluster_type

const (
	// ClusterPerFlow clusters by <src ip, src port, dst ip, dst port, proto,
	// vlan>
	ClusterPerFlow ClusterType = C.cluster_per_flow
	// ClusterRoundRobin round-robins packets between applications, ignoring
	// packet information.
	ClusterRoundRobin ClusterType = C.cluster_round_robin
	// ClusterPerFlow2Tuple clusters by <src ip, dst ip>
	ClusterPerFlow2Tuple ClusterType = C.cluster_per_flow_2_tuple
	// ClusterPerFlow4Tuple clusters by <src ip, src port, dst ip, dst port>
	ClusterPerFlow4Tuple ClusterType = C.cluster_per_flow_4_tuple
	// ClusterPerFlow5Tuple clusters by <src ip, src port, dst ip, dst port,
	// proto>
	ClusterPerFlow5Tuple ClusterType = C.cluster_per_flow_5_tuple
	// ClusterPerFlowTCP5Tuple acts like ClusterPerFlow5Tuple for TCP packets and
	// like ClusterPerFlow2Tuple for all other packets.
	ClusterPerFlowTCP5Tuple ClusterType = C.cluster_per_flow_tcp_5_tuple
)

// SetCluster sets which cluster the ring should be part of, and the cluster
// type to use.
func (r *Ring) SetCluster(cluster int, typ ClusterType) error {
	if rv := C.pfring_set_cluster(r.cptr, C.u_int(cluster), C.cluster_type(typ)); rv != 0 {
		return fmt.Errorf("Unable to set cluster, got error code %d", rv)
	}
	return nil
}

// RemoveFromCluster removes the ring from the cluster it was put in with
// SetCluster.
func (r *Ring) RemoveFromCluster() error {
	if rv := C.pfring_remove_from_cluster(r.cptr); rv != 0 {
		return fmt.Errorf("Unable to remove from cluster, got error code %d", rv)
	}
	return nil
}

// SetSamplingRate sets the sampling rate to 1/<rate>.
func (r *Ring) SetSamplingRate(rate int) error {
	if rv := C.pfring_set_sampling_rate(r.cptr, C.u_int32_t(rate)); rv != 0 {
		return fmt.Errorf("Unable to set sampling rate, got error code %d", rv)
	}
	return nil
}

// SetBPFFilter sets the BPF filter for the ring.
func (r *Ring) SetBPFFilter(bpf_filter string) error {
	filter := C.CString(bpf_filter)
	defer C.free(unsafe.Pointer(filter))
	if rv := C.pfring_set_bpf_filter(r.cptr, filter); rv != 0 {
		return fmt.Errorf("Unable to set BPF filter, got error code %d", rv)
	}
	return nil
}

// RemoveBPFFilter removes the BPF filter from the ring.
func (r *Ring) RemoveBPFFilter() error {
	if rv := C.pfring_remove_bpf_filter(r.cptr); rv != 0 {
		return fmt.Errorf("Unable to remove BPF filter, got error code %d", rv)
	}
	return nil
}

// WritePacketData uses the ring to send raw packet data to the interface.
func (r *Ring) WritePacketData(data []byte) error {
	buf := (*C.char)(unsafe.Pointer(&data[0]))
	if rv := C.pfring_send(r.cptr, buf, C.u_int(len(data)), 1); rv < 0 {
		return fmt.Errorf("Unable to send packet data, got error code %d", rv)
	}
	return nil
}

// Enable enables the given ring.  This function MUST be called on each new
// ring after it has been set up, or that ring will NOT receive packets.
func (r *Ring) Enable() error {
	if rv := C.pfring_enable_ring(r.cptr); rv != 0 {
		return fmt.Errorf("Unable to enable ring, got error code %d", rv)
	}
	return nil
}

// Disable disables the given ring.  After this call, it will no longer receive
// packets.
func (r *Ring) Disable() error {
	if rv := C.pfring_disable_ring(r.cptr); rv != 0 {
		return fmt.Errorf("Unable to disable ring, got error code %d", rv)
	}
	return nil
}

type Stats struct {
	Received, Dropped uint64
}

// Stats returns statistsics for the ring.
func (r *Ring) Stats() (s Stats, err error) {
	var stats C.pfring_stat
	if rv := C.pfring_stats(r.cptr, &stats); rv != 0 {
		err = fmt.Errorf("Unable to get ring stats, got error code %d", rv)
		return
	}
	s.Received = uint64(stats.recv)
	s.Dropped = uint64(stats.drop)
	return
}

type Direction C.packet_direction

const (
	// TransmitOnly will only capture packets transmitted by the ring's
	// interface(s).
	TransmitOnly Direction = C.tx_only_direction
	// ReceiveOnly will only capture packets received by the ring's
	// interface(s).
	ReceiveOnly Direction = C.rx_only_direction
	// ReceiveAndTransmit will capture both received and transmitted packets on
	// the ring's interface(s).
	ReceiveAndTransmit Direction = C.rx_and_tx_direction
)

// SetDirection sets which packets should be captured by the ring.
func (r *Ring) SetDirection(d Direction) error {
	if rv := C.pfring_set_direction(r.cptr, C.packet_direction(d)); rv != 0 {
		return fmt.Errorf("Unable to set ring direction, got error code %d", rv)
	}
	return nil
}

type SocketMode C.socket_mode

const (
	// WriteOnly sets up the ring to only send packets (Inject), not read them.
	WriteOnly SocketMode = C.send_only_mode
	// ReadOnly sets up the ring to only receive packets (ReadPacketData), not
	// send them.
	ReadOnly SocketMode = C.recv_only_mode
	// WriteAndRead sets up the ring to both send and receive packets.
	WriteAndRead SocketMode = C.send_and_recv_mode
)

// SetSocketMode sets the mode of the ring socket to send, receive, or both.
func (r *Ring) SetSocketMode(s SocketMode) error {
	if rv := C.pfring_set_socket_mode(r.cptr, C.socket_mode(s)); rv != 0 {
		return fmt.Errorf("Unable to set socket mode, got error code %d", rv)
	}
	return nil
}

// SetApplicationName sets a string name to the ring.  This name is available in
// /proc stats for pf_ring.  By default, NewRing automatically calls this with
// argv[0].
func (r *Ring) SetApplicationName(name string) error {
	buf := C.CString(name)
	defer C.free(unsafe.Pointer(buf))
	if rv := C.pfring_set_application_name(r.cptr, buf); rv != 0 {
		return fmt.Errorf("Unable to set ring application name, got error code %d", rv)
	}
	return nil
}
<|endoftext|>


<commit_before>package google

import (
	"fmt"
	"sort"
	"testing"

	"github.com/hashicorp/terraform/helper/acctest"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
	"google.golang.org/api/cloudresourcemanager/v1"
)

// Test that an IAM binding can be applied to a project
func TestAccGoogleProjectIamBinding_basic(t *testing.T) {
	pid := "terraform-" + acctest.RandString(10)
	resource.Test(t, resource.TestCase{
		PreCheck:  func() { testAccPreCheck(t) },
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			// Create a new project
			{
				Config: testAccGoogleProject_create(pid, pname, org),
				Check: resource.ComposeTestCheckFunc(
					testAccGoogleProjectExistingPolicy(pid),
				),
			},
			// Apply an IAM binding
			{
				Config: testAccGoogleProjectAssociateBindingBasic(pid, pname, org),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckGoogleProjectIamBindingExists("google_project_iam_binding.acceptance", &cloudresourcemanager.Binding{
						Role:    "roles/compute.instanceAdmin",
						Members: []string{"user:admin@hashicorptest.com"},
					}, pid),
				),
			},
		},
	})
}

// Test that multiple IAM bindings can be applied to a project
func TestAccGoogleProjectIamBinding_multiple(t *testing.T) {
	pid := "terraform-" + acctest.RandString(10)
	resource.Test(t, resource.TestCase{
		PreCheck:  func() { testAccPreCheck(t) },
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			// Create a new project
			{
				Config: testAccGoogleProject_create(pid, pname, org),
				Check: resource.ComposeTestCheckFunc(
					testAccGoogleProjectExistingPolicy(pid),
				),
			},
			// Apply an IAM binding
			{
				Config: testAccGoogleProjectAssociateBindingBasic(pid, pname, org),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckGoogleProjectIamBindingExists("google_project_iam_binding.acceptance", &cloudresourcemanager.Binding{
						Role:    "roles/compute.instanceAdmin",
						Members: []string{"user:admin@hashicorptest.com"},
					}, pid),
				),
			},
			// Apply another IAM binding
			{
				Config: testAccGoogleProjectAssociateBindingMultiple(pid, pname, org),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckGoogleProjectIamBindingExists("google_project_iam_binding.multiple", &cloudresourcemanager.Binding{
						Role:    "roles/viewer",
						Members: []string{"user:paddy@hashicorp.com"},
					}, pid),
				),
			},
		},
	})
}

// Test that an IAM binding can be updated once applied to a project
func TestAccGoogleProjectIamBinding_update(t *testing.T) {
	pid := "terraform-" + acctest.RandString(10)
	resource.Test(t, resource.TestCase{
		PreCheck:  func() { testAccPreCheck(t) },
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			// Create a new project
			{
				Config: testAccGoogleProject_create(pid, pname, org),
				Check: resource.ComposeTestCheckFunc(
					testAccGoogleProjectExistingPolicy(pid),
				),
			},
			// Apply an IAM binding
			{
				Config: testAccGoogleProjectAssociateBindingBasic(pid, pname, org),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckGoogleProjectIamBindingExists("google_project_iam_binding.acceptance", &cloudresourcemanager.Binding{
						Role:    "roles/compute.instanceAdmin",
						Members: []string{"user:admin@hashicorptest.com"},
					}, pid),
				),
			},
			// Apply an updated IAM binding
			{
				Config: testAccGoogleProjectAssociateBindingUpdated(pid, pname, org),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckGoogleProjectIamBindingExists("google_project_iam_binding.updated", &cloudresourcemanager.Binding{
						Role:    "roles/compute.instanceAdmin",
						Members: []string{"user:admin@hashicorptest.com", "user:paddy@hashicorp.com"},
					}, pid),
				),
			},
			// Drop the original member
			{
				Config: testAccGoogleProjectAssociateBindingDropMemberFromBasic(pid, pname, org),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckGoogleProjectIamBindingExists("google_project_iam_binding.dropped", &cloudresourcemanager.Binding{
						Role:    "roles/compute.instanceAdmin",
						Members: []string{"user:paddy@hashicorp.com"},
					}, pid),
				),
			},
		},
	})
}

// Test that an IAM binding can be removed from a project
func TestAccGoogleProjectIamBinding_remove(t *testing.T) {
	pid := "terraform-" + acctest.RandString(10)
	resource.Test(t, resource.TestCase{
		PreCheck:  func() { testAccPreCheck(t) },
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			// Create a new project
			{
				Config: testAccGoogleProject_create(pid, pname, org),
				Check: resource.ComposeTestCheckFunc(
					testAccGoogleProjectExistingPolicy(pid),
				),
			},
			// Apply multiple IAM bindings
			{
				Config: testAccGoogleProjectAssociateBindingMultiple(pid, pname, org),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckGoogleProjectIamBindingExists("google_project_iam_binding.multiple", &cloudresourcemanager.Binding{
						Role:    "roles/viewer",
						Members: []string{"user:paddy@hashicorp.com"},
					}, pid),
					testAccCheckGoogleProjectIamBindingExists("google_project_iam_binding.acceptance", &cloudresourcemanager.Binding{
						Role:    "roles/compute.instanceAdmin",
						Members: []string{"user:admin@hashicorptest.com"},
					}, pid),
				),
			},
			// Remove the bindings
			{
				Config: testAccGoogleProject_create(pid, pname, org),
				Check: resource.ComposeTestCheckFunc(
					testAccGoogleProjectExistingPolicy(pid),
				),
			},
		},
	})
}

func testAccCheckGoogleProjectIamBindingExists(key string, expected *cloudresourcemanager.Binding, pid string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		config := testAccProvider.Meta().(*Config)
		projectPolicy, err := getProjectIamPolicy(pid, config)
		if err != nil {
			return fmt.Errorf("Failed to retrieve IAM policy for project %q: %s", pid, err)
		}

		var result *cloudresourcemanager.Binding
		for _, binding := range projectPolicy.Bindings {
			if binding.Role == expected.Role {
				result = binding
				break
			}
		}
		if result == nil {
			return fmt.Errorf("IAM policy for project %q had no role %q", pid, expected.Role)
		}
		if len(result.Members) != len(expected.Members) {
			return fmt.Errorf("Got %v as members for role %q of project %q, expected %v", result.Members, expected.Role, pid, expected.Members)
		}
		sort.Strings(result.Members)
		sort.Strings(expected.Members)
		for pos, exp := range expected.Members {
			if result.Members[pos] != exp {
				return fmt.Errorf("Expected members for role %q of project %q to be %v, got %v", expected.Role, pid, expected.Members, result.Members)
			}
		}
		return nil
	}
}

func testAccGoogleProjectAssociateBindingBasic(pid, name, org string) string {
	return fmt.Sprintf(`
resource "google_project" "acceptance" {
  project_id = "%s"
  name       = "%s"
  org_id     = "%s"
}

resource "google_project_iam_binding" "acceptance" {
  project = "${google_project.acceptance.project_id}"
  members = ["user:admin@hashicorptest.com"]
  role    = "roles/compute.instanceAdmin"
}
`, pid, name, org)
}

func testAccGoogleProjectAssociateBindingMultiple(pid, name, org string) string {
	return fmt.Sprintf(`
resource "google_project" "acceptance" {
  project_id = "%s"
  name       = "%s"
  org_id     = "%s"
}

resource "google_project_iam_binding" "acceptance" {
  project = "${google_project.acceptance.project_id}"
  members = ["user:admin@hashicorptest.com"]
  role    = "roles/compute.instanceAdmin"
}

resource "google_project_iam_binding" "multiple" {
  project = "${google_project.acceptance.project_id}"
  members = ["user:paddy@hashicorp.com"]
  role    = "roles/viewer"
}
`, pid, name, org)
}

func testAccGoogleProjectAssociateBindingUpdated(pid, name, org string) string {
	return fmt.Sprintf(`
resource "google_project" "acceptance" {
  project_id = "%s"
  name       = "%s"
  org_id     = "%s"
}

resource "google_project_iam_binding" "acceptance" {
  project = "${google_project.acceptance.project_id}"
  members = ["user:admin@hashicorptest.com", "user:paddy@hashicorp.com"]
  role    = "roles/compute.instanceAdmin"
}
`, pid, name, org)
}

func testAccGoogleProjectAssociateBindingDropMemberFromBasic(pid, name, org string) string {
	return fmt.Sprintf(`
resource "google_project" "acceptance" {
  project_id = "%s"
  name       = "%s"
  org_id     = "%s"
}

resource "google_project_iam_binding" "dropped" {
  project = "${google_project.acceptance.project_id}"
  members = ["user:paddy@hashicorp.com"]
  role    = "roles/compute.instanceAdmin"
}
`, pid, name, org)
}
<commit_msg>Test adding multiple bindings at once.<commit_after>package google

import (
	"fmt"
	"sort"
	"testing"

	"github.com/hashicorp/terraform/helper/acctest"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
	"google.golang.org/api/cloudresourcemanager/v1"
)

// Test that an IAM binding can be applied to a project
func TestAccGoogleProjectIamBinding_basic(t *testing.T) {
	pid := "terraform-" + acctest.RandString(10)
	resource.Test(t, resource.TestCase{
		PreCheck:  func() { testAccPreCheck(t) },
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			// Create a new project
			{
				Config: testAccGoogleProject_create(pid, pname, org),
				Check: resource.ComposeTestCheckFunc(
					testAccGoogleProjectExistingPolicy(pid),
				),
			},
			// Apply an IAM binding
			{
				Config: testAccGoogleProjectAssociateBindingBasic(pid, pname, org),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckGoogleProjectIamBindingExists("google_project_iam_binding.acceptance", &cloudresourcemanager.Binding{
						Role:    "roles/compute.instanceAdmin",
						Members: []string{"user:admin@hashicorptest.com"},
					}, pid),
				),
			},
		},
	})
}

// Test that multiple IAM bindings can be applied to a project, one at a time
func TestAccGoogleProjectIamBinding_multiple(t *testing.T) {
	pid := "terraform-" + acctest.RandString(10)
	resource.Test(t, resource.TestCase{
		PreCheck:  func() { testAccPreCheck(t) },
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			// Create a new project
			{
				Config: testAccGoogleProject_create(pid, pname, org),
				Check: resource.ComposeTestCheckFunc(
					testAccGoogleProjectExistingPolicy(pid),
				),
			},
			// Apply an IAM binding
			{
				Config: testAccGoogleProjectAssociateBindingBasic(pid, pname, org),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckGoogleProjectIamBindingExists("google_project_iam_binding.acceptance", &cloudresourcemanager.Binding{
						Role:    "roles/compute.instanceAdmin",
						Members: []string{"user:admin@hashicorptest.com"},
					}, pid),
				),
			},
			// Apply another IAM binding
			{
				Config: testAccGoogleProjectAssociateBindingMultiple(pid, pname, org),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckGoogleProjectIamBindingExists("google_project_iam_binding.multiple", &cloudresourcemanager.Binding{
						Role:    "roles/viewer",
						Members: []string{"user:paddy@hashicorp.com"},
					}, pid),
					testAccCheckGoogleProjectIamBindingExists("google_project_iam_binding.multiple", &cloudresourcemanager.Binding{
						Role:    "roles/compute.instanceAdmin",
						Members: []string{"user:admin@hashicorptest.com"},
					}, pid),
				),
			},
		},
	})
}

// Test that multiple IAM bindings can be applied to a project all at once
func TestAccGoogleProjectIamBinding_basic(t *testing.T) {
	pid := "terraform-" + acctest.RandString(10)
	resource.Test(t, resource.TestCase{
		PreCheck:  func() { testAccPreCheck(t) },
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			// Create a new project
			{
				Config: testAccGoogleProject_create(pid, pname, org),
				Check: resource.ComposeTestCheckFunc(
					testAccGoogleProjectExistingPolicy(pid),
				),
			},
			// Apply an IAM binding
			{
				Config: testAccGoogleProjectAssociateBindingMultiple(pid, pname, org),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckGoogleProjectIamBindingExists("google_project_iam_binding.acceptance", &cloudresourcemanager.Binding{
						Role:    "roles/compute.instanceAdmin",
						Members: []string{"user:admin@hashicorptest.com"},
					}, pid),
					testAccCheckGoogleProjectIamBindingExists("google_project_iam_binding.multiple", &cloudresourcemanager.Binding{
						Role:    "roles/compute.instanceAdmin",
						Members: []string{"user:admin@hashicorptest.com"},
					}, pid),
				),
			},
		},
	})
}

// Test that an IAM binding can be updated once applied to a project
func TestAccGoogleProjectIamBinding_update(t *testing.T) {
	pid := "terraform-" + acctest.RandString(10)
	resource.Test(t, resource.TestCase{
		PreCheck:  func() { testAccPreCheck(t) },
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			// Create a new project
			{
				Config: testAccGoogleProject_create(pid, pname, org),
				Check: resource.ComposeTestCheckFunc(
					testAccGoogleProjectExistingPolicy(pid),
				),
			},
			// Apply an IAM binding
			{
				Config: testAccGoogleProjectAssociateBindingBasic(pid, pname, org),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckGoogleProjectIamBindingExists("google_project_iam_binding.acceptance", &cloudresourcemanager.Binding{
						Role:    "roles/compute.instanceAdmin",
						Members: []string{"user:admin@hashicorptest.com"},
					}, pid),
				),
			},
			// Apply an updated IAM binding
			{
				Config: testAccGoogleProjectAssociateBindingUpdated(pid, pname, org),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckGoogleProjectIamBindingExists("google_project_iam_binding.updated", &cloudresourcemanager.Binding{
						Role:    "roles/compute.instanceAdmin",
						Members: []string{"user:admin@hashicorptest.com", "user:paddy@hashicorp.com"},
					}, pid),
				),
			},
			// Drop the original member
			{
				Config: testAccGoogleProjectAssociateBindingDropMemberFromBasic(pid, pname, org),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckGoogleProjectIamBindingExists("google_project_iam_binding.dropped", &cloudresourcemanager.Binding{
						Role:    "roles/compute.instanceAdmin",
						Members: []string{"user:paddy@hashicorp.com"},
					}, pid),
				),
			},
		},
	})
}

// Test that an IAM binding can be removed from a project
func TestAccGoogleProjectIamBinding_remove(t *testing.T) {
	pid := "terraform-" + acctest.RandString(10)
	resource.Test(t, resource.TestCase{
		PreCheck:  func() { testAccPreCheck(t) },
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			// Create a new project
			{
				Config: testAccGoogleProject_create(pid, pname, org),
				Check: resource.ComposeTestCheckFunc(
					testAccGoogleProjectExistingPolicy(pid),
				),
			},
			// Apply multiple IAM bindings
			{
				Config: testAccGoogleProjectAssociateBindingMultiple(pid, pname, org),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckGoogleProjectIamBindingExists("google_project_iam_binding.multiple", &cloudresourcemanager.Binding{
						Role:    "roles/viewer",
						Members: []string{"user:paddy@hashicorp.com"},
					}, pid),
					testAccCheckGoogleProjectIamBindingExists("google_project_iam_binding.acceptance", &cloudresourcemanager.Binding{
						Role:    "roles/compute.instanceAdmin",
						Members: []string{"user:admin@hashicorptest.com"},
					}, pid),
				),
			},
			// Remove the bindings
			{
				Config: testAccGoogleProject_create(pid, pname, org),
				Check: resource.ComposeTestCheckFunc(
					testAccGoogleProjectExistingPolicy(pid),
				),
			},
		},
	})
}

func testAccCheckGoogleProjectIamBindingExists(key string, expected *cloudresourcemanager.Binding, pid string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		config := testAccProvider.Meta().(*Config)
		projectPolicy, err := getProjectIamPolicy(pid, config)
		if err != nil {
			return fmt.Errorf("Failed to retrieve IAM policy for project %q: %s", pid, err)
		}

		var result *cloudresourcemanager.Binding
		for _, binding := range projectPolicy.Bindings {
			if binding.Role == expected.Role {
				result = binding
				break
			}
		}
		if result == nil {
			return fmt.Errorf("IAM policy for project %q had no role %q", pid, expected.Role)
		}
		if len(result.Members) != len(expected.Members) {
			return fmt.Errorf("Got %v as members for role %q of project %q, expected %v", result.Members, expected.Role, pid, expected.Members)
		}
		sort.Strings(result.Members)
		sort.Strings(expected.Members)
		for pos, exp := range expected.Members {
			if result.Members[pos] != exp {
				return fmt.Errorf("Expected members for role %q of project %q to be %v, got %v", expected.Role, pid, expected.Members, result.Members)
			}
		}
		return nil
	}
}

func testAccGoogleProjectAssociateBindingBasic(pid, name, org string) string {
	return fmt.Sprintf(`
resource "google_project" "acceptance" {
  project_id = "%s"
  name       = "%s"
  org_id     = "%s"
}

resource "google_project_iam_binding" "acceptance" {
  project = "${google_project.acceptance.project_id}"
  members = ["user:admin@hashicorptest.com"]
  role    = "roles/compute.instanceAdmin"
}
`, pid, name, org)
}

func testAccGoogleProjectAssociateBindingMultiple(pid, name, org string) string {
	return fmt.Sprintf(`
resource "google_project" "acceptance" {
  project_id = "%s"
  name       = "%s"
  org_id     = "%s"
}

resource "google_project_iam_binding" "acceptance" {
  project = "${google_project.acceptance.project_id}"
  members = ["user:admin@hashicorptest.com"]
  role    = "roles/compute.instanceAdmin"
}

resource "google_project_iam_binding" "multiple" {
  project = "${google_project.acceptance.project_id}"
  members = ["user:paddy@hashicorp.com"]
  role    = "roles/viewer"
}
`, pid, name, org)
}

func testAccGoogleProjectAssociateBindingUpdated(pid, name, org string) string {
	return fmt.Sprintf(`
resource "google_project" "acceptance" {
  project_id = "%s"
  name       = "%s"
  org_id     = "%s"
}

resource "google_project_iam_binding" "acceptance" {
  project = "${google_project.acceptance.project_id}"
  members = ["user:admin@hashicorptest.com", "user:paddy@hashicorp.com"]
  role    = "roles/compute.instanceAdmin"
}
`, pid, name, org)
}

func testAccGoogleProjectAssociateBindingDropMemberFromBasic(pid, name, org string) string {
	return fmt.Sprintf(`
resource "google_project" "acceptance" {
  project_id = "%s"
  name       = "%s"
  org_id     = "%s"
}

resource "google_project_iam_binding" "dropped" {
  project = "${google_project.acceptance.project_id}"
  members = ["user:paddy@hashicorp.com"]
  role    = "roles/compute.instanceAdmin"
}
`, pid, name, org)
}
<|endoftext|>


<commit_before>package parser

import (
	"archive/zip"
	"encoding/csv"
	"errors"
	"io"
	"log"
	"net"
	"regexp"
	"strconv"

	"github.com/m-lab/annotation-service/loader"
)

const (
	ipNumColumnsGlite2        = 10
	locationNumColumnsGlite2  = 13
	gLite2Prefix              = "GeoLite2-City"
	geoLite2BlocksFilenameIP4 = "GeoLite2-City-Blocks-IPv4.csv"  // Filename of ipv4 blocks file
	geoLite2BlocksFilenameIP6 = "GeoLite2-City-Blocks-IPv6.csv"  // Filename of ipv6 blocks file
	geoLite2LocationsFilename = "GeoLite2-City-Locations-en.csv" // Filename of locations file
)

func LoadGeoLite2(zip *zip.Reader) (*GeoDataset, error) {
	locations, err := loader.FindFile(geoLite2LocationsFilename, zip)
	if err != nil {
		return nil, err
	}
	// geoidMap is just a temporary map that will be discarded once the blocks are parsed
	locationNode, geoidMap, err := LoadLocListGLite2(locations)
	locations.Close()
	if err != nil {
		return nil, err
	}

	blocks4, err := loader.FindFile(geoLite2BlocksFilenameIP4, zip)
	if err != nil {
		return nil, err
	}
	ipNodes4, err := LoadIPListGLite2(blocks4, geoidMap)
	blocks4.Close()
	if err != nil {
		return nil, err
	}
	blocks6, err := loader.FindFile(geoLite2BlocksFilenameIP6, zip)
	if err != nil {
		return nil, err
	}
	ipNodes6, err := LoadIPListGLite2(blocks6, geoidMap)
	blocks6.Close()
	if err != nil {
		return nil, err
	}
	return &GeoDataset{IP4Nodes: ipNodes4, IP6Nodes: ipNodes6, LocationNodes: locationNode}, nil
}

// Finds the smallest and largest net.IP from a CIDR range
// Example: "1.0.0.0/24" -> 1.0.0.0 , 1.0.0.255
func rangeCIDR(cidr string) (net.IP, net.IP, error) {
	ip, ipnet, err := net.ParseCIDR(cidr)
	if err != nil {
		return nil, nil, errors.New("Invalid CIDR IP range")
	}
	lowIp := make(net.IP, len(ip))
	copy(lowIp, ip)
	mask := ipnet.Mask
	for x, _ := range ip {
		if len(mask) == 4 {
			if x < 12 {
				ip[x] |= 0
			} else {
				ip[x] |= ^mask[x-12]
			}
		} else {
			ip[x] |= ^mask[x]
		}
	}
	return lowIp, ip, nil
}

// Create Location list for GLite2 databases
// TODO This code is a bit fragile.  Should probably parse the header and
// use that to guide the parsing of the rows.
func LoadLocListGLite2(reader io.Reader) ([]LocationNode, map[int]int, error) {
	idMap := make(map[int]int, mapMax)
	list := []LocationNode{}
	r := csv.NewReader(reader)
	// Skip the first line
	// TODO - we should parse the first line, instead of skipping it!!
	// This should set r.FieldsPerRecord.
	first, err := r.Read()
	if err == io.EOF {
		log.Println("Empty input data")
		return nil, nil, errors.New("Empty input data")
	}
	// TODO - this is a bit hacky.  May want to improve it.
	// Older geoLite2 have 13 columns, but since 2018/03, they have 14 columns.
	// This will print a log every time it loads a newer location file.
	if len(first) != locationNumColumnsGlite2 {
		log.Println("Incorrect number of columns in header, got: ", len(first), " wanted: ", locationNumColumnsGlite2)
		log.Println(first)
		if len(first) < locationNumColumnsGlite2 {
			return nil, nil, errors.New("Corrupted Data: wrong number of columns")
		}
	}
	// FieldsPerRecord is the expected column length
	// r.FieldsPerRecord = locationNumColumnsGlite2
	for {
		record, err := r.Read()
		if err != nil {
			if err == io.EOF {
				break
			}
			if len(record) != r.FieldsPerRecord {
				log.Println("Incorrect number of columns in IP list got: ", len(record), " wanted: ", r.FieldsPerRecord)
				log.Println(record)
				return nil, nil, errors.New("Corrupted Data: wrong number of columns")

			} else {
				log.Println(err, ": ", record)
				return nil, nil, err
			}
		}
		var lNode LocationNode
		lNode.GeonameID, err = strconv.Atoi(record[0])
		if err != nil {
			if len(record[0]) > 0 {
				log.Println("GeonameID should be a number ", record[0])
				return nil, nil, errors.New("Corrupted Data: GeonameID should be a number")
			}
		}
		lNode.ContinentCode, err = checkCaps(record[2], "Continent code")
		if err != nil {
			continue
		}
		lNode.CountryCode, err = checkCaps(record[4], "Country code")
		if err != nil {
			continue
		}
		match, _ := regexp.MatchString(`^[^0-9]*$`, record[5])
		if match {
			lNode.CountryName = record[5]
		} else {
			log.Println("Country name should be letters only : ", record[5])
			return nil, nil, errors.New("Corrupted Data: country name should be letters")
		}
		// TODO - should probably do some validation.
		lNode.RegionCode = record[6]
		lNode.RegionName = record[7]
		lNode.MetroCode, err = strconv.ParseInt(record[11], 10, 64)
		if err != nil {
			if len(record[11]) > 0 {
				log.Println("MetroCode should be a number")
				continue
			}
		}
		lNode.CityName = record[10]
		list = append(list, lNode)
		idMap[lNode.GeonameID] = len(list) - 1
	}
	return list, idMap, nil
}

// LoadIPListGLite2 creates a List of IPNodes from a GeoLite2 reader.
// TODO(gfr) Update to use recursion instead of stack.
func LoadIPListGLite2(reader io.Reader, idMap map[int]int) ([]IPNode, error) {
	list := []IPNode{}
	r := csv.NewReader(reader)
	stack := []IPNode{}
	// Skip first line
	_, err := r.Read()
	if err == io.EOF {
		log.Println("Empty input data")
		return nil, errors.New("Empty input data")
	}
	for {
		var newNode IPNode
		// Example:
		// GLite2 : record = [2a04:97c0::/29,2658434,2658434,0,0,47,8,100]
		record, err := r.Read()
		if err == io.EOF {
			break
		}
		err = checkNumColumns(record, ipNumColumnsGlite2)
		if err != nil {
			continue
		}
		lowIp, highIp, err := rangeCIDR(record[0])
		if err != nil {
			continue
		}
		newNode.IPAddressLow = lowIp
		newNode.IPAddressHigh = highIp
		// Look for GeoId within idMap and return index
		index, err := lookupGeoId(record[1], idMap)
		if err != nil {
			if backupIndex, err := lookupGeoId(record[2], idMap); err == nil {
				index = backupIndex
			} else {
				log.Println("Couldn't get a valid Geoname id!", record)
				//TODO: Add a prometheus metric here
			}

		}
		newNode.LocationIndex = index
		newNode.PostalCode = record[6]
		newNode.Latitude, err = stringToFloat(record[7], "Latitude")
		if err != nil {
			return nil, err
		}
		newNode.Longitude, err = stringToFloat(record[8], "Longitude")
		if err != nil {
			return nil, err
		}
		stack, list = handleStack(stack, list, newNode)
	}
	var pop IPNode
	pop, stack = stack[len(stack)-1], stack[:len(stack)-1]
	for ; len(stack) > 0; pop, stack = stack[len(stack)-1], stack[:len(stack)-1] {
		peek := stack[len(stack)-1]
		peek.IPAddressLow = PlusOne(pop.IPAddressHigh)
		list = append(list, peek)
	}
	return list, nil
}
<commit_msg>add log and error counter<commit_after>package parser

import (
	"archive/zip"
	"encoding/csv"
	"errors"
	"io"
	"log"
	"net"
	"regexp"
	"strconv"

	"github.com/m-lab/annotation-service/loader"
)

const (
	ipNumColumnsGlite2        = 10
	locationNumColumnsGlite2  = 13
	gLite2Prefix              = "GeoLite2-City"
	geoLite2BlocksFilenameIP4 = "GeoLite2-City-Blocks-IPv4.csv"  // Filename of ipv4 blocks file
	geoLite2BlocksFilenameIP6 = "GeoLite2-City-Blocks-IPv6.csv"  // Filename of ipv6 blocks file
	geoLite2LocationsFilename = "GeoLite2-City-Locations-en.csv" // Filename of locations file
)

func LoadGeoLite2(zip *zip.Reader) (*GeoDataset, error) {
	locations, err := loader.FindFile(geoLite2LocationsFilename, zip)
	if err != nil {
		return nil, err
	}
	// geoidMap is just a temporary map that will be discarded once the blocks are parsed
	locationNode, geoidMap, err := LoadLocListGLite2(locations)
	locations.Close()
	if err != nil {
		return nil, err
	}

	blocks4, err := loader.FindFile(geoLite2BlocksFilenameIP4, zip)
	if err != nil {
		return nil, err
	}
	ipNodes4, err := LoadIPListGLite2(blocks4, geoidMap)
	blocks4.Close()
	if err != nil {
		return nil, err
	}
	blocks6, err := loader.FindFile(geoLite2BlocksFilenameIP6, zip)
	if err != nil {
		return nil, err
	}
	ipNodes6, err := LoadIPListGLite2(blocks6, geoidMap)
	blocks6.Close()
	if err != nil {
		return nil, err
	}
	return &GeoDataset{IP4Nodes: ipNodes4, IP6Nodes: ipNodes6, LocationNodes: locationNode}, nil
}

// Finds the smallest and largest net.IP from a CIDR range
// Example: "1.0.0.0/24" -> 1.0.0.0 , 1.0.0.255
func rangeCIDR(cidr string) (net.IP, net.IP, error) {
	ip, ipnet, err := net.ParseCIDR(cidr)
	if err != nil {
		return nil, nil, errors.New("Invalid CIDR IP range")
	}
	lowIp := make(net.IP, len(ip))
	copy(lowIp, ip)
	mask := ipnet.Mask
	for x, _ := range ip {
		if len(mask) == 4 {
			if x < 12 {
				ip[x] |= 0
			} else {
				ip[x] |= ^mask[x-12]
			}
		} else {
			ip[x] |= ^mask[x]
		}
	}
	return lowIp, ip, nil
}

// Create Location list for GLite2 databases
// TODO This code is a bit fragile.  Should probably parse the header and
// use that to guide the parsing of the rows.
func LoadLocListGLite2(reader io.Reader) ([]LocationNode, map[int]int, error) {
	idMap := make(map[int]int, mapMax)
	list := []LocationNode{}
	r := csv.NewReader(reader)
	// Skip the first line
	// TODO - we should parse the first line, instead of skipping it!!
	// This should set r.FieldsPerRecord.
	first, err := r.Read()
	if err == io.EOF {
		log.Println("Empty input data")
		return nil, nil, errors.New("Empty input data")
	}
	// TODO - this is a bit hacky.  May want to improve it.
	// Older geoLite2 have 13 columns, but since 2018/03, they have 14 columns.
	// This will print a log every time it loads a newer location file.
	if len(first) != locationNumColumnsGlite2 {
		log.Println("Incorrect number of columns in header, got: ", len(first), " wanted: ", locationNumColumnsGlite2)
		log.Println(first)
		if len(first) < locationNumColumnsGlite2 {
			return nil, nil, errors.New("Corrupted Data: wrong number of columns")
		}
	}
	// FieldsPerRecord is the expected column length
	// r.FieldsPerRecord = locationNumColumnsGlite2
	errorCount := 0
	maxErrorCount := 50
	for {
		record, err := r.Read()
		if err != nil {
			if err == io.EOF {
				break
			}
			if len(record) != r.FieldsPerRecord {
				log.Println("Incorrect number of columns in IP list got: ", len(record), " wanted: ", r.FieldsPerRecord)
				log.Println(record)
				return nil, nil, errors.New("Corrupted Data: wrong number of columns")

			} else {
				log.Println(err, ": ", record)
				return nil, nil, err
			}
		}
		var lNode LocationNode
		lNode.GeonameID, err = strconv.Atoi(record[0])
		if err != nil {
			if len(record[0]) > 0 {
				log.Println("GeonameID should be a number ", record[0])
				return nil, nil, errors.New("Corrupted Data: GeonameID should be a number")
			}
		}
		lNode.ContinentCode, err = checkCaps(record[2], "Continent code")
		if err != nil {
			log.Println(err)
			errorCount += 1
			if errorCount > maxErrorCount {
				return nil, nil, errors.New("Too many errors during loading the dataset location list")
			}
			continue
		}
		lNode.CountryCode, err = checkCaps(record[4], "Country code")
		if err != nil {
			log.Println(err)
			errorCount += 1
			if errorCount > maxErrorCount {
				return nil, nil, errors.New("Too many errors during loading the dataset location list")
			}
			continue
		}
		match, _ := regexp.MatchString(`^[^0-9]*$`, record[5])
		if match {
			lNode.CountryName = record[5]
		} else {
			log.Println("Country name should be letters only : ", record[5])
			return nil, nil, errors.New("Corrupted Data: country name should be letters")
		}
		// TODO - should probably do some validation.
		lNode.RegionCode = record[6]
		lNode.RegionName = record[7]
		lNode.MetroCode, err = strconv.ParseInt(record[11], 10, 64)
		if err != nil {
			if len(record[11]) > 0 {
				log.Println("MetroCode should be a number")
				errorCount += 1
				if errorCount > maxErrorCount {
					return nil, nil, errors.New("Too many errors during loading the dataset location list")
				}
				continue
			}
		}
		lNode.CityName = record[10]
		list = append(list, lNode)
		idMap[lNode.GeonameID] = len(list) - 1
	}
	return list, idMap, nil
}

// LoadIPListGLite2 creates a List of IPNodes from a GeoLite2 reader.
// TODO(gfr) Update to use recursion instead of stack.
func LoadIPListGLite2(reader io.Reader, idMap map[int]int) ([]IPNode, error) {
	list := []IPNode{}
	r := csv.NewReader(reader)
	stack := []IPNode{}
	// Skip first line
	_, err := r.Read()
	if err == io.EOF {
		log.Println("Empty input data")
		return nil, errors.New("Empty input data")
	}
	errorCount := 0
	maxErrorCount := 50
	for {
		var newNode IPNode
		// Example:
		// GLite2 : record = [2a04:97c0::/29,2658434,2658434,0,0,47,8,100]
		record, err := r.Read()
		if err == io.EOF {
			break
		}
		err = checkNumColumns(record, ipNumColumnsGlite2)
		if err != nil {
			log.Println(err)
			errorCount += 1
			if errorCount > maxErrorCount {
				return nil, errors.New("Too many errors during loading the dataset IP list.")
			}
			continue
		}
		lowIp, highIp, err := rangeCIDR(record[0])
		if err != nil {
			log.Println(err)
			errorCount += 1
			if errorCount > maxErrorCount {
				return nil, errors.New("Too many errors during loading the dataset IP list")
			}
			continue
		}
		newNode.IPAddressLow = lowIp
		newNode.IPAddressHigh = highIp
		// Look for GeoId within idMap and return index
		index, err := lookupGeoId(record[1], idMap)
		if err != nil {
			if backupIndex, err := lookupGeoId(record[2], idMap); err == nil {
				index = backupIndex
			} else {
				log.Println("Couldn't get a valid Geoname id!", record)
				//TODO: Add a prometheus metric here
			}

		}
		newNode.LocationIndex = index
		newNode.PostalCode = record[6]
		newNode.Latitude, err = stringToFloat(record[7], "Latitude")
		if err != nil {
			log.Println(err)
			errorCount += 1
			if errorCount > maxErrorCount {
				return nil, errors.New("Too many errors during loading the dataset IP list")
			}
			continue
		}
		newNode.Longitude, err = stringToFloat(record[8], "Longitude")
		if err != nil {
			log.Println(err)
			errorCount += 1
			if errorCount > maxErrorCount {
				return nil, errors.New("Too many errors during loading the dataset IP list")
			}
			continue
		}
		stack, list = handleStack(stack, list, newNode)
	}
	var pop IPNode
	pop, stack = stack[len(stack)-1], stack[:len(stack)-1]
	for ; len(stack) > 0; pop, stack = stack[len(stack)-1], stack[:len(stack)-1] {
		peek := stack[len(stack)-1]
		peek.IPAddressLow = PlusOne(pop.IPAddressHigh)
		list = append(list, peek)
	}
	return list, nil
}
<|endoftext|>