<commit_before>// Copyright (c) 2018-2019, The Decred developers
// Copyright (c) 2017, Jonathan Chappelow
// See LICENSE for details.

package notification

import (
	"strings"
	"sync"
	"time"

	"github.com/decred/dcrd/chaincfg/chainhash"
	"github.com/decred/dcrd/dcrjson/v2"
	"github.com/decred/dcrd/dcrutil"
	"github.com/decred/dcrd/rpcclient/v2"
	"github.com/decred/dcrd/wire"
	"github.com/decred/dcrdata/v4/api/insight"
	"github.com/decred/dcrdata/v4/rpcutils"
	"github.com/decred/dcrdata/v4/txhelpers"
	"github.com/decred/dcrwallet/wallet/udb"
)

var nodeClient *rpcclient.Client

// RegisterNodeNtfnHandlers registers with dcrd to receive new block,
// transaction and winning ticket notifications.
func RegisterNodeNtfnHandlers(dcrdClient *rpcclient.Client) *ContextualError {
	// Set the node client for use by signalReorg to determine the common
	// ancestor of two chains.
	nodeClient = dcrdClient

	// Register for block connection and chain reorg notifications.
	var err error
	if err = dcrdClient.NotifyBlocks(); err != nil {
		return newContextualError("block notification "+
			"registration failed", err)
	}

	// Register for tx accepted into mempool ntfns
	if err = dcrdClient.NotifyNewTransactions(true); err != nil {
		return newContextualError("new transaction verbose notification registration failed", err)
	}

	// For OnNewTickets
	//  Commented since there is a bug in rpcclient/notify.go
	// dcrdClient.NotifyNewTickets()

	if err = dcrdClient.NotifyWinningTickets(); err != nil {
		return newContextualError("winning ticket "+
			"notification registration failed", err)
	}

	// Register a Tx filter for addresses (receiving).  The filter applies to
	// OnRelevantTxAccepted.
	// TODO: register outpoints (third argument).
	// if len(addresses) > 0 {
	// 	if err = dcrdClient.LoadTxFilter(true, addresses, nil); err != nil {
	// 		return newContextualError("load tx filter failed", err)
	// 	}
	// }

	return nil
}

type blockHashHeight struct {
	hash     chainhash.Hash
	height   int64
	prevHash chainhash.Hash
}

type collectionQueue struct {
	q            chan *blockHashHeight
	syncHandlers []func(hash *chainhash.Hash) error
	prevHash     chainhash.Hash
	prevHeight   int64
}

// NewCollectionQueue creates a new collectionQueue with an unbuffered channel
// to facilitate synchronous processing. Run (*collectionQueue).ProcessBlocks as
// a goroutine to begin processing blocks sent to the channel, or call
// processBlock for each new block.
func NewCollectionQueue() *collectionQueue {
	return &collectionQueue{
		q: make(chan *blockHashHeight),
	}
}

var blockQueue = NewCollectionQueue()

// SetSynchronousHandlers sets the slice of synchronous new block handler
// functions, which are called in the order they occur in the slice.
func (q *collectionQueue) SetSynchronousHandlers(syncHandlers []func(hash *chainhash.Hash) error) {
	q.syncHandlers = syncHandlers
}

// SetPreviousBlock modifies the height and hash of the best block. This data is
// required to avoid connecting new blocks that are not next in the chain. It is
// only necessary to call SetPreviousBlock if blocks are connected or
// disconnected by a mechanism other than (*collectionQueue).processBlock, which
// keeps this data up-to-date. For example, signalReorg will use
// SetPreviousBlock after the reorg is complete.
func (q *collectionQueue) SetPreviousBlock(prevHash chainhash.Hash, prevHeight int64) {
	q.prevHash = prevHash
	q.prevHeight = prevHeight
}

// ProcessBlocks receives *blockHashHeights on the channel collectionQueue.q,
// and calls processBlock synchronously for the received data. This should be
// run as a goroutine, or use processBlock directly.
func (q *collectionQueue) ProcessBlocks() {
	// Process queued blocks one at a time.
	for bh := range q.q {
		q.processBlock(bh)
	}
}

// processBlock calls each synchronous new block handler with the given
// blockHashHeight, and then signals to the monitors that a new block was mined.
func (q *collectionQueue) processBlock(bh *blockHashHeight) {
	hash := bh.hash
	height := bh.height

	// Ensure that the received block (bh.hash, bh.height) connects to the
	// previously connected block (q.prevHash, q.prevHeight).
	if bh.prevHash != q.prevHash {
		log.Infof("Received block at %d (%v) does not connect to %d (%v). "+
			"This is normal before reorganization.",
			height, hash, q.prevHeight, q.prevHash)
		return
	}

	start := time.Now()

	// Run synchronous block connected handlers in order.
	for _, h := range q.syncHandlers {
		if err := h(&hash); err != nil {
			log.Errorf("synchronous handler failed: %v", err)
			return
		}
	}

	// Record this block as the best block connected by the collectionQueue.
	q.SetPreviousBlock(hash, height)

	log.Debugf("Synchronous handlers of collectionQueue.processBlock() completed in %v", time.Since(start))

	// Signal to mempool monitors that a block was mined.
	select {
	case NtfnChans.NewTxChan <- nil:
	default:
	}

	// API status update handler
	select {
	case NtfnChans.UpdateStatusNodeHeight <- uint32(height):
	default:
	}
}

// ReorgData describes the old and new chain tips involved in a reorganization.
type ReorgData struct {
	OldChainHead   chainhash.Hash
	OldChainHeight int32
	NewChainHead   chainhash.Hash
	NewChainHeight int32
	WG             *sync.WaitGroup
}

// signalReorg takes the basic reorganization data from dcrd, determines the two
// chains and their common ancestor, and signals the reorg to each packages'
// reorganization handler. Lastly, the collectionQueue's best block data is
// updated so that it will successfully accept new blocks building on the new
// chain tip.
func signalReorg(d ReorgData) {
	if nodeClient == nil {
		log.Errorf("The daemon RPC client for signalReorg is not configured!")
		return
	}

	// Determine the common ancestor of the two chains, and get the full
	// list of blocks in each chain back to but not including the common
	// ancestor.
	ancestor, newChain, oldChain, err := rpcutils.CommonAncestor(nodeClient,
		d.NewChainHead, d.OldChainHead)
	if err != nil {
		log.Errorf("Failed to determine common ancestor. Aborting reorg.")
		return
	}

	fullData := &txhelpers.ReorgData{
		CommonAncestor: *ancestor,
		NewChain:       newChain,
		NewChainHead:   d.NewChainHead,
		NewChainHeight: d.NewChainHeight,
		OldChain:       oldChain,
		OldChainHead:   d.OldChainHead,
		OldChainHeight: d.OldChainHeight,
		WG:             d.WG,
	}

	// Send reorg data to stakedb's monitor.
	d.WG.Add(1)
	select {
	case NtfnChans.ReorgChanStakeDB <- fullData:
	default:
		d.WG.Done()
	}
	d.WG.Wait()

	// Send reorg data to dcrsqlite's monitor.
	d.WG.Add(1)
	select {
	case NtfnChans.ReorgChanWiredDB <- fullData:
	default:
		d.WG.Done()
	}

	// Send reorg data to blockdata's monitor.
	d.WG.Add(1)
	select {
	case NtfnChans.ReorgChanBlockData <- fullData:
	default:
		d.WG.Done()
	}

	// Send reorg data to ChainDB's monitor.
	d.WG.Add(1)
	select {
	case NtfnChans.ReorgChanDcrpgDB <- fullData:
	default:
		d.WG.Done()
	}
	d.WG.Wait()

	// Update prevHash and prevHeight in collectionQueue.
	blockQueue.SetPreviousBlock(d.NewChainHead, int64(d.NewChainHeight))
}

// anyQueue is a buffered channel used queueing the handling of different types
// of event notifications, while keeping the order in which they were received
// from the daemon. See superQueue.
var anyQueue = make(chan interface{}, 1e7)

// superQueue manages the event notification queue, keeping the events in the
// same order they were received by the rpcclient.NotificationHandlers, and
// sending them to the appropriate handlers. This should be run as a goroutine.
func superQueue() {
	for e := range anyQueue {
		// Do not allow new blocks to process while running reorg. Only allow
		// them to be processed after this reorg completes.
		switch et := e.(type) {
		case *blockHashHeight:
			// Process the new block.
			log.Infof("superQueue: Processing new block %v (height %d).", et.hash, et.height)
			blockQueue.processBlock(et)
		case ReorgData:
			// Process the reorg.
			log.Infof("superQueue: Processing reorganization from %s (height %d) to %s (height %d).",
				et.OldChainHead.String(), et.OldChainHeight,
				et.NewChainHead.String(), et.NewChainHeight)
			signalReorg(et)
		default:
			log.Errorf("Unknown event data type: %T", et)
		}
	}
}

// MakeNodeNtfnHandlers defines the dcrd notification handlers.
func MakeNodeNtfnHandlers() (*rpcclient.NotificationHandlers, *collectionQueue) {
	go superQueue()
	// ReorgSignaler must be started after creating the RPC client.
	return &rpcclient.NotificationHandlers{
		// TODO: considering using txns [][]byte to save on downstream RPCs.
		OnBlockConnected: func(blockHeaderSerialized []byte, _ [][]byte) {
			blockHeader := new(wire.BlockHeader)
			err := blockHeader.FromBytes(blockHeaderSerialized)
			if err != nil {
				log.Error("Failed to deserialize blockHeader in new block notification: "+
					"%v", err)
				return
			}
			height := int32(blockHeader.Height)
			hash := blockHeader.BlockHash()
			prevHash := blockHeader.PrevBlock // to ensure this is the next block

			log.Tracef("OnBlockConnected: %d / %v (previous: %v)", height, hash, prevHash)

			// Queue this block.
			anyQueue <- &blockHashHeight{
				hash:     hash,
				height:   int64(height),
				prevHash: prevHash,
			}
		},
		OnBlockDisconnected: func(blockHeaderSerialized []byte) {
			blockHeader := new(wire.BlockHeader)
			err := blockHeader.FromBytes(blockHeaderSerialized)
			if err != nil {
				log.Error("Failed to deserialize blockHeader in block disconnect notification: "+
					"%v", err)
				return
			}
			height := int32(blockHeader.Height)
			hash := blockHeader.BlockHash()

			log.Tracef("OnBlockDisconnected: %d / %v", height, hash)
		},
		OnReorganization: func(oldHash *chainhash.Hash, oldHeight int32,
			newHash *chainhash.Hash, newHeight int32) {
			log.Tracef("OnReorganization: %d / %s --> %d / %s",
				oldHeight, oldHash, newHeight, newHash)

			// Queue sending the details of this reorganization to each
			// package's reorg handler. This is not blocking.
			anyQueue <- ReorgData{
				OldChainHead:   *oldHash,
				OldChainHeight: oldHeight,
				NewChainHead:   *newHash,
				NewChainHeight: newHeight,
				WG:             new(sync.WaitGroup),
			}
		},

		// block height and hash are unused by OnWinningTickets.
		OnWinningTickets: func(_ *chainhash.Hash, _ int64, tickets []*chainhash.Hash) {
			var txstr []string
			for _, t := range tickets {
				txstr = append(txstr, t.String())
			}
			log.Tracef("Winning tickets: %v", strings.Join(txstr, ", "))
		},

		// Maturing tickets (maturing or mined?  clarify before use!).
		OnNewTickets: func(_ *chainhash.Hash, _ int64, _ int64, tickets []*chainhash.Hash) {
			for _, tick := range tickets {
				log.Tracef("Mined new ticket: %v", tick.String())
			}
		},

		// OnRelevantTxAccepted is invoked when a transaction containing a
		// registered address is inserted into mempool.
		OnRelevantTxAccepted: func(transaction []byte) {
			rec, err := udb.NewTxRecord(transaction, time.Now())
			if err != nil {
				return
			}
			tx := dcrutil.NewTx(&rec.MsgTx)
			txHash := rec.Hash
			select {
			case NtfnChans.RelevantTxMempoolChan <- tx:
				log.Debugf("Detected transaction %v in mempool containing registered address.",
					txHash.String())
			default:
			}
		},

		// OnTxAcceptedVerbose is invoked same as OnTxAccepted but is used here
		// for the mempool monitors to avoid an extra call to dcrd for
		// the tx details
		OnTxAcceptedVerbose: func(txDetails *dcrjson.TxRawResult) {
			// Current UNIX time to assign the new transaction.
			now := time.Now().Unix()

			select {
			case NtfnChans.InsightNewTxChan <- &insight.NewTx{
				Hex:   txDetails.Hex,
				Vouts: txDetails.Vout,
			}:
			default:
				if NtfnChans.InsightNewTxChan != nil {
					log.Warn("InsightNewTxChan buffer full!")
				}
			}

			// Patch txDetails.Time, which is 0, with current time.
			txDetails.Time = now

			select {
			case NtfnChans.NewTxChan <- txDetails:
			default:
				log.Warn("NewTxChan buffer full!")
			}
		},
	}, blockQueue
}
<commit_msg>notification: lower anyQueue to 16 chans<commit_after>// Copyright (c) 2018-2019, The Decred developers
// Copyright (c) 2017, Jonathan Chappelow
// See LICENSE for details.

package notification

import (
	"strings"
	"sync"
	"time"

	"github.com/decred/dcrd/chaincfg/chainhash"
	"github.com/decred/dcrd/dcrjson/v2"
	"github.com/decred/dcrd/dcrutil"
	"github.com/decred/dcrd/rpcclient/v2"
	"github.com/decred/dcrd/wire"
	"github.com/decred/dcrdata/v4/api/insight"
	"github.com/decred/dcrdata/v4/rpcutils"
	"github.com/decred/dcrdata/v4/txhelpers"
	"github.com/decred/dcrwallet/wallet/udb"
)

var nodeClient *rpcclient.Client

// RegisterNodeNtfnHandlers registers with dcrd to receive new block,
// transaction and winning ticket notifications.
func RegisterNodeNtfnHandlers(dcrdClient *rpcclient.Client) *ContextualError {
	// Set the node client for use by signalReorg to determine the common
	// ancestor of two chains.
	nodeClient = dcrdClient

	// Register for block connection and chain reorg notifications.
	var err error
	if err = dcrdClient.NotifyBlocks(); err != nil {
		return newContextualError("block notification "+
			"registration failed", err)
	}

	// Register for tx accepted into mempool ntfns
	if err = dcrdClient.NotifyNewTransactions(true); err != nil {
		return newContextualError("new transaction verbose notification registration failed", err)
	}

	// For OnNewTickets
	//  Commented since there is a bug in rpcclient/notify.go
	// dcrdClient.NotifyNewTickets()

	if err = dcrdClient.NotifyWinningTickets(); err != nil {
		return newContextualError("winning ticket "+
			"notification registration failed", err)
	}

	// Register a Tx filter for addresses (receiving).  The filter applies to
	// OnRelevantTxAccepted.
	// TODO: register outpoints (third argument).
	// if len(addresses) > 0 {
	// 	if err = dcrdClient.LoadTxFilter(true, addresses, nil); err != nil {
	// 		return newContextualError("load tx filter failed", err)
	// 	}
	// }

	return nil
}

type blockHashHeight struct {
	hash     chainhash.Hash
	height   int64
	prevHash chainhash.Hash
}

type collectionQueue struct {
	q            chan *blockHashHeight
	syncHandlers []func(hash *chainhash.Hash) error
	prevHash     chainhash.Hash
	prevHeight   int64
}

// NewCollectionQueue creates a new collectionQueue with an unbuffered channel
// to facilitate synchronous processing. Run (*collectionQueue).ProcessBlocks as
// a goroutine to begin processing blocks sent to the channel, or call
// processBlock for each new block.
func NewCollectionQueue() *collectionQueue {
	return &collectionQueue{
		q: make(chan *blockHashHeight),
	}
}

var blockQueue = NewCollectionQueue()

// SetSynchronousHandlers sets the slice of synchronous new block handler
// functions, which are called in the order they occur in the slice.
func (q *collectionQueue) SetSynchronousHandlers(syncHandlers []func(hash *chainhash.Hash) error) {
	q.syncHandlers = syncHandlers
}

// SetPreviousBlock modifies the height and hash of the best block. This data is
// required to avoid connecting new blocks that are not next in the chain. It is
// only necessary to call SetPreviousBlock if blocks are connected or
// disconnected by a mechanism other than (*collectionQueue).processBlock, which
// keeps this data up-to-date. For example, signalReorg will use
// SetPreviousBlock after the reorg is complete.
func (q *collectionQueue) SetPreviousBlock(prevHash chainhash.Hash, prevHeight int64) {
	q.prevHash = prevHash
	q.prevHeight = prevHeight
}

// ProcessBlocks receives *blockHashHeights on the channel collectionQueue.q,
// and calls processBlock synchronously for the received data. This should be
// run as a goroutine, or use processBlock directly.
func (q *collectionQueue) ProcessBlocks() {
	// Process queued blocks one at a time.
	for bh := range q.q {
		q.processBlock(bh)
	}
}

// processBlock calls each synchronous new block handler with the given
// blockHashHeight, and then signals to the monitors that a new block was mined.
func (q *collectionQueue) processBlock(bh *blockHashHeight) {
	hash := bh.hash
	height := bh.height

	// Ensure that the received block (bh.hash, bh.height) connects to the
	// previously connected block (q.prevHash, q.prevHeight).
	if bh.prevHash != q.prevHash {
		log.Infof("Received block at %d (%v) does not connect to %d (%v). "+
			"This is normal before reorganization.",
			height, hash, q.prevHeight, q.prevHash)
		return
	}

	start := time.Now()

	// Run synchronous block connected handlers in order.
	for _, h := range q.syncHandlers {
		if err := h(&hash); err != nil {
			log.Errorf("synchronous handler failed: %v", err)
			return
		}
	}

	// Record this block as the best block connected by the collectionQueue.
	q.SetPreviousBlock(hash, height)

	log.Debugf("Synchronous handlers of collectionQueue.processBlock() completed in %v", time.Since(start))

	// Signal to mempool monitors that a block was mined.
	select {
	case NtfnChans.NewTxChan <- nil:
	default:
	}

	// API status update handler
	select {
	case NtfnChans.UpdateStatusNodeHeight <- uint32(height):
	default:
	}
}

// ReorgData describes the old and new chain tips involved in a reorganization.
type ReorgData struct {
	OldChainHead   chainhash.Hash
	OldChainHeight int32
	NewChainHead   chainhash.Hash
	NewChainHeight int32
	WG             *sync.WaitGroup
}

// signalReorg takes the basic reorganization data from dcrd, determines the two
// chains and their common ancestor, and signals the reorg to each packages'
// reorganization handler. Lastly, the collectionQueue's best block data is
// updated so that it will successfully accept new blocks building on the new
// chain tip.
func signalReorg(d ReorgData) {
	if nodeClient == nil {
		log.Errorf("The daemon RPC client for signalReorg is not configured!")
		return
	}

	// Determine the common ancestor of the two chains, and get the full
	// list of blocks in each chain back to but not including the common
	// ancestor.
	ancestor, newChain, oldChain, err := rpcutils.CommonAncestor(nodeClient,
		d.NewChainHead, d.OldChainHead)
	if err != nil {
		log.Errorf("Failed to determine common ancestor. Aborting reorg.")
		return
	}

	fullData := &txhelpers.ReorgData{
		CommonAncestor: *ancestor,
		NewChain:       newChain,
		NewChainHead:   d.NewChainHead,
		NewChainHeight: d.NewChainHeight,
		OldChain:       oldChain,
		OldChainHead:   d.OldChainHead,
		OldChainHeight: d.OldChainHeight,
		WG:             d.WG,
	}

	// Send reorg data to stakedb's monitor.
	d.WG.Add(1)
	select {
	case NtfnChans.ReorgChanStakeDB <- fullData:
	default:
		d.WG.Done()
	}
	d.WG.Wait()

	// Send reorg data to dcrsqlite's monitor.
	d.WG.Add(1)
	select {
	case NtfnChans.ReorgChanWiredDB <- fullData:
	default:
		d.WG.Done()
	}

	// Send reorg data to blockdata's monitor.
	d.WG.Add(1)
	select {
	case NtfnChans.ReorgChanBlockData <- fullData:
	default:
		d.WG.Done()
	}

	// Send reorg data to ChainDB's monitor.
	d.WG.Add(1)
	select {
	case NtfnChans.ReorgChanDcrpgDB <- fullData:
	default:
		d.WG.Done()
	}
	d.WG.Wait()

	// Update prevHash and prevHeight in collectionQueue.
	blockQueue.SetPreviousBlock(d.NewChainHead, int64(d.NewChainHeight))
}

// anyQueue is a buffered channel used queueing the handling of different types
// of event notifications, while keeping the order in which they were received
// from the daemon. See superQueue.
var anyQueue = make(chan interface{}, 16)

// superQueue manages the event notification queue, keeping the events in the
// same order they were received by the rpcclient.NotificationHandlers, and
// sending them to the appropriate handlers. This should be run as a goroutine.
func superQueue() {
	for e := range anyQueue {
		// Do not allow new blocks to process while running reorg. Only allow
		// them to be processed after this reorg completes.
		switch et := e.(type) {
		case *blockHashHeight:
			// Process the new block.
			log.Infof("superQueue: Processing new block %v (height %d).", et.hash, et.height)
			blockQueue.processBlock(et)
		case ReorgData:
			// Process the reorg.
			log.Infof("superQueue: Processing reorganization from %s (height %d) to %s (height %d).",
				et.OldChainHead.String(), et.OldChainHeight,
				et.NewChainHead.String(), et.NewChainHeight)
			signalReorg(et)
		default:
			log.Errorf("Unknown event data type: %T", et)
		}
	}
}

// MakeNodeNtfnHandlers defines the dcrd notification handlers.
func MakeNodeNtfnHandlers() (*rpcclient.NotificationHandlers, *collectionQueue) {
	go superQueue()
	// ReorgSignaler must be started after creating the RPC client.
	return &rpcclient.NotificationHandlers{
		// TODO: considering using txns [][]byte to save on downstream RPCs.
		OnBlockConnected: func(blockHeaderSerialized []byte, _ [][]byte) {
			blockHeader := new(wire.BlockHeader)
			err := blockHeader.FromBytes(blockHeaderSerialized)
			if err != nil {
				log.Error("Failed to deserialize blockHeader in new block notification: "+
					"%v", err)
				return
			}
			height := int32(blockHeader.Height)
			hash := blockHeader.BlockHash()
			prevHash := blockHeader.PrevBlock // to ensure this is the next block

			log.Tracef("OnBlockConnected: %d / %v (previous: %v)", height, hash, prevHash)

			// Queue this block.
			anyQueue <- &blockHashHeight{
				hash:     hash,
				height:   int64(height),
				prevHash: prevHash,
			}
		},
		OnBlockDisconnected: func(blockHeaderSerialized []byte) {
			blockHeader := new(wire.BlockHeader)
			err := blockHeader.FromBytes(blockHeaderSerialized)
			if err != nil {
				log.Error("Failed to deserialize blockHeader in block disconnect notification: "+
					"%v", err)
				return
			}
			height := int32(blockHeader.Height)
			hash := blockHeader.BlockHash()

			log.Tracef("OnBlockDisconnected: %d / %v", height, hash)
		},
		OnReorganization: func(oldHash *chainhash.Hash, oldHeight int32,
			newHash *chainhash.Hash, newHeight int32) {
			log.Tracef("OnReorganization: %d / %s --> %d / %s",
				oldHeight, oldHash, newHeight, newHash)

			// Queue sending the details of this reorganization to each
			// package's reorg handler. This is not blocking.
			anyQueue <- ReorgData{
				OldChainHead:   *oldHash,
				OldChainHeight: oldHeight,
				NewChainHead:   *newHash,
				NewChainHeight: newHeight,
				WG:             new(sync.WaitGroup),
			}
		},

		// block height and hash are unused by OnWinningTickets.
		OnWinningTickets: func(_ *chainhash.Hash, _ int64, tickets []*chainhash.Hash) {
			var txstr []string
			for _, t := range tickets {
				txstr = append(txstr, t.String())
			}
			log.Tracef("Winning tickets: %v", strings.Join(txstr, ", "))
		},

		// Maturing tickets (maturing or mined?  clarify before use!).
		OnNewTickets: func(_ *chainhash.Hash, _ int64, _ int64, tickets []*chainhash.Hash) {
			for _, tick := range tickets {
				log.Tracef("Mined new ticket: %v", tick.String())
			}
		},

		// OnRelevantTxAccepted is invoked when a transaction containing a
		// registered address is inserted into mempool.
		OnRelevantTxAccepted: func(transaction []byte) {
			rec, err := udb.NewTxRecord(transaction, time.Now())
			if err != nil {
				return
			}
			tx := dcrutil.NewTx(&rec.MsgTx)
			txHash := rec.Hash
			select {
			case NtfnChans.RelevantTxMempoolChan <- tx:
				log.Debugf("Detected transaction %v in mempool containing registered address.",
					txHash.String())
			default:
			}
		},

		// OnTxAcceptedVerbose is invoked same as OnTxAccepted but is used here
		// for the mempool monitors to avoid an extra call to dcrd for
		// the tx details
		OnTxAcceptedVerbose: func(txDetails *dcrjson.TxRawResult) {
			// Current UNIX time to assign the new transaction.
			now := time.Now().Unix()

			select {
			case NtfnChans.InsightNewTxChan <- &insight.NewTx{
				Hex:   txDetails.Hex,
				Vouts: txDetails.Vout,
			}:
			default:
				if NtfnChans.InsightNewTxChan != nil {
					log.Warn("InsightNewTxChan buffer full!")
				}
			}

			// Patch txDetails.Time, which is 0, with current time.
			txDetails.Time = now

			select {
			case NtfnChans.NewTxChan <- txDetails:
			default:
				log.Warn("NewTxChan buffer full!")
			}
		},
	}, blockQueue
}
<|endoftext|>


<commit_before>// Copyright 2015 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package orm

import (
	"bytes"
	"database/sql"
	"errors"

	"github.com/issue9/orm/fetch"
)

const (
	and = iota
	or
)

// 用于产生条件语句。
type Where struct {
	e     engine
	table string
	cond  *bytes.Buffer
	args  []interface{}
}

func newWhere(engine engine) *Where {
	return &Where{
		e:    engine,
		cond: new(bytes.Buffer),
		args: []interface{}{},
	}
}

func (w *Where) where(op int, cond string, args ...interface{}) *Where {
	switch {
	case w.cond.Len() == 0:
		w.cond.WriteString(" WHERE(")
	case op == and:
		w.cond.WriteString(" AND(")
	case op == or:
		w.cond.WriteString(" OR(")
	default:
		panic("Where:where:错误的参数op")
	}
	w.cond.WriteString(cond)
	w.cond.WriteByte(')')
	w.args = append(w.args, args...)

	return w
}

// 将之后的语句以and的形式与当前的语句进行连接
func (w *Where) And(cond string, args ...interface{}) *Where {
	return w.where(and, cond, args...)
}

// 将之后的语句以or的形式与当前的语句进行连接
func (w *Where) Or(cond string, args ...interface{}) *Where {
	return w.where(or, cond, args...)
}

// 指定表名。
func (w *Where) Table(tableName string) *Where {
	w.table = tableName
	return w
}

// 将符合当前条件的所有记录删除。
// replace，是否需将对语句的占位符进行替换。
func (w *Where) Delete(replace bool) error {
	if len(w.table) == 0 {
		return errors.New("Where:Delete:未指定表名")
	}

	sql := bytes.NewBufferString("DELETE FROM ")
	w.e.Dialect().Quote(sql, w.table)
	sql.WriteString(w.cond.String())
	_, err := w.e.Exec(replace, sql.String(), w.args...)
	return err
}

// 更新符合当前条件的所有记录。
// replace，是否需将对语句的占位符进行替换。
func (w *Where) Update(replace bool, data map[string]interface{}) error {
	if len(w.table) == 0 {
		return errors.New("Where.Update:未指定表名")
	}

	if len(data) == 0 {
		return errors.New("Where.Update:未指定需要更新的数据")
	}

	sql := bytes.NewBufferString("UPDATE ")
	w.e.Dialect().Quote(sql, w.table)
	vals := make([]interface{}, 0, len(data)+len(w.args))
	sql.WriteString(" SET ")
	for k, v := range data {
		w.e.Dialect().Quote(sql, k)
		sql.WriteString("=?,")
		vals = append(vals, v)
	}
	sql.Truncate(sql.Len() - 1) // 去掉最后一个逗号

	sql.WriteString(w.cond.String())
	_, err := w.e.Exec(replace, sql.String(), append(vals, w.args...)...)
	return err
}

// 将符合当前条件的所有记录依次写入objs中。
// replace，是否需将对语句的占位符进行替换。
func (w *Where) Select(replace bool, objs interface{}) error {
	rows, err := w.buildSelectSQL(replace)
	if err != nil {
		return err
	}
	return fetch.Obj(objs, rows)
}

// 返回符合当前条件的所有记录。
// replace，是否需将对语句的占位符进行替换。
func (w *Where) SelectMap(replace bool, cols ...string) ([]map[string]interface{}, error) {
	rows, err := w.buildSelectSQL(replace, cols...)
	if err != nil {
		return nil, err
	}
	return fetch.Map(false, rows)
}

func (w *Where) buildSelectSQL(replace bool, cols ...string) (*sql.Rows, error) {
	if len(w.table) == 0 {
		return nil, errors.New("Where:Select:未指定表名")
	}

	sql := bytes.NewBufferString("SELECT ")

	if len(cols) == 0 {
		sql.WriteString(" * ")
	} else {
		for _, v := range cols {
			w.e.Dialect().Quote(sql, v)
			sql.WriteByte(',')
		}
		sql.Truncate(sql.Len() - 1)
	}

	sql.WriteString(" FROM ")
	w.e.Dialect().Quote(sql, w.table)
	sql.WriteByte(' ')
	sql.WriteString(w.cond.String())

	return w.e.Query(replace, sql.String(), w.args...)
}
<commit_msg>将where.go中的bytes.Buffer统一由pool管理<commit_after>// Copyright 2015 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package orm

import (
	"bytes"
	"database/sql"
	"errors"

	"github.com/issue9/orm/fetch"
)

const (
	and = iota
	or
)

// 用于产生条件语句。
type Where struct {
	e     engine
	table string
	cond  *bytes.Buffer
	args  []interface{}
}

func newWhere(engine engine) *Where {
	return &Where{
		e:    engine,
		cond: new(bytes.Buffer),
		args: []interface{}{},
	}
}

func (w *Where) where(op int, cond string, args ...interface{}) *Where {
	switch {
	case w.cond.Len() == 0:
		w.cond.WriteString(" WHERE(")
	case op == and:
		w.cond.WriteString(" AND(")
	case op == or:
		w.cond.WriteString(" OR(")
	default:
		panic("where:错误的参数op")
	}
	w.cond.WriteString(cond)
	w.cond.WriteByte(')')
	w.args = append(w.args, args...)

	return w
}

// 将之后的语句以and的形式与当前的语句进行连接
func (w *Where) And(cond string, args ...interface{}) *Where {
	return w.where(and, cond, args...)
}

// 将之后的语句以or的形式与当前的语句进行连接
func (w *Where) Or(cond string, args ...interface{}) *Where {
	return w.where(or, cond, args...)
}

// 指定表名。
func (w *Where) Table(tableName string) *Where {
	w.table = tableName
	return w
}

// 将符合当前条件的所有记录删除。
// replace，是否需将对语句的占位符进行替换。
func (w *Where) Delete(replace bool) error {
	if len(w.table) == 0 {
		return errors.New("Where.Delete:未指定表名")
	}

	sql := pool.Get().(*bytes.Buffer)
	defer pool.Put(sql)

	sql.Reset()
	sql.WriteString("DELETE FROM ")
	w.e.Dialect().Quote(sql, w.table)
	sql.WriteString(w.cond.String())
	_, err := w.e.Exec(replace, sql.String(), w.args...)
	return err
}

// 更新符合当前条件的所有记录。
// replace，是否需将对语句的占位符进行替换。
func (w *Where) Update(replace bool, data map[string]interface{}) error {
	if len(w.table) == 0 {
		return errors.New("Where.Update:未指定表名")
	}

	if len(data) == 0 {
		return errors.New("Where.Update:未指定需要更新的数据")
	}

	sql := pool.Get().(*bytes.Buffer)
	defer pool.Put(sql)

	sql.Reset()
	sql.WriteString("UPDATE ")
	w.e.Dialect().Quote(sql, w.table)
	vals := make([]interface{}, 0, len(data)+len(w.args))
	sql.WriteString(" SET ")
	for k, v := range data {
		w.e.Dialect().Quote(sql, k)
		sql.WriteString("=?,")
		vals = append(vals, v)
	}
	sql.Truncate(sql.Len() - 1) // 去掉最后一个逗号

	sql.WriteString(w.cond.String())
	_, err := w.e.Exec(replace, sql.String(), append(vals, w.args...)...)
	return err
}

// 将符合当前条件的所有记录依次写入objs中。
// replace，是否需将对语句的占位符进行替换。
func (w *Where) Select(replace bool, objs interface{}) error {
	rows, err := w.buildSelectSQL(replace)
	if err != nil {
		return err
	}
	return fetch.Obj(objs, rows)
}

// 返回符合当前条件的所有记录。
// replace，是否需将对语句的占位符进行替换。
func (w *Where) SelectMap(replace bool, cols ...string) ([]map[string]interface{}, error) {
	rows, err := w.buildSelectSQL(replace, cols...)
	if err != nil {
		return nil, err
	}
	return fetch.Map(false, rows)
}

func (w *Where) buildSelectSQL(replace bool, cols ...string) (*sql.Rows, error) {
	if len(w.table) == 0 {
		return nil, errors.New("Where:buildSelectSQL:未指定表名")
	}

	sql := pool.Get().(*bytes.Buffer)
	defer pool.Put(sql)

	sql.Reset()
	sql.WriteString("SELECT ")
	if len(cols) == 0 {
		sql.WriteString(" * ")
	} else {
		for _, v := range cols {
			w.e.Dialect().Quote(sql, v)
			sql.WriteByte(',')
		}
		sql.Truncate(sql.Len() - 1)
	}

	sql.WriteString(" FROM ")
	w.e.Dialect().Quote(sql, w.table)
	sql.WriteString(w.cond.String())

	return w.e.Query(replace, sql.String(), w.args...)
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package zoekt

import (
	"bufio"
	"encoding/binary"
	"encoding/json"
	"io"
	"log"
	"sort"
	"time"
)

// FormatVersion is a version number. It is increased every time the
// on-disk index format is changed.
// 5: subrepositories.
// 6: remove size prefix for posting varint list.
// 7: move subrepos into Repository struct.
// 8: move repoMetaData out of indexMetadata
// 9: use bigendian uint64 for trigrams.
// 10: sections for rune offsets.
const IndexFormatVersion = 8

// FeatureVersion is increased if a feature is added that requires reindexing data.
const FeatureVersion = 0

var _ = log.Println

type indexTOC struct {
	fileContents compoundSection
	fileNames    compoundSection
	fileSections compoundSection
	postings     compoundSection
	newlines     compoundSection
	ngramText    simpleSection
	runeOffsets  simpleSection

	branchMasks simpleSection
	subRepos    simpleSection

	nameNgramText   simpleSection
	namePostings    compoundSection
	nameRuneOffsets simpleSection
	metaData        simpleSection
	repoMetaData    simpleSection
}

func (t *indexTOC) sections() []section {
	return []section{
		// This must be first, so it can be reliably read across
		// file format versions.
		&t.metaData,
		&t.repoMetaData,
		&t.fileContents,
		&t.fileNames,
		&t.fileSections,
		&t.newlines,
		&t.ngramText,
		&t.postings,
		&t.nameNgramText,
		&t.namePostings,
		&t.branchMasks,
		&t.subRepos,
		&t.runeOffsets,
		&t.nameRuneOffsets,
	}
}

func (w *writer) writeTOC(toc *indexTOC) {
	secs := toc.sections()
	w.U32(uint32(len(secs)))
	for _, s := range secs {
		s.write(w)
	}
}

func (s *compoundSection) writeStrings(w *writer, strs []*searchableString) {
	s.start(w)
	for _, f := range strs {
		s.addItem(w, f.data)
	}
	s.end(w)
}

func writePostings(w *writer, s *postingsBuilder, ngramText *simpleSection,
	charOffsets *simpleSection,
	postings *compoundSection) {
	var keys ngramSlice
	for k := range s.postings {
		keys = append(keys, k)
	}
	sort.Sort(keys)

	ngramText.start(w)
	for _, k := range keys {
		var buf [8]byte
		binary.BigEndian.PutUint64(buf[:], uint64(k))
		w.Write(buf[:])
	}
	ngramText.end(w)

	postings.start(w)
	for _, k := range keys {
		postings.addItem(w, s.postings[k])
	}
	postings.end(w)

	charOffsets.start(w)
	w.Write(toSizedDeltas(s.runeOffsets))
	charOffsets.end(w)
}

func (b *IndexBuilder) Write(out io.Writer) error {
	buffered := bufio.NewWriterSize(out, 1<<20)
	defer buffered.Flush()

	w := &writer{w: buffered}
	toc := indexTOC{}

	toc.fileContents.writeStrings(w, b.files)

	toc.newlines.start(w)
	for _, f := range b.files {
		toc.newlines.addItem(w, toSizedDeltas(newLinesIndices(f.data)))
	}
	toc.newlines.end(w)

	toc.branchMasks.start(w)
	for _, m := range b.branchMasks {
		w.U32(m)
	}
	toc.branchMasks.end(w)

	toc.fileSections.start(w)
	for _, s := range b.docSections {
		toc.fileSections.addItem(w, marshalDocSections(s))
	}
	toc.fileSections.end(w)

	writePostings(w, b.contents, &toc.ngramText, &toc.runeOffsets, &toc.postings)

	// names.
	toc.fileNames.writeStrings(w, b.fileNames)

	writePostings(w, b.names, &toc.nameNgramText, &toc.nameRuneOffsets, &toc.namePostings)

	toc.subRepos.start(w)
	w.Write(toSizedDeltas(b.subRepos))
	toc.subRepos.end(w)

	if err := b.writeJSON(&IndexMetadata{
		IndexFormatVersion:  IndexFormatVersion,
		IndexTime:           time.Now(),
		IndexFeatureVersion: FeatureVersion,
	}, &toc.metaData, w); err != nil {
		return err
	}
	if err := b.writeJSON(b.repo, &toc.repoMetaData, w); err != nil {
		return err
	}

	var tocSection simpleSection

	tocSection.start(w)
	w.writeTOC(&toc)
	tocSection.end(w)
	tocSection.write(w)
	return w.err
}

func (b *IndexBuilder) writeJSON(data interface{}, sec *simpleSection, w *writer) error {
	blob, err := json.Marshal(data)
	if err != nil {
		return err
	}
	sec.start(w)
	w.Write(blob)
	sec.end(w)
	return nil
}

func newLinesIndices(in []byte) []uint32 {
	out := make([]uint32, 0, len(in)/30)
	for i, c := range in {
		if c == '\n' {
			out = append(out, uint32(i))
		}
	}
	return out
}
<commit_msg>Actually bump the index format.<commit_after>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package zoekt

import (
	"bufio"
	"encoding/binary"
	"encoding/json"
	"io"
	"log"
	"sort"
	"time"
)

// FormatVersion is a version number. It is increased every time the
// on-disk index format is changed.
// 5: subrepositories.
// 6: remove size prefix for posting varint list.
// 7: move subrepos into Repository struct.
// 8: move repoMetaData out of indexMetadata
// 9: use bigendian uint64 for trigrams.
// 10: sections for rune offsets.
const IndexFormatVersion = 10

// FeatureVersion is increased if a feature is added that requires reindexing data.
const FeatureVersion = 0

var _ = log.Println

type indexTOC struct {
	fileContents compoundSection
	fileNames    compoundSection
	fileSections compoundSection
	postings     compoundSection
	newlines     compoundSection
	ngramText    simpleSection
	runeOffsets  simpleSection

	branchMasks simpleSection
	subRepos    simpleSection

	nameNgramText   simpleSection
	namePostings    compoundSection
	nameRuneOffsets simpleSection
	metaData        simpleSection
	repoMetaData    simpleSection
}

func (t *indexTOC) sections() []section {
	return []section{
		// This must be first, so it can be reliably read across
		// file format versions.
		&t.metaData,
		&t.repoMetaData,
		&t.fileContents,
		&t.fileNames,
		&t.fileSections,
		&t.newlines,
		&t.ngramText,
		&t.postings,
		&t.nameNgramText,
		&t.namePostings,
		&t.branchMasks,
		&t.subRepos,
		&t.runeOffsets,
		&t.nameRuneOffsets,
	}
}

func (w *writer) writeTOC(toc *indexTOC) {
	secs := toc.sections()
	w.U32(uint32(len(secs)))
	for _, s := range secs {
		s.write(w)
	}
}

func (s *compoundSection) writeStrings(w *writer, strs []*searchableString) {
	s.start(w)
	for _, f := range strs {
		s.addItem(w, f.data)
	}
	s.end(w)
}

func writePostings(w *writer, s *postingsBuilder, ngramText *simpleSection,
	charOffsets *simpleSection,
	postings *compoundSection) {
	var keys ngramSlice
	for k := range s.postings {
		keys = append(keys, k)
	}
	sort.Sort(keys)

	ngramText.start(w)
	for _, k := range keys {
		var buf [8]byte
		binary.BigEndian.PutUint64(buf[:], uint64(k))
		w.Write(buf[:])
	}
	ngramText.end(w)

	postings.start(w)
	for _, k := range keys {
		postings.addItem(w, s.postings[k])
	}
	postings.end(w)

	charOffsets.start(w)
	w.Write(toSizedDeltas(s.runeOffsets))
	charOffsets.end(w)
}

func (b *IndexBuilder) Write(out io.Writer) error {
	buffered := bufio.NewWriterSize(out, 1<<20)
	defer buffered.Flush()

	w := &writer{w: buffered}
	toc := indexTOC{}

	toc.fileContents.writeStrings(w, b.files)

	toc.newlines.start(w)
	for _, f := range b.files {
		toc.newlines.addItem(w, toSizedDeltas(newLinesIndices(f.data)))
	}
	toc.newlines.end(w)

	toc.branchMasks.start(w)
	for _, m := range b.branchMasks {
		w.U32(m)
	}
	toc.branchMasks.end(w)

	toc.fileSections.start(w)
	for _, s := range b.docSections {
		toc.fileSections.addItem(w, marshalDocSections(s))
	}
	toc.fileSections.end(w)

	writePostings(w, b.contents, &toc.ngramText, &toc.runeOffsets, &toc.postings)

	// names.
	toc.fileNames.writeStrings(w, b.fileNames)

	writePostings(w, b.names, &toc.nameNgramText, &toc.nameRuneOffsets, &toc.namePostings)

	toc.subRepos.start(w)
	w.Write(toSizedDeltas(b.subRepos))
	toc.subRepos.end(w)

	if err := b.writeJSON(&IndexMetadata{
		IndexFormatVersion:  IndexFormatVersion,
		IndexTime:           time.Now(),
		IndexFeatureVersion: FeatureVersion,
	}, &toc.metaData, w); err != nil {
		return err
	}
	if err := b.writeJSON(b.repo, &toc.repoMetaData, w); err != nil {
		return err
	}

	var tocSection simpleSection

	tocSection.start(w)
	w.writeTOC(&toc)
	tocSection.end(w)
	tocSection.write(w)
	return w.err
}

func (b *IndexBuilder) writeJSON(data interface{}, sec *simpleSection, w *writer) error {
	blob, err := json.Marshal(data)
	if err != nil {
		return err
	}
	sec.start(w)
	w.Write(blob)
	sec.end(w)
	return nil
}

func newLinesIndices(in []byte) []uint32 {
	out := make([]uint32, 0, len(in)/30)
	for i, c := range in {
		if c == '\n' {
			out = append(out, uint32(i))
		}
	}
	return out
}
<|endoftext|>


<commit_before>// Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package xurls

import "regexp"

//go:generate go run generate/tldsgen/main.go
//go:generate go run generate/regexgen/main.go

const (
	letter    = `\p{L}`
	number    = `\p{N}`
	iriChar   = letter + number
	pathChar  = iriChar + `/\-+_@|&=#~*%.,:;'?!` + `\p{Sc}\p{Sk}\p{So}`
	endChar   = iriChar + `/\-+_&~*%` + `\p{Sc}`
	octet     = `(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9][0-9]|[0-9])`
	ipv4Addr  = `\b` + octet + `\.` + octet + `\.` + octet + `\.` + octet + `\b`
	ipv6Addr  = `([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:[0-9a-fA-F]{0,4}|:[0-9a-fA-F]{1,4})?|(:[0-9a-fA-F]{1,4}){0,2})|(:[0-9a-fA-F]{1,4}){0,3})|(:[0-9a-fA-F]{1,4}){0,4})|:(:[0-9a-fA-F]{1,4}){0,5})((:[0-9a-fA-F]{1,4}){2}|:(25[0-5]|(2[0-4]|1[0-9]|[1-9])?[0-9])(\.(25[0-5]|(2[0-4]|1[0-9]|[1-9])?[0-9])){3})|(([0-9a-fA-F]{1,4}:){1,6}|:):[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){7}:`
	ipAddr    = `(` + ipv4Addr + `|` + ipv6Addr + `)`
	iri       = `[` + iriChar + `]([` + iriChar + `\-]*[` + iriChar + `])?`
	domain    = `(` + iri + `\.)+`
	hostName  = `(` + domain + gtld + `|` + ipAddr + `)`
	wellParen = `\([` + pathChar + `]*(\([` + pathChar + `]*\)[` + pathChar + `]*)*\)`
	pathCont  = `([` + pathChar + `]*(` + wellParen + `|[` + endChar + `])+)+`
	path      = `(/|/` + pathCont + `?|\b|$)`
	port      = `(:[0-9]*)?`
	webURL    = hostName + port + path
	email     = `[a-zA-Z0-9._%\-+]+@` + hostName

	comScheme = `[a-zA-Z][a-zA-Z.\-+]*://`
	scheme    = `(` + comScheme + `|` + otherScheme + `)`
	strict    = `\b` + scheme + pathCont
	relaxed   = strict + `|` + webURL + `|` + email
)

var (
	// Relaxed matches all the urls it can find.
	Relaxed = regexp.MustCompile(relaxed)
	// Strict only matches urls with a scheme to avoid false positives.
	Strict = regexp.MustCompile(strict)
)

func init() {
	Relaxed.Longest()
	Strict.Longest()
}

// StrictMatching produces a regexp that matches urls like Strict but whose
// scheme matches the given regular expression.
func StrictMatching(schemeExp string) (*regexp.Regexp, error) {
	strictMatching := `\b(?i)(` + schemeExp + `)(?-i)` + pathCont
	re, err := regexp.Compile(strictMatching)
	if err != nil {
		return nil, err
	}
	re.Longest()
	return re, nil
}
<commit_msg>Roerder path chars to match end chars<commit_after>// Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package xurls

import "regexp"

//go:generate go run generate/tldsgen/main.go
//go:generate go run generate/regexgen/main.go

const (
	letter    = `\p{L}`
	number    = `\p{N}`
	iriChar   = letter + number
	pathChar  = iriChar + `/\-+_&~*%@|=#.,:;'?!` + `\p{Sc}\p{Sk}\p{So}`
	endChar   = iriChar + `/\-+_&~*%` + `\p{Sc}`
	octet     = `(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9][0-9]|[0-9])`
	ipv4Addr  = `\b` + octet + `\.` + octet + `\.` + octet + `\.` + octet + `\b`
	ipv6Addr  = `([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:[0-9a-fA-F]{0,4}|:[0-9a-fA-F]{1,4})?|(:[0-9a-fA-F]{1,4}){0,2})|(:[0-9a-fA-F]{1,4}){0,3})|(:[0-9a-fA-F]{1,4}){0,4})|:(:[0-9a-fA-F]{1,4}){0,5})((:[0-9a-fA-F]{1,4}){2}|:(25[0-5]|(2[0-4]|1[0-9]|[1-9])?[0-9])(\.(25[0-5]|(2[0-4]|1[0-9]|[1-9])?[0-9])){3})|(([0-9a-fA-F]{1,4}:){1,6}|:):[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){7}:`
	ipAddr    = `(` + ipv4Addr + `|` + ipv6Addr + `)`
	iri       = `[` + iriChar + `]([` + iriChar + `\-]*[` + iriChar + `])?`
	domain    = `(` + iri + `\.)+`
	hostName  = `(` + domain + gtld + `|` + ipAddr + `)`
	wellParen = `\([` + pathChar + `]*(\([` + pathChar + `]*\)[` + pathChar + `]*)*\)`
	pathCont  = `([` + pathChar + `]*(` + wellParen + `|[` + endChar + `])+)+`
	path      = `(/|/` + pathCont + `?|\b|$)`
	port      = `(:[0-9]*)?`
	webURL    = hostName + port + path
	email     = `[a-zA-Z0-9._%\-+]+@` + hostName

	comScheme = `[a-zA-Z][a-zA-Z.\-+]*://`
	scheme    = `(` + comScheme + `|` + otherScheme + `)`
	strict    = `\b` + scheme + pathCont
	relaxed   = strict + `|` + webURL + `|` + email
)

var (
	// Relaxed matches all the urls it can find.
	Relaxed = regexp.MustCompile(relaxed)
	// Strict only matches urls with a scheme to avoid false positives.
	Strict = regexp.MustCompile(strict)
)

func init() {
	Relaxed.Longest()
	Strict.Longest()
}

// StrictMatching produces a regexp that matches urls like Strict but whose
// scheme matches the given regular expression.
func StrictMatching(schemeExp string) (*regexp.Regexp, error) {
	strictMatching := `\b(?i)(` + schemeExp + `)(?-i)` + pathCont
	re, err := regexp.Compile(strictMatching)
	if err != nil {
		return nil, err
	}
	re.Longest()
	return re, nil
}
<|endoftext|>


<commit_before>package flagx

import "strconv"

// YesNo returns a flag.Value for a boolean value, but accepting "yes"/"y"/"no"/"n" in addition to strconv.ParseBool values.
func YesNo(b *bool) Value {
	_ = *b // Check not nil
	return yesNo{b}
}

type yesNo struct {
	b *bool
}

func (yn yesNo) Set(s string) error {
	switch s {
	case "yes", "y", "true", "1":
		*yn.b = true
	case "no", "n", "false", "0":
		*yn.b = false
	default:
		v, err := strconv.ParseBool(s)

		if err == nil {
			*yn.b = v
		}
		return err

	}
	return nil
}

func (yn yesNo) Get() interface{} { return *yn.b }

func (yn yesNo) String() string { return strconv.FormatBool(*yn.b) }

func (yn yesNo) IsBoolFlag() bool { return true }
<commit_msg>doc: add doc links for YesNo<commit_after>package flagx

import "strconv"

// YesNo returns a [flag.Value] for a boolean value, but accepting "yes"/"y"/"no"/"n" in addition to [strconv.ParseBool] values.
func YesNo(b *bool) Value {
	_ = *b // Check not nil
	return yesNo{b}
}

type yesNo struct {
	b *bool
}

func (yn yesNo) Set(s string) error {
	switch s {
	case "yes", "y", "true", "1":
		*yn.b = true
	case "no", "n", "false", "0":
		*yn.b = false
	default:
		v, err := strconv.ParseBool(s)

		if err == nil {
			*yn.b = v
		}
		return err

	}
	return nil
}

func (yn yesNo) Get() interface{} { return *yn.b }

func (yn yesNo) String() string { return strconv.FormatBool(*yn.b) }

func (yn yesNo) IsBoolFlag() bool { return true }
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package features

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"strings"

	"k8s.io/contrib/mungegithub/github"
	"k8s.io/kubernetes/pkg/util/sets"
	"k8s.io/kubernetes/pkg/util/yaml"

	parseYaml "github.com/ghodss/yaml"
	"github.com/golang/glog"
	"github.com/spf13/cobra"
)

const (
	ownerFilename = "OWNERS" // file which contains owners and assignees
	// RepoFeatureName is how mungers should indicate this is required
	RepoFeatureName = "gitrepos"
)

type assignmentConfig struct {
	Assignees []string `json:assignees yaml:assignees`
	Approvers []string `json:approvers yaml:approvers`
	//Reviewers []string `json:reviewers yaml:reviewers`
	//Owners []string `json:owners`
}

// RepoInfo provides information about users in OWNERS files in a git repo
type RepoInfo struct {
	BaseDir      string
	EnableMdYaml bool

	enabled    bool
	projectDir string
	assignees  map[string]sets.String
	config     *github.Config
	//owners   map[string]sets.String
}

func init() {
	RegisterFeature(&RepoInfo{})
}

// Name is just going to return the name mungers use to request this feature
func (o *RepoInfo) Name() string {
	return RepoFeatureName
}

func (o *RepoInfo) walkFunc(path string, info os.FileInfo, err error) error {
	if err != nil {
		glog.Errorf("%v", err)
		return nil
	}
	filename := filepath.Base(path)
	if info.Mode().IsDir() {
		switch filename {
		case ".git":
			return filepath.SkipDir
		case "_output":
			return filepath.SkipDir
		}
	}
	if !info.Mode().IsRegular() {
		return nil
	}

	c := &assignmentConfig{}

	// '.md' files may contain assignees at the top of the file in a yaml header
	// Flag guarded because this is only enabled in some repos
	if o.EnableMdYaml && filename != ownerFilename && strings.HasSuffix(filename, "md") {
		// Parse the yaml header from the file if it exists and marshal into the config
		if err := decodeAssignmentConfig(path, c); err != nil {
			glog.Errorf("%v", err)
			return err
		}

		// Set assignees for this file using the relative path if they were found
		path, err = filepath.Rel(o.projectDir, path)
		if err != nil {
			glog.Errorf("Unable to find relative path between %q and %q: %v", o.projectDir, path, err)
			return err
		}
		o.assignees[path] = sets.NewString()
		if len(c.Assignees) > 0 {
			o.assignees[path] = o.assignees[path].Union(sets.NewString(c.Assignees...))
		}
		if len(c.Approvers) > 0 {
			o.assignees[path] = o.assignees[path].Union(sets.NewString(c.Approvers...))
		}

		return nil
	}

	if filename != ownerFilename {
		return nil
	}

	file, err := os.Open(path)
	if err != nil {
		glog.Errorf("%v", err)
		return nil
	}
	defer file.Close()

	if err := yaml.NewYAMLToJSONDecoder(file).Decode(c); err != nil {
		glog.Errorf("%v", err)
		return nil
	}

	path, err = filepath.Rel(o.projectDir, path)
	if err != nil {
		glog.Errorf("Unable to find relative path between %q and %q: %v", o.projectDir, path, err)
		return err
	}
	path = filepath.Dir(path)
	// Make the root explicitly / so its easy to distinguish. Nothing else is `/` anchored
	if path == "." {
		path = "/"
	}
	o.assignees[path] = sets.NewString()
	if len(c.Assignees) > 0 {
		o.assignees[path] = o.assignees[path].Union(sets.NewString(c.Assignees...))
	} else if len(c.Approvers) > 0 {
		o.assignees[path] = o.assignees[path].Union(sets.NewString(c.Approvers...))
	}
	//if len(c.Owners) > 0 {
	//o.owners[path] = sets.NewString(c.Owners...)
	//}
	return nil
}

// decodeAssignmentConfig will parse the yaml header if it exists and unmarshal it into an assignmentConfig.
// If no yaml header is found, do nothing
// Returns an error if the file cannot be read or the yaml header is found but cannot be unmarshalled
var mdStructuredHeaderRegex = regexp.MustCompile("^---\n(.|\n)*\n---")

func decodeAssignmentConfig(path string, config *assignmentConfig) error {
	fileBytes, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}
	// Parse the yaml header from the top of the file.  Will return an empty string if regex does not match.
	meta := mdStructuredHeaderRegex.FindString(string(fileBytes))

	// Unmarshal the yaml header into the config
	return parseYaml.Unmarshal([]byte(meta), &config)
}

func (o *RepoInfo) updateRepoUsers() error {
	out, err := o.GitCommand([]string{"pull"})
	if err != nil {
		glog.Errorf("Unable to run git pull:\n%s\n%v", string(out), err)
		return err
	}

	out, err = o.GitCommand([]string{"rev-parse", "HEAD"})
	if err != nil {
		glog.Errorf("Unable get sha of HEAD:\n%s\n%v", string(out), err)
		return err
	}
	sha := out

	o.assignees = map[string]sets.String{}
	//o.owners = map[string]sets.String{}
	err = filepath.Walk(o.projectDir, o.walkFunc)
	if err != nil {
		glog.Errorf("Got error %v", err)
	}
	glog.Infof("Loaded config from %s:%s", o.projectDir, sha)
	glog.V(5).Infof("assignees: %v", o.assignees)
	//glog.V(5).Infof("owners: %v", o.owners)
	return nil
}

// Initialize will initialize the munger
func (o *RepoInfo) Initialize(config *github.Config) error {
	o.enabled = true
	o.config = config
	o.projectDir = path.Join(o.BaseDir, o.config.Project)

	if len(o.BaseDir) == 0 {
		glog.Fatalf("--repo-dir is required with selected munger(s)")
	}
	finfo, err := os.Stat(o.BaseDir)
	if err != nil {
		return fmt.Errorf("Unable to stat --repo-dir: %v", err)
	}
	if !finfo.IsDir() {
		return fmt.Errorf("--repo-dir is not a directory")
	}

	// check if the cloned dir already exists, if yes, cleanup.
	if _, err := os.Stat(o.projectDir); !os.IsNotExist(err) {
		if err := o.cleanUp(o.projectDir); err != nil {
			return fmt.Errorf("Unable to remove old clone directory at %v: %v", o.projectDir, err)
		}
	}

	if cloneUrl, err := o.cloneRepo(); err != nil {
		return fmt.Errorf("Unable to clone %v: %v", cloneUrl, err)
	}
	return o.updateRepoUsers()
}

func (o *RepoInfo) cleanUp(path string) error {
	return os.RemoveAll(path)
}

func (o *RepoInfo) cloneRepo() (string, error) {
	cloneUrl := fmt.Sprintf("https://github.com/%s/%s.git", o.config.Org, o.config.Project)
	output, err := o.gitCommandDir([]string{"clone", cloneUrl, o.projectDir}, o.BaseDir)
	if err != nil {
		glog.Errorf("Failed to clone github repo: %s", output)
	}
	return cloneUrl, err
}

// EachLoop is called at the start of every munge loop
func (o *RepoInfo) EachLoop() error {
	if !o.enabled {
		return nil
	}
	_, err := o.GitCommand([]string{"remote", "update"})
	if err != nil {
		glog.Errorf("Unable to git remote update: %v", err)
	}
	return o.updateRepoUsers()
}

// AddFlags will add any request flags to the cobra `cmd`
func (o *RepoInfo) AddFlags(cmd *cobra.Command) {
	cmd.Flags().StringVar(&o.BaseDir, "repo-dir", "", "Path to perform checkout of repository")
	cmd.Flags().BoolVar(&o.EnableMdYaml, "enable-md-yaml", false, "If true, look for assignees in md yaml headers.")
}

// GitCommand will execute the git command with the `args` within the project directory.
func (o *RepoInfo) GitCommand(args []string) ([]byte, error) {
	return o.gitCommandDir(args, o.projectDir)
}

// GitCommandDir will execute the git command with the `args` within the 'dir' directory.
func (o *RepoInfo) gitCommandDir(args []string, cmdDir string) ([]byte, error) {
	cmd := exec.Command("git", args...)
	cmd.Dir = cmdDir
	return cmd.CombinedOutput()
}

func peopleForPath(path string, people map[string]sets.String, leafOnly bool, enableMdYaml bool) sets.String {
	d := path
	if !enableMdYaml {
		d = filepath.Dir(path)
	}

	out := sets.NewString()
	for {
		// special case the root
		if d == "" {
			d = "/"
		}
		s, ok := people[d]
		if ok {
			out = out.Union(s)
			if leafOnly {
				break
			}
		}
		if d == "/" {
			break
		}
		d, _ = filepath.Split(d)
		d = strings.TrimSuffix(d, "/")
	}
	return out
}

// LeafAssignees returns a set of users who are the closest assignees to the
// requested file. If pkg/OWNERS has user1 and pkg/util/OWNERS has user2 this
// will only return user2 for the path pkg/util/sets/file.go
func (o *RepoInfo) LeafAssignees(path string) sets.String {
	return peopleForPath(path, o.assignees, true, o.EnableMdYaml)
}

// Assignees returns a set of users who are the closest assignees to the
// requested file. If pkg/OWNERS has user1 and pkg/util/OWNERS has user2 this
// will return both user1 and user2 for the path pkg/util/sets/file.go
func (o *RepoInfo) Assignees(path string) sets.String {
	return peopleForPath(path, o.assignees, false, o.EnableMdYaml)
}

//func (o *RepoInfo) LeafOwners(path string) sets.String {
//return people(path, o.owners, true)
//}

//func (o *RepoInfo) Owners(path string) sets.String {
//return people(path, o.owners, false)
//}
<commit_msg>Keep looking up the tree if we found an empty set<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package features

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"strings"

	"k8s.io/contrib/mungegithub/github"
	"k8s.io/kubernetes/pkg/util/sets"
	"k8s.io/kubernetes/pkg/util/yaml"

	parseYaml "github.com/ghodss/yaml"
	"github.com/golang/glog"
	"github.com/spf13/cobra"
)

const (
	ownerFilename = "OWNERS" // file which contains owners and assignees
	// RepoFeatureName is how mungers should indicate this is required
	RepoFeatureName = "gitrepos"
)

type assignmentConfig struct {
	Assignees []string `json:assignees yaml:assignees`
	Approvers []string `json:approvers yaml:approvers`
	//Reviewers []string `json:reviewers yaml:reviewers`
	//Owners []string `json:owners`
}

// RepoInfo provides information about users in OWNERS files in a git repo
type RepoInfo struct {
	BaseDir      string
	EnableMdYaml bool

	enabled    bool
	projectDir string
	assignees  map[string]sets.String
	config     *github.Config
	//owners   map[string]sets.String
}

func init() {
	RegisterFeature(&RepoInfo{})
}

// Name is just going to return the name mungers use to request this feature
func (o *RepoInfo) Name() string {
	return RepoFeatureName
}

func (o *RepoInfo) walkFunc(path string, info os.FileInfo, err error) error {
	if err != nil {
		glog.Errorf("%v", err)
		return nil
	}
	filename := filepath.Base(path)
	if info.Mode().IsDir() {
		switch filename {
		case ".git":
			return filepath.SkipDir
		case "_output":
			return filepath.SkipDir
		}
	}
	if !info.Mode().IsRegular() {
		return nil
	}

	c := &assignmentConfig{}

	// '.md' files may contain assignees at the top of the file in a yaml header
	// Flag guarded because this is only enabled in some repos
	if o.EnableMdYaml && filename != ownerFilename && strings.HasSuffix(filename, "md") {
		// Parse the yaml header from the file if it exists and marshal into the config
		if err := decodeAssignmentConfig(path, c); err != nil {
			glog.Errorf("%v", err)
			return err
		}

		// Set assignees for this file using the relative path if they were found
		path, err = filepath.Rel(o.projectDir, path)
		if err != nil {
			glog.Errorf("Unable to find relative path between %q and %q: %v", o.projectDir, path, err)
			return err
		}
		o.assignees[path] = sets.NewString()
		if len(c.Assignees) > 0 {
			o.assignees[path] = o.assignees[path].Union(sets.NewString(c.Assignees...))
		}
		if len(c.Approvers) > 0 {
			o.assignees[path] = o.assignees[path].Union(sets.NewString(c.Approvers...))
		}

		return nil
	}

	if filename != ownerFilename {
		return nil
	}

	file, err := os.Open(path)
	if err != nil {
		glog.Errorf("%v", err)
		return nil
	}
	defer file.Close()

	if err := yaml.NewYAMLToJSONDecoder(file).Decode(c); err != nil {
		glog.Errorf("%v", err)
		return nil
	}

	path, err = filepath.Rel(o.projectDir, path)
	if err != nil {
		glog.Errorf("Unable to find relative path between %q and %q: %v", o.projectDir, path, err)
		return err
	}
	path = filepath.Dir(path)
	// Make the root explicitly / so its easy to distinguish. Nothing else is `/` anchored
	if path == "." {
		path = "/"
	}
	o.assignees[path] = sets.NewString()
	if len(c.Assignees) > 0 {
		o.assignees[path] = o.assignees[path].Union(sets.NewString(c.Assignees...))
	} else if len(c.Approvers) > 0 {
		o.assignees[path] = o.assignees[path].Union(sets.NewString(c.Approvers...))
	}
	//if len(c.Owners) > 0 {
	//o.owners[path] = sets.NewString(c.Owners...)
	//}
	return nil
}

// decodeAssignmentConfig will parse the yaml header if it exists and unmarshal it into an assignmentConfig.
// If no yaml header is found, do nothing
// Returns an error if the file cannot be read or the yaml header is found but cannot be unmarshalled
var mdStructuredHeaderRegex = regexp.MustCompile("^---\n(.|\n)*\n---")

func decodeAssignmentConfig(path string, config *assignmentConfig) error {
	fileBytes, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}
	// Parse the yaml header from the top of the file.  Will return an empty string if regex does not match.
	meta := mdStructuredHeaderRegex.FindString(string(fileBytes))

	// Unmarshal the yaml header into the config
	return parseYaml.Unmarshal([]byte(meta), &config)
}

func (o *RepoInfo) updateRepoUsers() error {
	out, err := o.GitCommand([]string{"pull"})
	if err != nil {
		glog.Errorf("Unable to run git pull:\n%s\n%v", string(out), err)
		return err
	}

	out, err = o.GitCommand([]string{"rev-parse", "HEAD"})
	if err != nil {
		glog.Errorf("Unable get sha of HEAD:\n%s\n%v", string(out), err)
		return err
	}
	sha := out

	o.assignees = map[string]sets.String{}
	//o.owners = map[string]sets.String{}
	err = filepath.Walk(o.projectDir, o.walkFunc)
	if err != nil {
		glog.Errorf("Got error %v", err)
	}
	glog.Infof("Loaded config from %s:%s", o.projectDir, sha)
	glog.V(5).Infof("assignees: %v", o.assignees)
	//glog.V(5).Infof("owners: %v", o.owners)
	return nil
}

// Initialize will initialize the munger
func (o *RepoInfo) Initialize(config *github.Config) error {
	o.enabled = true
	o.config = config
	o.projectDir = path.Join(o.BaseDir, o.config.Project)

	if len(o.BaseDir) == 0 {
		glog.Fatalf("--repo-dir is required with selected munger(s)")
	}
	finfo, err := os.Stat(o.BaseDir)
	if err != nil {
		return fmt.Errorf("Unable to stat --repo-dir: %v", err)
	}
	if !finfo.IsDir() {
		return fmt.Errorf("--repo-dir is not a directory")
	}

	// check if the cloned dir already exists, if yes, cleanup.
	if _, err := os.Stat(o.projectDir); !os.IsNotExist(err) {
		if err := o.cleanUp(o.projectDir); err != nil {
			return fmt.Errorf("Unable to remove old clone directory at %v: %v", o.projectDir, err)
		}
	}

	if cloneUrl, err := o.cloneRepo(); err != nil {
		return fmt.Errorf("Unable to clone %v: %v", cloneUrl, err)
	}
	return o.updateRepoUsers()
}

func (o *RepoInfo) cleanUp(path string) error {
	return os.RemoveAll(path)
}

func (o *RepoInfo) cloneRepo() (string, error) {
	cloneUrl := fmt.Sprintf("https://github.com/%s/%s.git", o.config.Org, o.config.Project)
	output, err := o.gitCommandDir([]string{"clone", cloneUrl, o.projectDir}, o.BaseDir)
	if err != nil {
		glog.Errorf("Failed to clone github repo: %s", output)
	}
	return cloneUrl, err
}

// EachLoop is called at the start of every munge loop
func (o *RepoInfo) EachLoop() error {
	if !o.enabled {
		return nil
	}
	_, err := o.GitCommand([]string{"remote", "update"})
	if err != nil {
		glog.Errorf("Unable to git remote update: %v", err)
	}
	return o.updateRepoUsers()
}

// AddFlags will add any request flags to the cobra `cmd`
func (o *RepoInfo) AddFlags(cmd *cobra.Command) {
	cmd.Flags().StringVar(&o.BaseDir, "repo-dir", "", "Path to perform checkout of repository")
	cmd.Flags().BoolVar(&o.EnableMdYaml, "enable-md-yaml", false, "If true, look for assignees in md yaml headers.")
}

// GitCommand will execute the git command with the `args` within the project directory.
func (o *RepoInfo) GitCommand(args []string) ([]byte, error) {
	return o.gitCommandDir(args, o.projectDir)
}

// GitCommandDir will execute the git command with the `args` within the 'dir' directory.
func (o *RepoInfo) gitCommandDir(args []string, cmdDir string) ([]byte, error) {
	cmd := exec.Command("git", args...)
	cmd.Dir = cmdDir
	return cmd.CombinedOutput()
}

func peopleForPath(path string, people map[string]sets.String, leafOnly bool, enableMdYaml bool) sets.String {
	d := path
	if !enableMdYaml {
		d = filepath.Dir(path)
	}

	out := sets.NewString()
	for {
		// special case the root
		if d == "" {
			d = "/"
		}
		s, ok := people[d]
		if ok {
			out = out.Union(s)
			if leafOnly && out.Len() > 0 {
				break
			}
		}
		if d == "/" {
			break
		}
		d, _ = filepath.Split(d)
		d = strings.TrimSuffix(d, "/")
	}
	return out
}

// LeafAssignees returns a set of users who are the closest assignees to the
// requested file. If pkg/OWNERS has user1 and pkg/util/OWNERS has user2 this
// will only return user2 for the path pkg/util/sets/file.go
func (o *RepoInfo) LeafAssignees(path string) sets.String {
	return peopleForPath(path, o.assignees, true, o.EnableMdYaml)
}

// Assignees returns a set of users who are the closest assignees to the
// requested file. If pkg/OWNERS has user1 and pkg/util/OWNERS has user2 this
// will return both user1 and user2 for the path pkg/util/sets/file.go
func (o *RepoInfo) Assignees(path string) sets.String {
	return peopleForPath(path, o.assignees, false, o.EnableMdYaml)
}

//func (o *RepoInfo) LeafOwners(path string) sets.String {
//return people(path, o.owners, true)
//}

//func (o *RepoInfo) Owners(path string) sets.String {
//return people(path, o.owners, false)
//}
<|endoftext|>


<commit_before>package client

import (
	"github.com/ibrt/go-oauto/oauto/api"
	"net/http"
	"fmt"
	"encoding/json"
	"github.com/go-errors/errors"
	"bytes"
)

func Authenticate(baseURL string, request *api.AuthenticateRequest) (*api.AuthenticateResponse, error) {
	body, err := json.Marshal(request)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	resp, err := http.Post(fmt.Sprintf("%v/api/authenticate", baseURL), "application/json", bytes.NewBuffer(body))
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}
	if resp.StatusCode != http.StatusOK {
		return nil, errors.Errorf("Authenticate request failed with status %v.", resp.StatusCode)
	}

	authResp := &api.AuthenticateResponse{}
	if err := json.NewDecoder(resp.Body).Decode(&authResp); err != nil {
		return nil, errors.Wrap(err, 0)
	}

	return authResp, nil
}
<commit_msg>Improve error reporting in client.<commit_after>package client

import (
	"github.com/ibrt/go-oauto/oauto/api"
	"net/http"
	"fmt"
	"encoding/json"
	"github.com/go-errors/errors"
	"bytes"
)

func Authenticate(baseURL string, request *api.AuthenticateRequest) (*api.AuthenticateResponse, error) {
	body, err := json.Marshal(request)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	resp, err := http.Post(fmt.Sprintf("%v/api/authenticate", baseURL), "application/json", bytes.NewBuffer(body))
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	authResp := &api.AuthenticateResponse{}
	if err := json.NewDecoder(resp.Body).Decode(&authResp); err != nil {
		return nil, errors.Wrap(err, 0)
	}

	if resp.StatusCode != http.StatusOK {
		return nil, errors.Errorf("Authenticate request failed with status %v: '%+v'.", resp.StatusCode, authResp)
	}

	return authResp, nil
}
<|endoftext|>


<commit_before>package messages

import (
	"database/sql"
	"fmt"
	"github.com/oipwg/media-protocol/utility"
	"log"
	"strconv"
)

func StoreOIP041Deactivate(o Oip041, dbtx *sql.Tx) error {
	// ToDo: perhaps save a record of this happening
	// ToDo: refactor the table deciding logic, it's terribad

	oip_d := o.Deactivate

	// Check media table
	table := "media"
	stmt, err := dbtx.Prepare(`SELECT publisher FROM ` + table + ` WHERE txid = ? LIMIT 1;`)
	if err != nil {
		log.Fatal(err)
	}
	row := stmt.QueryRow(oip_d.Reference)
	var publisher string
	err = row.Scan(&publisher)
	if err != nil {
		// Close media check
		stmt.Close()

		// Check oip table
		fmt.Println("Might be an OIP")
		table = "oip_artifact"
		stmt, err := dbtx.Prepare(`SELECT publisher FROM ` + table + ` WHERE txid = ? LIMIT 1;`)
		if err != nil {
			log.Fatal(err)
		}
		row := stmt.QueryRow(oip_d.Reference)
		err = row.Scan(&publisher)
		if err != nil {
			if err == sql.ErrNoRows {
				return err
			} else {
				log.Fatal(err)
			}
		}
		stmt.Close()
	} else {
		// Close media check
		stmt.Close()
	}

	fmt.Println(publisher)

	preImage := oip_d.Reference + "-" + strconv.FormatInt(oip_d.Timestamp, 10)
	valid, err := utility.CheckSignature(publisher, o.Signature, preImage)
	if !valid {
		fmt.Println("Signature check failed.")
		return err
	}

	stmtstr := `update ` + table + ` set invalidated = 1 where publisher = ? and txid = ?`

	stmt, err = dbtx.Prepare(stmtstr)
	if err != nil {
		fmt.Println("exit 606")
		log.Fatal(err)
	}

	_, stmterr := stmt.Exec(publisher, oip_d.Reference)
	if stmterr != nil {
		fmt.Println("exit 607")
		log.Fatal(stmterr)
	}

	stmt.Close()
	return nil
}

func VerifyOIP041Deactivate(o Oip041) (Oip041, error) {
	oip_d := o.Deactivate

	if len(oip_d.Reference) != 64 {
		return o, ErrInvalidReference
	}
	return o, nil
}
<commit_msg>Correct deactivation signature check<commit_after>package messages

import (
	"database/sql"
	"fmt"
	"github.com/oipwg/media-protocol/utility"
	"log"
	"strconv"
)

func StoreOIP041Deactivate(o Oip041, dbtx *sql.Tx) error {
	// ToDo: perhaps save a record of this happening
	// ToDo: refactor the table deciding logic, it's terribad

	oip_d := o.Deactivate

	// Check media table
	table := "media"
	stmt, err := dbtx.Prepare(`SELECT publisher FROM ` + table + ` WHERE txid = ? LIMIT 1;`)
	if err != nil {
		log.Fatal(err)
	}
	row := stmt.QueryRow(oip_d.Reference)
	var publisher string
	err = row.Scan(&publisher)
	if err != nil {
		// Close media check
		stmt.Close()

		// Check oip table
		fmt.Println("Might be an OIP")
		table = "oip_artifact"
		stmt, err := dbtx.Prepare(`SELECT publisher FROM ` + table + ` WHERE txid = ? LIMIT 1;`)
		if err != nil {
			log.Fatal(err)
		}
		row := stmt.QueryRow(oip_d.Reference)
		err = row.Scan(&publisher)
		if err != nil {
			if err == sql.ErrNoRows {
				return err
			} else {
				log.Fatal(err)
			}
		}
		stmt.Close()
	} else {
		// Close media check
		stmt.Close()
	}

	fmt.Println(publisher)

	preImage := oip_d.Reference + "-" + publisher + "-" + strconv.FormatInt(oip_d.Timestamp, 10)
	valid, err := utility.CheckSignature(publisher, o.Signature, preImage)
	if !valid {
		fmt.Println("Signature check failed.")
		return err
	}

	stmtstr := `update ` + table + ` set invalidated = 1 where publisher = ? and txid = ?`

	stmt, err = dbtx.Prepare(stmtstr)
	if err != nil {
		fmt.Println("exit 606")
		log.Fatal(err)
	}

	_, stmterr := stmt.Exec(publisher, oip_d.Reference)
	if stmterr != nil {
		fmt.Println("exit 607")
		log.Fatal(stmterr)
	}

	stmt.Close()
	return nil
}

func VerifyOIP041Deactivate(o Oip041) (Oip041, error) {
	oip_d := o.Deactivate

	if len(oip_d.Reference) != 64 {
		return o, ErrInvalidReference
	}
	return o, nil
}
<|endoftext|>


<commit_before>// Package cluster holds the cluster CRD logic and definitions
// A cluster is comprised of a primary service, replica service,
// primary deployment, and replica deployment
package cluster

/*
 Copyright 2017-2018 Crunchy Data Solutions, Inc.
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

import (
	log "github.com/Sirupsen/logrus"
	crv1 "github.com/crunchydata/postgres-operator/apis/cr/v1"
	"github.com/crunchydata/postgres-operator/kubeapi"
	"github.com/crunchydata/postgres-operator/operator"
	"github.com/crunchydata/postgres-operator/util"
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
)

// FailoverBase ...
func FailoverBase(namespace string, clientset *kubernetes.Clientset, client *rest.RESTClient, task *crv1.Pgtask, restconfig *rest.Config) {
	var err error

	//look up the pgcluster for this task
	//in the case, the clustername is passed as a key in the
	//parameters map
	var clusterName string
	for k, _ := range task.Spec.Parameters {
		clusterName = k
	}

	cluster := crv1.Pgcluster{}
	_, err = kubeapi.Getpgcluster(client, &cluster,
		clusterName, namespace)
	if err != nil {
		return
	}

	if cluster.Spec.Strategy == "" {
		cluster.Spec.Strategy = "1"
		log.Info("using default strategy")
	}

	strategy, ok := strategyMap[cluster.Spec.Strategy]
	if ok {
		log.Info("strategy found")
	} else {
		log.Error("invalid Strategy requested for cluster failover " + cluster.Spec.Strategy)
		return
	}

	strategy.Failover(clientset, client, clusterName, task, namespace, restconfig)

	//scale up the replicas to replace the failover target
	replaceReplica(client, &cluster)

}

func replaceReplica(client *rest.RESTClient, cluster *crv1.Pgcluster) {

	//generate new replica name
	uniqueName := cluster.Spec.Name + "-" + util.RandStringBytesRmndr(4)

	spec := crv1.PgreplicaSpec{}
	spec.Name = uniqueName
	spec.ClusterName = cluster.Spec.Name
	spec.ReplicaStorage = cluster.Spec.ReplicaStorage
	spec.ContainerResources = cluster.Spec.ContainerResources
	spec.UserLabels = make(map[string]string)

	for k, v := range cluster.Spec.UserLabels {
		spec.UserLabels[k] = v
	}
	spec.UserLabels[util.LABEL_PRIMARY] = "false"
	spec.UserLabels[util.LABEL_PG_CLUSTER] = cluster.Spec.Name

	newInstance := &crv1.Pgreplica{
		ObjectMeta: meta_v1.ObjectMeta{
			Name: uniqueName,
		},
		Spec: spec,
		Status: crv1.PgreplicaStatus{
			State:   crv1.PgreplicaStateCreated,
			Message: "Created, not processed yet",
		},
	}

	newInstance.ObjectMeta.Labels = make(map[string]string)
	for x, y := range cluster.ObjectMeta.Labels {
		newInstance.ObjectMeta.Labels[x] = y
	}
	newInstance.ObjectMeta.Labels[util.LABEL_PRIMARY] = "false"
	newInstance.ObjectMeta.Labels[util.LABEL_NAME] = uniqueName

	kubeapi.Createpgreplica(client, newInstance, operator.NAMESPACE)

}
<commit_msg>add logic to remove pgreplica as part of failover to reflect the promotion of the replica to primary<commit_after>// Package cluster holds the cluster CRD logic and definitions
// A cluster is comprised of a primary service, replica service,
// primary deployment, and replica deployment
package cluster

/*
 Copyright 2017-2018 Crunchy Data Solutions, Inc.
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

import (
	log "github.com/Sirupsen/logrus"
	crv1 "github.com/crunchydata/postgres-operator/apis/cr/v1"
	"github.com/crunchydata/postgres-operator/kubeapi"
	"github.com/crunchydata/postgres-operator/operator"
	"github.com/crunchydata/postgres-operator/util"
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
)

// FailoverBase ...
func FailoverBase(namespace string, clientset *kubernetes.Clientset, client *rest.RESTClient, task *crv1.Pgtask, restconfig *rest.Config) {
	var err error

	//look up the pgcluster for this task
	//in the case, the clustername is passed as a key in the
	//parameters map
	var clusterName string
	for k, _ := range task.Spec.Parameters {
		clusterName = k
	}

	cluster := crv1.Pgcluster{}
	_, err = kubeapi.Getpgcluster(client, &cluster,
		clusterName, namespace)
	if err != nil {
		return
	}

	if cluster.Spec.Strategy == "" {
		cluster.Spec.Strategy = "1"
		log.Info("using default strategy")
	}

	strategy, ok := strategyMap[cluster.Spec.Strategy]
	if ok {
		log.Info("strategy found")
	} else {
		log.Error("invalid Strategy requested for cluster failover " + cluster.Spec.Strategy)
		return
	}

	strategy.Failover(clientset, client, clusterName, task, namespace, restconfig)
	//remove the pgreplica CRD for the promoted replica
	kubeapi.Deletepgreplica(client, task.ObjectMeta.Labels[util.LABEL_TARGET], namespace)

	//scale up the replicas to replace the failover target
	replaceReplica(client, &cluster)

}

func replaceReplica(client *rest.RESTClient, cluster *crv1.Pgcluster) {

	//generate new replica name
	uniqueName := cluster.Spec.Name + "-" + util.RandStringBytesRmndr(4)

	spec := crv1.PgreplicaSpec{}
	spec.Name = uniqueName
	spec.ClusterName = cluster.Spec.Name
	spec.ReplicaStorage = cluster.Spec.ReplicaStorage
	spec.ContainerResources = cluster.Spec.ContainerResources
	spec.UserLabels = make(map[string]string)

	for k, v := range cluster.Spec.UserLabels {
		spec.UserLabels[k] = v
	}
	spec.UserLabels[util.LABEL_PRIMARY] = "false"
	spec.UserLabels[util.LABEL_PG_CLUSTER] = cluster.Spec.Name

	newInstance := &crv1.Pgreplica{
		ObjectMeta: meta_v1.ObjectMeta{
			Name: uniqueName,
		},
		Spec: spec,
		Status: crv1.PgreplicaStatus{
			State:   crv1.PgreplicaStateCreated,
			Message: "Created, not processed yet",
		},
	}

	newInstance.ObjectMeta.Labels = make(map[string]string)
	for x, y := range cluster.ObjectMeta.Labels {
		newInstance.ObjectMeta.Labels[x] = y
	}
	newInstance.ObjectMeta.Labels[util.LABEL_PRIMARY] = "false"
	newInstance.ObjectMeta.Labels[util.LABEL_NAME] = uniqueName

	kubeapi.Createpgreplica(client, newInstance, operator.NAMESPACE)

}
<|endoftext|>


<commit_before>/*

   Copyright 2016 Wenhui Shen <www.webx.top>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

*/
package tplfunc

import (
	"fmt"
	"html/template"
	"net/http"
	"strings"
	"time"

	captcha "github.com/webx-top/webx/lib/captcha"
	"github.com/webx-top/webx/lib/com"
)

var TplFuncMap template.FuncMap = template.FuncMap{
	"Now":             Now,
	"Eq":              Eq,
	"Add":             Add,
	"Sub":             Sub,
	"IsNil":           IsNil,
	"Html":            Html,
	"Js":              Js,
	"Css":             Css,
	"HtmlAttr":        HtmlAttr,
	"ToHtmlAttrs":     ToHtmlAttrs,
	"ElapsedMemory":   com.ElapsedMemory,  //内存消耗
	"TotalRunTime":    com.TotalRunTime,   //运行时长(从启动服务时算起)
	"CaptchaForm":     CaptchaForm,        //验证码图片
	"FormatByte":      com.FormatByte,     //字节转为适合理解的格式
	"FormatPastTime":  com.FormatPastTime, //以前距离现在多长时间
	"DateFormat":      com.DateFormat,
	"DateFormatShort": com.DateFormatShort,
	"Replace":         strings.Replace, //strings.Replace(s, old, new, n)
	"Contains":        strings.Contains,
	"HasPrefix":       strings.HasPrefix,
	"HasSuffix":       strings.HasSuffix,
	"Split":           strings.Split,
	"Join":            strings.Join,
	"Str":             com.Str,
	"Int":             com.Int,
	"Int32":           com.Int32,
	"Int64":           com.Int64,
	"Float32":         com.Float32,
	"Float64":         com.Float64,
	"InSlice":         com.InSlice,
	"InSliceI":        com.InSliceIface,
	"Substr":          com.Substr,
	"StripTags":       com.StripTags,
	"Default": func(defaultV interface{}, v interface{}) interface{} {
		if v == nil || com.Str(v) == "" {
			return defaultV
		}
		return v
	},
	"JsonEncode": com.SetJson,

	"Set": func(renderArgs map[string]interface{}, key string, value interface{}) template.HTML {
		renderArgs[key] = value
		return template.HTML("")
	},
	"Append": func(renderArgs map[string]interface{}, key string, value interface{}) template.HTML {
		if renderArgs[key] == nil {
			renderArgs[key] = []interface{}{value}
		} else {
			renderArgs[key] = append(renderArgs[key].([]interface{}), value)
		}
		return template.HTML("")
	},
	// Replaces newlines with

	"Nl2br": func(text string) template.HTML {
		return template.HTML(Nl2br(text))
	},
}

// 验证码表单域
func CaptchaForm(args ...string) template.HTML {
	var id string
	if len(args) == 0 {
		id = "captcha"
	} else {
		id = args[0]
	}
	captchaId := captcha.New()
	return template.HTML(fmt.Sprintf(`<img id="`+id+`Image" src="%scaptcha/%s.png" alt="Captcha image" onclick="this.src=this.src.split('?')[0]+'?reload='+Math.random();" /><input type="hidden" name="captchaId" id="`+id+`Id" value="%s" />`, `/`, captchaId, captchaId))
}

// 验证码验证
func CaptchaVerify(captchaSolution string, r *http.Request) bool {
	captchaId := r.FormValue("captchaId")
	if !captcha.VerifyString(captchaId, captchaSolution) {
		return false
	}
	return true
}

//将换行符替换为

func Nl2br(text string) string {
	return com.Nl2br(template.HTMLEscapeString(text))
}

func IsNil(a interface{}) bool {
	switch a.(type) {
	case nil:
		return true
	}
	return false
}

func Add(left interface{}, right interface{}) interface{} {
	var rleft, rright int64
	var fleft, fright float64
	var isInt bool = true
	switch left.(type) {
	case int:
		rleft = int64(left.(int))
	case int8:
		rleft = int64(left.(int8))
	case int16:
		rleft = int64(left.(int16))
	case int32:
		rleft = int64(left.(int32))
	case int64:
		rleft = left.(int64)
	case float32:
		fleft = float64(left.(float32))
		isInt = false
	case float64:
		fleft = left.(float64)
		isInt = false
	}

	switch right.(type) {
	case int:
		rright = int64(right.(int))
	case int8:
		rright = int64(right.(int8))
	case int16:
		rright = int64(right.(int16))
	case int32:
		rright = int64(right.(int32))
	case int64:
		rright = right.(int64)
	case float32:
		fright = float64(left.(float32))
		isInt = false
	case float64:
		fleft = left.(float64)
		isInt = false
	}

	var intSum int64 = rleft + rright

	if isInt {
		return intSum
	} else {
		return fleft + fright + float64(intSum)
	}
}

func Sub(left interface{}, right interface{}) interface{} {
	var rleft, rright int64
	var fleft, fright float64
	var isInt bool = true
	switch left.(type) {
	case int:
		rleft = int64(left.(int))
	case int8:
		rleft = int64(left.(int8))
	case int16:
		rleft = int64(left.(int16))
	case int32:
		rleft = int64(left.(int32))
	case int64:
		rleft = left.(int64)
	case float32:
		fleft = float64(left.(float32))
		isInt = false
	case float64:
		fleft = left.(float64)
		isInt = false
	}

	switch right.(type) {
	case int:
		rright = int64(right.(int))
	case int8:
		rright = int64(right.(int8))
	case int16:
		rright = int64(right.(int16))
	case int32:
		rright = int64(right.(int32))
	case int64:
		rright = right.(int64)
	case float32:
		fright = float64(left.(float32))
		isInt = false
	case float64:
		fleft = left.(float64)
		isInt = false
	}

	if isInt {
		return rleft - rright
	} else {
		return fleft + float64(rleft) - (fright + float64(rright))
	}
}

func Now() time.Time {
	return time.Now()
}

func Eq(left interface{}, right interface{}) bool {
	leftIsNil := (left == nil)
	rightIsNil := (right == nil)
	if leftIsNil || rightIsNil {
		if leftIsNil && rightIsNil {
			return true
		}
		return false
	}
	return fmt.Sprintf("%v", left) == fmt.Sprintf("%v", right)
}

func Html(raw string) template.HTML {
	return template.HTML(raw)
}

func HtmlAttr(raw string) template.HTMLAttr {
	return template.HTMLAttr(raw)
}

func ToHtmlAttrs(raw map[string]interface{}) (r map[template.HTMLAttr]interface{}) {
	r = make(map[template.HTMLAttr]interface{})
	for k, v := range raw {
		r[HtmlAttr(k)] = v
	}
	return
}

func Js(raw string) template.JS {
	return template.JS(raw)
}

func Css(raw string) template.CSS {
	return template.CSS(raw)
}
<commit_msg>update<commit_after>/*

   Copyright 2016 Wenhui Shen <www.webx.top>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

*/
package tplfunc

import (
	"fmt"
	"html/template"
	"net/http"
	"strings"
	"time"

	captcha "github.com/webx-top/webx/lib/captcha"
	"github.com/webx-top/webx/lib/com"
)

var TplFuncMap template.FuncMap = template.FuncMap{
	"Now":             Now,
	"Eq":              Eq,
	"Add":             Add,
	"Sub":             Sub,
	"IsNil":           IsNil,
	"Html":            Html,
	"Js":              Js,
	"Css":             Css,
	"HtmlAttr":        HtmlAttr,
	"ToHtmlAttrs":     ToHtmlAttrs,
	"ElapsedMemory":   com.ElapsedMemory,  //内存消耗
	"TotalRunTime":    com.TotalRunTime,   //运行时长(从启动服务时算起)
	"CaptchaForm":     CaptchaForm,        //验证码图片
	"FormatByte":      com.FormatByte,     //字节转为适合理解的格式
	"FormatPastTime":  com.FormatPastTime, //以前距离现在多长时间
	"DateFormat":      com.DateFormat,
	"DateFormatShort": com.DateFormatShort,
	"Replace":         strings.Replace, //strings.Replace(s, old, new, n)
	"Contains":        strings.Contains,
	"HasPrefix":       strings.HasPrefix,
	"HasSuffix":       strings.HasSuffix,
	"Split":           strings.Split,
	"Join":            strings.Join,
	"Str":             com.Str,
	"Int":             com.Int,
	"Int32":           com.Int32,
	"Int64":           com.Int64,
	"Float32":         com.Float32,
	"Float64":         com.Float64,
	"InSlice":         com.InSlice,
	"InSliceI":        com.InSliceIface,
	"Substr":          com.Substr,
	"StripTags":       com.StripTags,
	"Default": func(defaultV interface{}, v interface{}) interface{} {
		switch v.(type) {
		case nil:
			return defaultV
		case string:
			val, _ := v.(string)
			if val == `` {
				return defaultV
			}
		case uint8, int8, uint, int, uint32, int32, int64, uint64:
			if com.Int64(v) == 0 {
				return defaultV
			}
		case float32, float64:
			if com.Float64(v) == 0.0 {
				return defaultV
			}
		default:
			if com.Str(v) == `` {
				return defaultV
			}
		}
		return v
	},
	"JsonEncode": com.SetJson,

	"Set": func(renderArgs map[string]interface{}, key string, value interface{}) template.HTML {
		renderArgs[key] = value
		return template.HTML("")
	},
	"Append": func(renderArgs map[string]interface{}, key string, value interface{}) template.HTML {
		if renderArgs[key] == nil {
			renderArgs[key] = []interface{}{value}
		} else {
			renderArgs[key] = append(renderArgs[key].([]interface{}), value)
		}
		return template.HTML("")
	},
	// Replaces newlines with

	"Nl2br": func(text string) template.HTML {
		return template.HTML(Nl2br(text))
	},
}

// 验证码表单域
func CaptchaForm(args ...string) template.HTML {
	var id string
	if len(args) == 0 {
		id = "captcha"
	} else {
		id = args[0]
	}
	captchaId := captcha.New()
	return template.HTML(fmt.Sprintf(`<img id="`+id+`Image" src="%scaptcha/%s.png" alt="Captcha image" onclick="this.src=this.src.split('?')[0]+'?reload='+Math.random();" /><input type="hidden" name="captchaId" id="`+id+`Id" value="%s" />`, `/`, captchaId, captchaId))
}

// 验证码验证
func CaptchaVerify(captchaSolution string, r *http.Request) bool {
	captchaId := r.FormValue("captchaId")
	if !captcha.VerifyString(captchaId, captchaSolution) {
		return false
	}
	return true
}

//将换行符替换为

func Nl2br(text string) string {
	return com.Nl2br(template.HTMLEscapeString(text))
}

func IsNil(a interface{}) bool {
	switch a.(type) {
	case nil:
		return true
	}
	return false
}

func Add(left interface{}, right interface{}) interface{} {
	var rleft, rright int64
	var fleft, fright float64
	var isInt bool = true
	switch left.(type) {
	case int:
		rleft = int64(left.(int))
	case int8:
		rleft = int64(left.(int8))
	case int16:
		rleft = int64(left.(int16))
	case int32:
		rleft = int64(left.(int32))
	case int64:
		rleft = left.(int64)
	case float32:
		fleft = float64(left.(float32))
		isInt = false
	case float64:
		fleft = left.(float64)
		isInt = false
	}

	switch right.(type) {
	case int:
		rright = int64(right.(int))
	case int8:
		rright = int64(right.(int8))
	case int16:
		rright = int64(right.(int16))
	case int32:
		rright = int64(right.(int32))
	case int64:
		rright = right.(int64)
	case float32:
		fright = float64(left.(float32))
		isInt = false
	case float64:
		fleft = left.(float64)
		isInt = false
	}

	var intSum int64 = rleft + rright

	if isInt {
		return intSum
	} else {
		return fleft + fright + float64(intSum)
	}
}

func Sub(left interface{}, right interface{}) interface{} {
	var rleft, rright int64
	var fleft, fright float64
	var isInt bool = true
	switch left.(type) {
	case int:
		rleft = int64(left.(int))
	case int8:
		rleft = int64(left.(int8))
	case int16:
		rleft = int64(left.(int16))
	case int32:
		rleft = int64(left.(int32))
	case int64:
		rleft = left.(int64)
	case float32:
		fleft = float64(left.(float32))
		isInt = false
	case float64:
		fleft = left.(float64)
		isInt = false
	}

	switch right.(type) {
	case int:
		rright = int64(right.(int))
	case int8:
		rright = int64(right.(int8))
	case int16:
		rright = int64(right.(int16))
	case int32:
		rright = int64(right.(int32))
	case int64:
		rright = right.(int64)
	case float32:
		fright = float64(left.(float32))
		isInt = false
	case float64:
		fleft = left.(float64)
		isInt = false
	}

	if isInt {
		return rleft - rright
	} else {
		return fleft + float64(rleft) - (fright + float64(rright))
	}
}

func Now() time.Time {
	return time.Now()
}

func Eq(left interface{}, right interface{}) bool {
	leftIsNil := (left == nil)
	rightIsNil := (right == nil)
	if leftIsNil || rightIsNil {
		if leftIsNil && rightIsNil {
			return true
		}
		return false
	}
	return fmt.Sprintf("%v", left) == fmt.Sprintf("%v", right)
}

func Html(raw string) template.HTML {
	return template.HTML(raw)
}

func HtmlAttr(raw string) template.HTMLAttr {
	return template.HTMLAttr(raw)
}

func ToHtmlAttrs(raw map[string]interface{}) (r map[template.HTMLAttr]interface{}) {
	r = make(map[template.HTMLAttr]interface{})
	for k, v := range raw {
		r[HtmlAttr(k)] = v
	}
	return
}

func Js(raw string) template.JS {
	return template.JS(raw)
}

func Css(raw string) template.CSS {
	return template.CSS(raw)
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/jessevdk/go-flags"
	"os"
)

var parser = flags.NewNamedParser("docket", flags.Default)

var EXIT_BADARGS = 1
var EXIT_PANIC = 2

func main() {
	_, err := parser.Parse()
	if err != nil {
		os.Exit(EXIT_BADARGS)
	}
	os.Exit(0)
}

func init() {
	// parser.AddCommand(
	// 	"command",
	// 	"description",
	// 	"long description",
	// 	&whateverCmd{}
	// )
	parser.AddCommand(
		"run",
		"run a container",
		"run a container based on configuration in the current directory.",
		&runCmdOpts{},
	)
	parser.AddCommand(
		"build",
		"transform a container",
		"transform a container based on configuration in the current directory.",
		&buildCmdOpts{},
	)
}
<commit_msg>Enabled help flag.<commit_after>package main

import (
	"github.com/jessevdk/go-flags"
	"os"
)

var parser = flags.NewNamedParser("docket", flags.Default | flags.HelpFlag)

var EXIT_BADARGS = 1
var EXIT_PANIC = 2

func main() {
	_, err := parser.Parse()
	if err != nil {
		os.Exit(EXIT_BADARGS)
	}
	os.Exit(0)
}

func init() {
	// parser.AddCommand(
	// 	"command",
	// 	"description",
	// 	"long description",
	// 	&whateverCmd{}
	// )
	parser.AddCommand(
		"run",
		"run a container",
		"run a container based on configuration in the current directory.",
		&runCmdOpts{},
	)
	parser.AddCommand(
		"build",
		"transform a container",
		"transform a container based on configuration in the current directory.",
		&buildCmdOpts{},
	)
}
<|endoftext|>