<commit_before>package main

import (
        "fmt"
        "time"
        "strings"
        "strconv"
        "net"
        "log"
        "os/exec"
        "sync"
        "flag"
        "runtime"
        "github.com/vaughan0/go-ini"
)

var (
        inifile = flag.String("inifile","/etc/carbonmax.ini", "path to your ini config file")
        loop = flag.Bool("loop", false, "switch on if you want to loop the program for daemonization")
)

func iniParser(inifile string) (map[string]string, map[string]string){
        file, err := ini.LoadFile(inifile)
        if err != nil {
                log.Fatal(err)
        }
        return file["carbonlink"], file["resources"]
}

func feedcarbon(status map[string]string, carbonlink map[string]string) {

        conn, err := net.Dial("tcp", carbonlink["server"] + ":" + carbonlink["port"])
        if err != nil {
                log.Fatal("Can not connect the carbon-cache, Please check setting")
        }
        defer conn.Close()

        var message string
        for mn, ms := range status {
                message = message + fmt.Sprintf("%s.%s %s %d\n", carbonlink["client"], mn, ms, time.Now().Unix())
        }

        conn.Write([]byte(message))
        verbose, _ := strconv.ParseBool(carbonlink["verbose"])
        if verbose {
                fmt.Println(message)
        }
}

func cmdExec(name string, command string, timeout string, status map[string]string, wg *sync.WaitGroup) {
        defer wg.Done()
        ch := make(chan string, 1)
        to, _ := time.ParseDuration(timeout)
        go func() {
                result, _ := exec.Command("sh", "-c", command).Output()
                ch <- string(result)
        }()

        select {
        case result := <-ch:
                if result != "" {
                        res := strings.Trim(result,"\n")
                        status[name] = res
                }
        case <-time.After(to):
                log.Printf("%s execution timed out", name)
        }
}

func main() {
        flag.Parse()

        var wg sync.WaitGroup

        if *loop {
                log.Println("Now Looping Carbonmax at Given Interval")

                for {
                        carbonlink, resources := iniParser(*inifile)
                        cpus, err := strconv.Atoi(carbonlink["cpus"])
                        if err != nil {
                                runtime.GOMAXPROCS(1)
                        } else {
                                runtime.GOMAXPROCS(cpus)
                        }

                        status := make(map[string]string)

                        for k, v := range resources {
                                if k != "" && v != "" {
                                        wg.Add(1)
                                        go cmdExec(k, strings.Trim(v, "`"), carbonlink["exectimeout"], status, &wg)
                                }
                        }
                        wg.Wait()

                        feedcarbon(status, carbonlink)
                        interval, _ := time.ParseDuration(carbonlink["interval"])

                        time.Sleep(interval)
                }
        } else {
                carbonlink, resources := iniParser(*inifile)
                cpus, err := strconv.Atoi(carbonlink["cpus"])
                if err != nil {
                        runtime.GOMAXPROCS(1)
                } else {
                        runtime.GOMAXPROCS(cpus)
                }

                status := make(map[string]string)

                for k, v := range resources {
                        if k != "" && v != "" {
                                        wg.Add(1)
                                go cmdExec(k, strings.Trim(v, "`"), carbonlink["exectimeout"], status, &wg)
                        }
                }
                wg.Wait()

                feedcarbon(status, carbonlink)
        }
}
<commit_msg>fix a bug which connection issue will stop program<commit_after>package main

import (
        "fmt"
        "time"
        "strings"
        "strconv"
        "net"
        "log"
        "os/exec"
        "sync"
        "flag"
        "runtime"
        "github.com/vaughan0/go-ini"
)

var (
        inifile = flag.String("inifile","/etc/carbonmax.ini", "path to your ini config file")
        loop = flag.Bool("loop", false, "switch on if you want to loop the program for daemonization")
)

func iniParser(inifile string) (map[string]string, map[string]string){
        file, err := ini.LoadFile(inifile)
        if err != nil {
                log.Fatal(err)
        }
        return file["carbonlink"], file["resources"]
}

func feedcarbon(status map[string]string, carbonlink map[string]string) {

        conn, err := net.Dial("tcp", carbonlink["server"] + ":" + carbonlink["port"])
        if err != nil {
                log.Println("Can not connect the carbon-cache, Please check setting")
        }
        defer conn.Close()

        var message string
        for mn, ms := range status {
                message = message + fmt.Sprintf("%s.%s %s %d\n", carbonlink["client"], mn, ms, time.Now().Unix())
        }

        conn.Write([]byte(message))
        verbose, _ := strconv.ParseBool(carbonlink["verbose"])
        if verbose {
                fmt.Println(message)
        }
}

func cmdExec(name string, command string, timeout string, status map[string]string, wg *sync.WaitGroup) {
        defer wg.Done()
        ch := make(chan string, 1)
        to, _ := time.ParseDuration(timeout)
        go func() {
                result, _ := exec.Command("sh", "-c", command).Output()
                ch <- string(result)
        }()

        select {
        case result := <-ch:
                if result != "" {
                        res := strings.Trim(result,"\n")
                        status[name] = res
                }
        case <-time.After(to):
                log.Printf("%s execution timed out", name)
        }
}

func main() {
        flag.Parse()

        var wg sync.WaitGroup

        if *loop {
                log.Println("Now Looping Carbonmax at Given Interval")

                for {
                        carbonlink, resources := iniParser(*inifile)
                        cpus, err := strconv.Atoi(carbonlink["cpus"])
                        if err != nil {
                                runtime.GOMAXPROCS(1)
                        } else {
                                runtime.GOMAXPROCS(cpus)
                        }

                        status := make(map[string]string)

                        for k, v := range resources {
                                if k != "" && v != "" {
                                        wg.Add(1)
                                        go cmdExec(k, strings.Trim(v, "`"), carbonlink["exectimeout"], status, &wg)
                                }
                        }
                        wg.Wait()

                        feedcarbon(status, carbonlink)
                        interval, _ := time.ParseDuration(carbonlink["interval"])

                        time.Sleep(interval)
                }
        } else {
                carbonlink, resources := iniParser(*inifile)
                cpus, err := strconv.Atoi(carbonlink["cpus"])
                if err != nil {
                        runtime.GOMAXPROCS(1)
                } else {
                        runtime.GOMAXPROCS(cpus)
                }

                status := make(map[string]string)

                for k, v := range resources {
                        if k != "" && v != "" {
                                        wg.Add(1)
                                go cmdExec(k, strings.Trim(v, "`"), carbonlink["exectimeout"], status, &wg)
                        }
                }
                wg.Wait()

                feedcarbon(status, carbonlink)
        }
}
<|endoftext|>


<commit_before><commit_msg>Fix bug in LOCKED_RECT.SetAllBytes<commit_after><|endoftext|>


<commit_before>package main

import (
	"bufio"
	"crypto/x509"
	"encoding/pem"
	"flag"
	"fmt"
	"html"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

type certInfo struct {
	FileName string
	NotAfter time.Time
	CertName string
}

var pageHead = `

`

// wg is used to pevent a webrequest from attempting to return information
// before the async directory scanner is finished.
var wg sync.WaitGroup

// dir is the directory passed in by the user
var dir string

// certInfos will contain all of the information about certs we find.
var certInfos []certInfo

// walkFunc is the walking function called by each file and directory
// found in the flag specified directory.
func walkFunc(path string, info os.FileInfo, err error) (e error) {
	if info.IsDir() {
		return
	}
	f, e := os.Open(path)
	// If we can't open it, forget about it.
	if e != nil {
		return
	}

	// isOpen will tell you whether or not we have found a BEGIN or
	// if we found an END. TRUE if we have found a BEGIN but no END.
	// FALSE if we have found nothing or END and no BEGIN.
	isOpen := false

	// tryCert is what we think might be a certificate. We will try
	// to parse it as an x509 cert.
	tryCert := ""
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		text := scanner.Text()

		// Try to find certificates deep in files. Once we find a begin
		// certificate and an end certificate, try to parse it.
		if strings.Contains(text, "BEGIN CERTIFICATE-----") {
			tryCert = fmt.Sprintf("%s\n", strings.TrimSpace(text))
			isOpen = true
		} else if strings.Contains(text, "END CERTIFICATE-----") {
			// This is a peculiar state. We found an END without a begin.
			// Just ignore it and try again later.
			if !isOpen {
				tryCert = ""
			}
			tryCert = fmt.Sprintf("%s%s", tryCert, strings.TrimSpace(text))
			// Try to parse the certificate.
			p, _ := pem.Decode([]byte(tryCert))
			if p == nil {
				return
			}
			cert, err := x509.ParseCertificate(p.Bytes)
			if err == nil {
				c := *new(certInfo)
				c.NotAfter = cert.NotAfter
				c.FileName = path
				dnsNames := ""
				for _, dnsName := range cert.DNSNames {
					if len(dnsNames) == 0 {
						dnsNames = dnsName
					} else {
						dnsNames = fmt.Sprintf("%s, %s", dnsNames, dnsName)
					}
				}
				c.CertName = dnsNames
				certInfos = append(certInfos, c)
			}
			isOpen = false
		} else {
			if isOpen {
				tryCert = fmt.Sprintf("%s%s\n", tryCert, strings.TrimSpace(text))
			}
		}
	}
	return
}
func serveResults(w http.ResponseWriter, r *http.Request) {
	wg.Wait()
	w.Write([]byte(pageHead))
	for _, cert := range certInfos {
		certClass := ""
		monthAway := time.Now().AddDate(0, 1, 0)
		if cert.NotAfter.Before(monthAway) {
			certClass = "warning"
		}
		htmlRow := fmt.Sprintf("

%s

%s

%s

", certClass, cert.NotAfter, html.EscapeString(cert.FileName), html.EscapeString(cert.CertName))
		w.Write([]byte(htmlRow))
	}
	w.Write([]byte("
"))
}
func main() {
	flag.StringVar(&dir, "d", "", "search `directory` for certificates")
	flag.Parse()

	if len(dir) == 0 {
		log.Fatalf("Must supply a directory to search for ceritficates.")
	}
	go func() {
		for {
			wg.Add(1)
			certInfos = *new([]certInfo)

			// walk through each file or directory and accumulate all the certificates
			filepath.Walk(dir, walkFunc)
			wg.Done()
			time.Sleep(time.Minute)
		}
	}()
	http.HandleFunc("/", serveResults)
	http.ListenAndServe(":8000", nil)
}
<commit_msg>Don't loop this. Run from CI<commit_after>package main

import (
	"bufio"
	"crypto/x509"
	"encoding/pem"
	"flag"
	"fmt"
	"html"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

type certInfo struct {
	FileName string
	NotAfter time.Time
	CertName string
}

var pageHead = `

`

// wg is used to pevent a webrequest from attempting to return information
// before the async directory scanner is finished.
var wg sync.WaitGroup

// dir is the directory passed in by the user
var dir string

// certInfos will contain all of the information about certs we find.
var certInfos []certInfo

// walkFunc is the walking function called by each file and directory
// found in the flag specified directory.
func walkFunc(path string, info os.FileInfo, err error) (e error) {
	if info.IsDir() {
		return
	}
	f, e := os.Open(path)
	// If we can't open it, forget about it.
	if e != nil {
		return
	}

	// isOpen will tell you whether or not we have found a BEGIN or
	// if we found an END. TRUE if we have found a BEGIN but no END.
	// FALSE if we have found nothing or END and no BEGIN.
	isOpen := false

	// tryCert is what we think might be a certificate. We will try
	// to parse it as an x509 cert.
	tryCert := ""
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		text := scanner.Text()

		// Try to find certificates deep in files. Once we find a begin
		// certificate and an end certificate, try to parse it.
		if strings.Contains(text, "BEGIN CERTIFICATE-----") {
			tryCert = fmt.Sprintf("%s\n", strings.TrimSpace(text))
			isOpen = true
		} else if strings.Contains(text, "END CERTIFICATE-----") {
			// This is a peculiar state. We found an END without a begin.
			// Just ignore it and try again later.
			if !isOpen {
				tryCert = ""
			}
			tryCert = fmt.Sprintf("%s%s", tryCert, strings.TrimSpace(text))
			// Try to parse the certificate.
			p, _ := pem.Decode([]byte(tryCert))
			if p == nil {
				return
			}
			cert, err := x509.ParseCertificate(p.Bytes)
			if err == nil {
				c := *new(certInfo)
				c.NotAfter = cert.NotAfter
				c.FileName = path
				dnsNames := ""
				for _, dnsName := range cert.DNSNames {
					if len(dnsNames) == 0 {
						dnsNames = dnsName
					} else {
						dnsNames = fmt.Sprintf("%s, %s", dnsNames, dnsName)
					}
				}
				c.CertName = dnsNames
				certInfos = append(certInfos, c)
			}
			isOpen = false
		} else {
			if isOpen {
				tryCert = fmt.Sprintf("%s%s\n", tryCert, strings.TrimSpace(text))
			}
		}
	}
	return
}
func serveResults(w http.ResponseWriter, r *http.Request) {
	wg.Wait()
	w.Write([]byte(pageHead))
	for _, cert := range certInfos {
		certClass := ""
		monthAway := time.Now().AddDate(0, 1, 0)
		if cert.NotAfter.Before(monthAway) {
			certClass = "warning"
		}
		htmlRow := fmt.Sprintf("

%s

%s

%s

", certClass, cert.NotAfter, html.EscapeString(cert.FileName), html.EscapeString(cert.CertName))
		w.Write([]byte(htmlRow))
	}
	w.Write([]byte("
"))
}
func main() {
	flag.StringVar(&dir, "d", "", "search `directory` for certificates")
	flag.Parse()

	if len(dir) == 0 {
		log.Fatalf("Must supply a directory to search for ceritficates.")
	}

	wg.Add(1)
	certInfos = *new([]certInfo)

	// walk through each file or directory and accumulate all the certificates
	filepath.Walk(dir, walkFunc)
	wg.Done()

	http.HandleFunc("/", serveResults)
	http.ListenAndServe(":30080", nil)
}
<|endoftext|>


<commit_before>// Copyright (c) 2013-2016 The btcsuite developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package chain

import (
	"errors"
	"sync"
	"time"

	"github.com/btcsuite/btcd/btcjson"
	"github.com/btcsuite/btcd/chaincfg"
	"github.com/btcsuite/btcd/chaincfg/chainhash"
	"github.com/btcsuite/btcd/rpcclient"
	"github.com/btcsuite/btcd/wire"
	"github.com/btcsuite/btcutil"
	"github.com/btcsuite/btcutil/gcs"
	"github.com/btcsuite/btcutil/gcs/builder"
	"github.com/btcsuite/btcwallet/waddrmgr"
	"github.com/btcsuite/btcwallet/wtxmgr"
)

// RPCClient represents a persistent client connection to a bitcoin RPC server
// for information regarding the current best block chain.
type RPCClient struct {
	*rpcclient.Client
	connConfig        *rpcclient.ConnConfig // Work around unexported field
	chainParams       *chaincfg.Params
	reconnectAttempts int

	enqueueNotification chan interface{}
	dequeueNotification chan interface{}
	currentBlock        chan *waddrmgr.BlockStamp

	quit    chan struct{}
	wg      sync.WaitGroup
	started bool
	quitMtx sync.Mutex
}

// NewRPCClient creates a client connection to the server described by the
// connect string.  If disableTLS is false, the remote RPC certificate must be
// provided in the certs slice.  The connection is not established immediately,
// but must be done using the Start method.  If the remote server does not
// operate on the same bitcoin network as described by the passed chain
// parameters, the connection will be disconnected.
func NewRPCClient(chainParams *chaincfg.Params, connect, user, pass string, certs []byte,
	disableTLS bool, reconnectAttempts int) (*RPCClient, error) {

	if reconnectAttempts < 0 {
		return nil, errors.New("reconnectAttempts must be positive")
	}

	client := &RPCClient{
		connConfig: &rpcclient.ConnConfig{
			Host:                 connect,
			Endpoint:             "ws",
			User:                 user,
			Pass:                 pass,
			Certificates:         certs,
			DisableAutoReconnect: false,
			DisableConnectOnNew:  true,
			DisableTLS:           disableTLS,
		},
		chainParams:         chainParams,
		reconnectAttempts:   reconnectAttempts,
		enqueueNotification: make(chan interface{}),
		dequeueNotification: make(chan interface{}),
		currentBlock:        make(chan *waddrmgr.BlockStamp),
		quit:                make(chan struct{}),
	}
	ntfnCallbacks := &rpcclient.NotificationHandlers{
		OnClientConnected:   client.onClientConnect,
		OnBlockConnected:    client.onBlockConnected,
		OnBlockDisconnected: client.onBlockDisconnected,
		OnRecvTx:            client.onRecvTx,
		OnRedeemingTx:       client.onRedeemingTx,
		OnRescanFinished:    client.onRescanFinished,
		OnRescanProgress:    client.onRescanProgress,
	}
	rpcClient, err := rpcclient.New(client.connConfig, ntfnCallbacks)
	if err != nil {
		return nil, err
	}
	client.Client = rpcClient
	return client, nil
}

// BackEnd returns the name of the driver.
func (c *RPCClient) BackEnd() string {
	return "btcd"
}

// Start attempts to establish a client connection with the remote server.
// If successful, handler goroutines are started to process notifications
// sent by the server.  After a limited number of connection attempts, this
// function gives up, and therefore will not block forever waiting for the
// connection to be established to a server that may not exist.
func (c *RPCClient) Start() error {
	err := c.Connect(c.reconnectAttempts)
	if err != nil {
		return err
	}

	// Verify that the server is running on the expected network.
	net, err := c.GetCurrentNet()
	if err != nil {
		c.Disconnect()
		return err
	}
	if net != c.chainParams.Net {
		c.Disconnect()
		return errors.New("mismatched networks")
	}

	c.quitMtx.Lock()
	c.started = true
	c.quitMtx.Unlock()

	c.wg.Add(1)
	go c.handler()
	return nil
}

// Stop disconnects the client and signals the shutdown of all goroutines
// started by Start.
func (c *RPCClient) Stop() {
	c.quitMtx.Lock()
	select {
	case <-c.quit:
	default:
		close(c.quit)
		c.Client.Shutdown()

		if !c.started {
			close(c.dequeueNotification)
		}
	}
	c.quitMtx.Unlock()
}

// Rescan wraps the normal Rescan command with an additional paramter that
// allows us to map an oupoint to the address in the chain that it pays to.
// This is useful when using BIP 158 filters as they include the prev pkScript
// rather than the full outpoint.
func (c *RPCClient) Rescan(startHash *chainhash.Hash, addrs []btcutil.Address,
	outPoints map[wire.OutPoint]btcutil.Address) error {

	flatOutpoints := make([]*wire.OutPoint, 0, len(outPoints))
	for ops := range outPoints {
		flatOutpoints = append(flatOutpoints, &ops)
	}

	return c.Client.Rescan(startHash, addrs, flatOutpoints)
}

// WaitForShutdown blocks until both the client has finished disconnecting
// and all handlers have exited.
func (c *RPCClient) WaitForShutdown() {
	c.Client.WaitForShutdown()
	c.wg.Wait()
}

// Notifications returns a channel of parsed notifications sent by the remote
// bitcoin RPC server.  This channel must be continually read or the process
// may abort for running out memory, as unread notifications are queued for
// later reads.
func (c *RPCClient) Notifications() <-chan interface{} {
	return c.dequeueNotification
}

// BlockStamp returns the latest block notified by the client, or an error
// if the client has been shut down.
func (c *RPCClient) BlockStamp() (*waddrmgr.BlockStamp, error) {
	select {
	case bs := <-c.currentBlock:
		return bs, nil
	case <-c.quit:
		return nil, errors.New("disconnected")
	}
}

// FilterBlocks scans the blocks contained in the FilterBlocksRequest for any
// addresses of interest. For each requested block, the corresponding compact
// filter will first be checked for matches, skipping those that do not report
// anything. If the filter returns a postive match, the full block will be
// fetched and filtered. This method returns a FilterBlocksReponse for the first
// block containing a matching address. If no matches are found in the range of
// blocks requested, the returned response will be nil.
func (c *RPCClient) FilterBlocks(
	req *FilterBlocksRequest) (*FilterBlocksResponse, error) {

	blockFilterer := NewBlockFilterer(c.chainParams, req)

	// Construct the watchlist using the addresses and outpoints contained
	// in the filter blocks request.
	watchList, err := buildFilterBlocksWatchList(req)
	if err != nil {
		return nil, err
	}

	// Iterate over the requested blocks, fetching the compact filter for
	// each one, and matching it against the watchlist generated above. If
	// the filter returns a positive match, the full block is then requested
	// and scanned for addresses using the block filterer.
	for i, blk := range req.Blocks {
		rawFilter, err := c.GetCFilter(&blk.Hash, wire.GCSFilterRegular)
		if err != nil {
			return nil, err
		}

		// Ensure the filter is large enough to be deserialized.
		if len(rawFilter.Data) < 4 {
			continue
		}

		filter, err := gcs.FromNBytes(
			builder.DefaultP, builder.DefaultM, rawFilter.Data,
		)
		if err != nil {
			return nil, err
		}

		// Skip any empty filters.
		if filter.N() == 0 {
			continue
		}

		key := builder.DeriveKey(&blk.Hash)
		matched, err := filter.MatchAny(key, watchList)
		if err != nil {
			return nil, err
		} else if !matched {
			continue
		}

		log.Infof("Fetching block height=%d hash=%v",
			blk.Height, blk.Hash)

		rawBlock, err := c.GetBlock(&blk.Hash)
		if err != nil {
			return nil, err
		}

		if !blockFilterer.FilterBlock(rawBlock) {
			continue
		}

		// If any external or internal addresses were detected in this
		// block, we return them to the caller so that the rescan
		// windows can widened with subsequent addresses. The
		// `BatchIndex` is returned so that the caller can compute the
		// *next* block from which to begin again.
		resp := &FilterBlocksResponse{
			BatchIndex:         uint32(i),
			BlockMeta:          blk,
			FoundExternalAddrs: blockFilterer.FoundExternal,
			FoundInternalAddrs: blockFilterer.FoundInternal,
			FoundOutPoints:     blockFilterer.FoundOutPoints,
			RelevantTxns:       blockFilterer.RelevantTxns,
		}

		return resp, nil
	}

	// No addresses were found for this range.
	return nil, nil
}

// parseBlock parses a btcws definition of the block a tx is mined it to the
// Block structure of the wtxmgr package, and the block index.  This is done
// here since rpcclient doesn't parse this nicely for us.
func parseBlock(block *btcjson.BlockDetails) (*wtxmgr.BlockMeta, error) {
	if block == nil {
		return nil, nil
	}
	blkHash, err := chainhash.NewHashFromStr(block.Hash)
	if err != nil {
		return nil, err
	}
	blk := &wtxmgr.BlockMeta{
		Block: wtxmgr.Block{
			Height: block.Height,
			Hash:   *blkHash,
		},
		Time: time.Unix(block.Time, 0),
	}
	return blk, nil
}

func (c *RPCClient) onClientConnect() {
	select {
	case c.enqueueNotification <- ClientConnected{}:
	case <-c.quit:
	}
}

func (c *RPCClient) onBlockConnected(hash *chainhash.Hash, height int32, time time.Time) {
	select {
	case c.enqueueNotification <- BlockConnected{
		Block: wtxmgr.Block{
			Hash:   *hash,
			Height: height,
		},
		Time: time,
	}:
	case <-c.quit:
	}
}

func (c *RPCClient) onBlockDisconnected(hash *chainhash.Hash, height int32, time time.Time) {
	select {
	case c.enqueueNotification <- BlockDisconnected{
		Block: wtxmgr.Block{
			Hash:   *hash,
			Height: height,
		},
		Time: time,
	}:
	case <-c.quit:
	}
}

func (c *RPCClient) onRecvTx(tx *btcutil.Tx, block *btcjson.BlockDetails) {
	blk, err := parseBlock(block)
	if err != nil {
		// Log and drop improper notification.
		log.Errorf("recvtx notification bad block: %v", err)
		return
	}

	rec, err := wtxmgr.NewTxRecordFromMsgTx(tx.MsgTx(), time.Now())
	if err != nil {
		log.Errorf("Cannot create transaction record for relevant "+
			"tx: %v", err)
		return
	}
	select {
	case c.enqueueNotification <- RelevantTx{rec, blk}:
	case <-c.quit:
	}
}

func (c *RPCClient) onRedeemingTx(tx *btcutil.Tx, block *btcjson.BlockDetails) {
	// Handled exactly like recvtx notifications.
	c.onRecvTx(tx, block)
}

func (c *RPCClient) onRescanProgress(hash *chainhash.Hash, height int32, blkTime time.Time) {
	select {
	case c.enqueueNotification <- &RescanProgress{hash, height, blkTime}:
	case <-c.quit:
	}
}

func (c *RPCClient) onRescanFinished(hash *chainhash.Hash, height int32, blkTime time.Time) {
	select {
	case c.enqueueNotification <- &RescanFinished{hash, height, blkTime}:
	case <-c.quit:
	}

}

// handler maintains a queue of notifications and the current state (best
// block) of the chain.
func (c *RPCClient) handler() {
	hash, height, err := c.GetBestBlock()
	if err != nil {
		log.Errorf("Failed to receive best block from chain server: %v", err)
		c.Stop()
		c.wg.Done()
		return
	}

	bs := &waddrmgr.BlockStamp{Hash: *hash, Height: height}

	// TODO: Rather than leaving this as an unbounded queue for all types of
	// notifications, try dropping ones where a later enqueued notification
	// can fully invalidate one waiting to be processed.  For example,
	// blockconnected notifications for greater block heights can remove the
	// need to process earlier blockconnected notifications still waiting
	// here.

	var notifications []interface{}
	enqueue := c.enqueueNotification
	var dequeue chan interface{}
	var next interface{}
	pingChan := time.After(time.Minute)
	pingChanReset := make(chan (<-chan time.Time))
out:
	for {
		select {
		case n, ok := <-enqueue:
			if !ok {
				// If no notifications are queued for handling,
				// the queue is finished.
				if len(notifications) == 0 {
					break out
				}
				// nil channel so no more reads can occur.
				enqueue = nil
				continue
			}
			if len(notifications) == 0 {
				next = n
				dequeue = c.dequeueNotification
			}
			notifications = append(notifications, n)
			pingChan = time.After(time.Minute)

		case dequeue <- next:
			if n, ok := next.(BlockConnected); ok {
				bs = &waddrmgr.BlockStamp{
					Height: n.Height,
					Hash:   n.Hash,
				}
			}

			notifications[0] = nil
			notifications = notifications[1:]
			if len(notifications) != 0 {
				next = notifications[0]
			} else {
				// If no more notifications can be enqueued, the
				// queue is finished.
				if enqueue == nil {
					break out
				}
				dequeue = nil
			}

		case <-pingChan:
			// No notifications were received in the last 60s. Ensure the
			// connection is still active by making a new request to the server.
			//
			// This MUST wait for the response in a new goroutine so as to not
			// block channel sends enqueueing more notifications.  Doing so
			// would cause a deadlock and after the timeout expires, the client
			// would be shut down.
			//
			// TODO: A minute timeout is used to prevent the handler loop from
			// blocking here forever, but this is much larger than it needs to
			// be due to dcrd processing websocket requests synchronously (see
			// https://github.com/btcsuite/btcd/issues/504).  Decrease this to
			// something saner like 3s when the above issue is fixed.
			type sessionResult struct {
				err error
			}
			sessionResponse := make(chan sessionResult, 1)
			go func() {
				_, err := c.Session()
				sessionResponse <- sessionResult{err}
			}()
			go func() {
				select {
				case resp := <-sessionResponse:
					if resp.err != nil {
						log.Errorf("Failed to receive session "+
							"result: %v", resp.err)
						c.Stop()
					}
					pingChanReset <- time.After(time.Minute)

				case <-time.After(time.Minute):
					log.Errorf("Timeout waiting for session RPC")
					c.Stop()
				}
			}()

		case ch := <-pingChanReset:
			pingChan = ch

		case c.currentBlock <- bs:

		case <-c.quit:
			break out
		}
	}

	c.Stop()
	close(c.dequeueNotification)
	c.wg.Done()
}

// POSTClient creates the equivalent HTTP POST rpcclient.Client.
func (c *RPCClient) POSTClient() (*rpcclient.Client, error) {
	configCopy := *c.connConfig
	configCopy.HTTPPostMode = true
	return rpcclient.New(&configCopy, nil)
}
<commit_msg>chain/rpc: remove unnecessary ping keep alive<commit_after>// Copyright (c) 2013-2016 The btcsuite developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package chain

import (
	"errors"
	"sync"
	"time"

	"github.com/btcsuite/btcd/btcjson"
	"github.com/btcsuite/btcd/chaincfg"
	"github.com/btcsuite/btcd/chaincfg/chainhash"
	"github.com/btcsuite/btcd/rpcclient"
	"github.com/btcsuite/btcd/wire"
	"github.com/btcsuite/btcutil"
	"github.com/btcsuite/btcutil/gcs"
	"github.com/btcsuite/btcutil/gcs/builder"
	"github.com/btcsuite/btcwallet/waddrmgr"
	"github.com/btcsuite/btcwallet/wtxmgr"
)

// RPCClient represents a persistent client connection to a bitcoin RPC server
// for information regarding the current best block chain.
type RPCClient struct {
	*rpcclient.Client
	connConfig        *rpcclient.ConnConfig // Work around unexported field
	chainParams       *chaincfg.Params
	reconnectAttempts int

	enqueueNotification chan interface{}
	dequeueNotification chan interface{}
	currentBlock        chan *waddrmgr.BlockStamp

	quit    chan struct{}
	wg      sync.WaitGroup
	started bool
	quitMtx sync.Mutex
}

// NewRPCClient creates a client connection to the server described by the
// connect string.  If disableTLS is false, the remote RPC certificate must be
// provided in the certs slice.  The connection is not established immediately,
// but must be done using the Start method.  If the remote server does not
// operate on the same bitcoin network as described by the passed chain
// parameters, the connection will be disconnected.
func NewRPCClient(chainParams *chaincfg.Params, connect, user, pass string, certs []byte,
	disableTLS bool, reconnectAttempts int) (*RPCClient, error) {

	if reconnectAttempts < 0 {
		return nil, errors.New("reconnectAttempts must be positive")
	}

	client := &RPCClient{
		connConfig: &rpcclient.ConnConfig{
			Host:                 connect,
			Endpoint:             "ws",
			User:                 user,
			Pass:                 pass,
			Certificates:         certs,
			DisableAutoReconnect: false,
			DisableConnectOnNew:  true,
			DisableTLS:           disableTLS,
		},
		chainParams:         chainParams,
		reconnectAttempts:   reconnectAttempts,
		enqueueNotification: make(chan interface{}),
		dequeueNotification: make(chan interface{}),
		currentBlock:        make(chan *waddrmgr.BlockStamp),
		quit:                make(chan struct{}),
	}
	ntfnCallbacks := &rpcclient.NotificationHandlers{
		OnClientConnected:   client.onClientConnect,
		OnBlockConnected:    client.onBlockConnected,
		OnBlockDisconnected: client.onBlockDisconnected,
		OnRecvTx:            client.onRecvTx,
		OnRedeemingTx:       client.onRedeemingTx,
		OnRescanFinished:    client.onRescanFinished,
		OnRescanProgress:    client.onRescanProgress,
	}
	rpcClient, err := rpcclient.New(client.connConfig, ntfnCallbacks)
	if err != nil {
		return nil, err
	}
	client.Client = rpcClient
	return client, nil
}

// BackEnd returns the name of the driver.
func (c *RPCClient) BackEnd() string {
	return "btcd"
}

// Start attempts to establish a client connection with the remote server.
// If successful, handler goroutines are started to process notifications
// sent by the server.  After a limited number of connection attempts, this
// function gives up, and therefore will not block forever waiting for the
// connection to be established to a server that may not exist.
func (c *RPCClient) Start() error {
	err := c.Connect(c.reconnectAttempts)
	if err != nil {
		return err
	}

	// Verify that the server is running on the expected network.
	net, err := c.GetCurrentNet()
	if err != nil {
		c.Disconnect()
		return err
	}
	if net != c.chainParams.Net {
		c.Disconnect()
		return errors.New("mismatched networks")
	}

	c.quitMtx.Lock()
	c.started = true
	c.quitMtx.Unlock()

	c.wg.Add(1)
	go c.handler()
	return nil
}

// Stop disconnects the client and signals the shutdown of all goroutines
// started by Start.
func (c *RPCClient) Stop() {
	c.quitMtx.Lock()
	select {
	case <-c.quit:
	default:
		close(c.quit)
		c.Client.Shutdown()

		if !c.started {
			close(c.dequeueNotification)
		}
	}
	c.quitMtx.Unlock()
}

// Rescan wraps the normal Rescan command with an additional paramter that
// allows us to map an oupoint to the address in the chain that it pays to.
// This is useful when using BIP 158 filters as they include the prev pkScript
// rather than the full outpoint.
func (c *RPCClient) Rescan(startHash *chainhash.Hash, addrs []btcutil.Address,
	outPoints map[wire.OutPoint]btcutil.Address) error {

	flatOutpoints := make([]*wire.OutPoint, 0, len(outPoints))
	for ops := range outPoints {
		flatOutpoints = append(flatOutpoints, &ops)
	}

	return c.Client.Rescan(startHash, addrs, flatOutpoints)
}

// WaitForShutdown blocks until both the client has finished disconnecting
// and all handlers have exited.
func (c *RPCClient) WaitForShutdown() {
	c.Client.WaitForShutdown()
	c.wg.Wait()
}

// Notifications returns a channel of parsed notifications sent by the remote
// bitcoin RPC server.  This channel must be continually read or the process
// may abort for running out memory, as unread notifications are queued for
// later reads.
func (c *RPCClient) Notifications() <-chan interface{} {
	return c.dequeueNotification
}

// BlockStamp returns the latest block notified by the client, or an error
// if the client has been shut down.
func (c *RPCClient) BlockStamp() (*waddrmgr.BlockStamp, error) {
	select {
	case bs := <-c.currentBlock:
		return bs, nil
	case <-c.quit:
		return nil, errors.New("disconnected")
	}
}

// FilterBlocks scans the blocks contained in the FilterBlocksRequest for any
// addresses of interest. For each requested block, the corresponding compact
// filter will first be checked for matches, skipping those that do not report
// anything. If the filter returns a postive match, the full block will be
// fetched and filtered. This method returns a FilterBlocksReponse for the first
// block containing a matching address. If no matches are found in the range of
// blocks requested, the returned response will be nil.
func (c *RPCClient) FilterBlocks(
	req *FilterBlocksRequest) (*FilterBlocksResponse, error) {

	blockFilterer := NewBlockFilterer(c.chainParams, req)

	// Construct the watchlist using the addresses and outpoints contained
	// in the filter blocks request.
	watchList, err := buildFilterBlocksWatchList(req)
	if err != nil {
		return nil, err
	}

	// Iterate over the requested blocks, fetching the compact filter for
	// each one, and matching it against the watchlist generated above. If
	// the filter returns a positive match, the full block is then requested
	// and scanned for addresses using the block filterer.
	for i, blk := range req.Blocks {
		rawFilter, err := c.GetCFilter(&blk.Hash, wire.GCSFilterRegular)
		if err != nil {
			return nil, err
		}

		// Ensure the filter is large enough to be deserialized.
		if len(rawFilter.Data) < 4 {
			continue
		}

		filter, err := gcs.FromNBytes(
			builder.DefaultP, builder.DefaultM, rawFilter.Data,
		)
		if err != nil {
			return nil, err
		}

		// Skip any empty filters.
		if filter.N() == 0 {
			continue
		}

		key := builder.DeriveKey(&blk.Hash)
		matched, err := filter.MatchAny(key, watchList)
		if err != nil {
			return nil, err
		} else if !matched {
			continue
		}

		log.Infof("Fetching block height=%d hash=%v",
			blk.Height, blk.Hash)

		rawBlock, err := c.GetBlock(&blk.Hash)
		if err != nil {
			return nil, err
		}

		if !blockFilterer.FilterBlock(rawBlock) {
			continue
		}

		// If any external or internal addresses were detected in this
		// block, we return them to the caller so that the rescan
		// windows can widened with subsequent addresses. The
		// `BatchIndex` is returned so that the caller can compute the
		// *next* block from which to begin again.
		resp := &FilterBlocksResponse{
			BatchIndex:         uint32(i),
			BlockMeta:          blk,
			FoundExternalAddrs: blockFilterer.FoundExternal,
			FoundInternalAddrs: blockFilterer.FoundInternal,
			FoundOutPoints:     blockFilterer.FoundOutPoints,
			RelevantTxns:       blockFilterer.RelevantTxns,
		}

		return resp, nil
	}

	// No addresses were found for this range.
	return nil, nil
}

// parseBlock parses a btcws definition of the block a tx is mined it to the
// Block structure of the wtxmgr package, and the block index.  This is done
// here since rpcclient doesn't parse this nicely for us.
func parseBlock(block *btcjson.BlockDetails) (*wtxmgr.BlockMeta, error) {
	if block == nil {
		return nil, nil
	}
	blkHash, err := chainhash.NewHashFromStr(block.Hash)
	if err != nil {
		return nil, err
	}
	blk := &wtxmgr.BlockMeta{
		Block: wtxmgr.Block{
			Height: block.Height,
			Hash:   *blkHash,
		},
		Time: time.Unix(block.Time, 0),
	}
	return blk, nil
}

func (c *RPCClient) onClientConnect() {
	select {
	case c.enqueueNotification <- ClientConnected{}:
	case <-c.quit:
	}
}

func (c *RPCClient) onBlockConnected(hash *chainhash.Hash, height int32, time time.Time) {
	select {
	case c.enqueueNotification <- BlockConnected{
		Block: wtxmgr.Block{
			Hash:   *hash,
			Height: height,
		},
		Time: time,
	}:
	case <-c.quit:
	}
}

func (c *RPCClient) onBlockDisconnected(hash *chainhash.Hash, height int32, time time.Time) {
	select {
	case c.enqueueNotification <- BlockDisconnected{
		Block: wtxmgr.Block{
			Hash:   *hash,
			Height: height,
		},
		Time: time,
	}:
	case <-c.quit:
	}
}

func (c *RPCClient) onRecvTx(tx *btcutil.Tx, block *btcjson.BlockDetails) {
	blk, err := parseBlock(block)
	if err != nil {
		// Log and drop improper notification.
		log.Errorf("recvtx notification bad block: %v", err)
		return
	}

	rec, err := wtxmgr.NewTxRecordFromMsgTx(tx.MsgTx(), time.Now())
	if err != nil {
		log.Errorf("Cannot create transaction record for relevant "+
			"tx: %v", err)
		return
	}
	select {
	case c.enqueueNotification <- RelevantTx{rec, blk}:
	case <-c.quit:
	}
}

func (c *RPCClient) onRedeemingTx(tx *btcutil.Tx, block *btcjson.BlockDetails) {
	// Handled exactly like recvtx notifications.
	c.onRecvTx(tx, block)
}

func (c *RPCClient) onRescanProgress(hash *chainhash.Hash, height int32, blkTime time.Time) {
	select {
	case c.enqueueNotification <- &RescanProgress{hash, height, blkTime}:
	case <-c.quit:
	}
}

func (c *RPCClient) onRescanFinished(hash *chainhash.Hash, height int32, blkTime time.Time) {
	select {
	case c.enqueueNotification <- &RescanFinished{hash, height, blkTime}:
	case <-c.quit:
	}

}

// handler maintains a queue of notifications and the current state (best
// block) of the chain.
func (c *RPCClient) handler() {
	hash, height, err := c.GetBestBlock()
	if err != nil {
		log.Errorf("Failed to receive best block from chain server: %v", err)
		c.Stop()
		c.wg.Done()
		return
	}

	bs := &waddrmgr.BlockStamp{Hash: *hash, Height: height}

	// TODO: Rather than leaving this as an unbounded queue for all types of
	// notifications, try dropping ones where a later enqueued notification
	// can fully invalidate one waiting to be processed.  For example,
	// blockconnected notifications for greater block heights can remove the
	// need to process earlier blockconnected notifications still waiting
	// here.

	var notifications []interface{}
	enqueue := c.enqueueNotification
	var dequeue chan interface{}
	var next interface{}
out:
	for {
		select {
		case n, ok := <-enqueue:
			if !ok {
				// If no notifications are queued for handling,
				// the queue is finished.
				if len(notifications) == 0 {
					break out
				}
				// nil channel so no more reads can occur.
				enqueue = nil
				continue
			}
			if len(notifications) == 0 {
				next = n
				dequeue = c.dequeueNotification
			}
			notifications = append(notifications, n)

		case dequeue <- next:
			if n, ok := next.(BlockConnected); ok {
				bs = &waddrmgr.BlockStamp{
					Height: n.Height,
					Hash:   n.Hash,
				}
			}

			notifications[0] = nil
			notifications = notifications[1:]
			if len(notifications) != 0 {
				next = notifications[0]
			} else {
				// If no more notifications can be enqueued, the
				// queue is finished.
				if enqueue == nil {
					break out
				}
				dequeue = nil
			}

		case c.currentBlock <- bs:

		case <-c.quit:
			break out
		}
	}

	c.Stop()
	close(c.dequeueNotification)
	c.wg.Done()
}

// POSTClient creates the equivalent HTTP POST rpcclient.Client.
func (c *RPCClient) POSTClient() (*rpcclient.Client, error) {
	configCopy := *c.connConfig
	configCopy.HTTPPostMode = true
	return rpcclient.New(&configCopy, nil)
}
<|endoftext|>


<commit_before><commit_msg>vcs: generate a version.hwaf suitable for gaudi<commit_after><|endoftext|>


<commit_before>// Copyright 2019 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package expensivequery

import (
	"fmt"
	"strconv"
	"strings"
	"sync/atomic"
	"time"

	"github.com/pingcap/log"
	"github.com/pingcap/tidb/sessionctx/variable"
	"github.com/pingcap/tidb/util"
	"github.com/pingcap/tidb/util/logutil"
	"go.uber.org/zap"
	"go.uber.org/zap/zapcore"
)

// Handle is the handler for expensive query.
type Handle struct {
	exitCh chan struct{}
	sm     util.SessionManager
}

// NewExpensiveQueryHandle builds a new expensive query handler.
func NewExpensiveQueryHandle(exitCh chan struct{}) *Handle {
	return &Handle{exitCh: exitCh}
}

// SetSessionManager sets the SessionManager which is used to fetching the info
// of all active sessions.
func (eqh *Handle) SetSessionManager(sm util.SessionManager) *Handle {
	eqh.sm = sm
	return eqh
}

// Run starts a expensive query checker goroutine at the start time of the server.
func (eqh *Handle) Run() {
	threshold := atomic.LoadUint64(&variable.ExpensiveQueryTimeThreshold)
	// use 100ms as tickInterval temply, may use given interval or use defined variable later
	tickInterval := time.Millisecond * time.Duration(100)
	ticker := time.NewTicker(tickInterval)
	defer ticker.Stop()
	for {
		select {
		case <-ticker.C:
			processInfo := eqh.sm.ShowProcessList()
			for _, info := range processInfo {
				if len(info.Info) == 0 || info.ExceedExpensiveTimeThresh {
					continue
				}
				costTime := time.Since(info.Time)
				if costTime >= time.Second*time.Duration(threshold) && log.GetLevel() <= zapcore.WarnLevel {
					logExpensiveQuery(costTime, info)
					info.ExceedExpensiveTimeThresh = true

				} else if info.MaxExecutionTime > 0 && costTime > time.Duration(info.MaxExecutionTime)*time.Millisecond {
					eqh.sm.Kill(info.ID, true)
				}
			}
			threshold = atomic.LoadUint64(&variable.ExpensiveQueryTimeThreshold)
		case <-eqh.exitCh:
			return
		}
	}
}

// LogOnQueryExceedMemQuota prints a log when memory usage of connID is out of memory quota.
func (eqh *Handle) LogOnQueryExceedMemQuota(connID uint64) {
	if log.GetLevel() > zapcore.WarnLevel {
		return
	}
	info, ok := eqh.sm.GetProcessInfo(connID)
	if !ok {
		return
	}
	logExpensiveQuery(time.Since(info.Time), info)
}

func genLogFields(costTime time.Duration, info *util.ProcessInfo) []zap.Field {
	logFields := make([]zap.Field, 0, 20)
	logFields = append(logFields, zap.String("cost_time", strconv.FormatFloat(costTime.Seconds(), 'f', -1, 64)+"s"))
	execDetail := info.StmtCtx.GetExecDetails()
	logFields = append(logFields, execDetail.ToZapFields()...)
	if copTaskInfo := info.StmtCtx.CopTasksDetails(); copTaskInfo != nil {
		logFields = append(logFields, copTaskInfo.ToZapFields()...)
	}
	if statsInfo := info.StatsInfo(info.Plan); len(statsInfo) > 0 {
		var buf strings.Builder
		firstComma := false
		vStr := ""
		for k, v := range statsInfo {
			if v == 0 {
				vStr = "pseudo"
			} else {
				vStr = strconv.FormatUint(v, 10)
			}
			if firstComma {
				buf.WriteString("," + k + ":" + vStr)
			} else {
				buf.WriteString(k + ":" + vStr)
				firstComma = true
			}
		}
		logFields = append(logFields, zap.String("stats", buf.String()))
	}
	if info.ID != 0 {
		logFields = append(logFields, zap.Uint64("conn_id", info.ID))
	}
	if len(info.User) > 0 {
		logFields = append(logFields, zap.String("user", info.User))
	}
	if len(info.DB) > 0 {
		logFields = append(logFields, zap.String("database", info.DB))
	}
	var tableIDs, indexNames string
	if len(info.StmtCtx.TableIDs) > 0 {
		tableIDs = strings.Replace(fmt.Sprintf("%v", info.StmtCtx.TableIDs), " ", ",", -1)
		logFields = append(logFields, zap.String("table_ids", tableIDs))
	}
	if len(info.StmtCtx.IndexNames) > 0 {
		indexNames = strings.Replace(fmt.Sprintf("%v", info.StmtCtx.IndexNames), " ", ",", -1)
		logFields = append(logFields, zap.String("index_names", indexNames))
	}
	logFields = append(logFields, zap.Uint64("txn_start_ts", info.CurTxnStartTS))
	if memTracker := info.StmtCtx.MemTracker; memTracker != nil {
		logFields = append(logFields, zap.String("mem_max", fmt.Sprintf("%d Bytes (%v)", memTracker.MaxConsumed(), memTracker.BytesToString(memTracker.MaxConsumed()))))
	}

	const logSQLLen = 1024 * 8
	var sql string
	if len(info.Info) > 0 {
		sql = info.Info
	}
	if len(sql) > logSQLLen {
		sql = fmt.Sprintf("%s len(%d)", sql[:logSQLLen], len(sql))
	}
	logFields = append(logFields, zap.String("sql", sql))
	return logFields
}

// logExpensiveQuery logs the queries which exceed the time threshold or memory threshold.
func logExpensiveQuery(costTime time.Duration, info *util.ProcessInfo) {
	logutil.BgLogger().Warn("expensive_query", genLogFields(costTime, info)...)
}
<commit_msg>util: refine expensive query log during bootstrap (#14181)<commit_after>// Copyright 2019 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package expensivequery

import (
	"fmt"
	"strconv"
	"strings"
	"sync/atomic"
	"time"

	"github.com/pingcap/log"
	"github.com/pingcap/tidb/sessionctx/variable"
	"github.com/pingcap/tidb/util"
	"github.com/pingcap/tidb/util/logutil"
	"go.uber.org/zap"
	"go.uber.org/zap/zapcore"
)

// Handle is the handler for expensive query.
type Handle struct {
	exitCh chan struct{}
	sm     atomic.Value
}

// NewExpensiveQueryHandle builds a new expensive query handler.
func NewExpensiveQueryHandle(exitCh chan struct{}) *Handle {
	return &Handle{exitCh: exitCh}
}

// SetSessionManager sets the SessionManager which is used to fetching the info
// of all active sessions.
func (eqh *Handle) SetSessionManager(sm util.SessionManager) *Handle {
	eqh.sm.Store(sm)
	return eqh
}

// Run starts a expensive query checker goroutine at the start time of the server.
func (eqh *Handle) Run() {
	threshold := atomic.LoadUint64(&variable.ExpensiveQueryTimeThreshold)
	// use 100ms as tickInterval temply, may use given interval or use defined variable later
	tickInterval := time.Millisecond * time.Duration(100)
	ticker := time.NewTicker(tickInterval)
	defer ticker.Stop()
	sm := eqh.sm.Load().(util.SessionManager)
	for {
		select {
		case <-ticker.C:
			processInfo := sm.ShowProcessList()
			for _, info := range processInfo {
				if len(info.Info) == 0 || info.ExceedExpensiveTimeThresh {
					continue
				}
				costTime := time.Since(info.Time)
				if costTime >= time.Second*time.Duration(threshold) && log.GetLevel() <= zapcore.WarnLevel {
					logExpensiveQuery(costTime, info)
					info.ExceedExpensiveTimeThresh = true

				} else if info.MaxExecutionTime > 0 && costTime > time.Duration(info.MaxExecutionTime)*time.Millisecond {
					sm.Kill(info.ID, true)
				}
			}
			threshold = atomic.LoadUint64(&variable.ExpensiveQueryTimeThreshold)
		case <-eqh.exitCh:
			return
		}
	}
}

// LogOnQueryExceedMemQuota prints a log when memory usage of connID is out of memory quota.
func (eqh *Handle) LogOnQueryExceedMemQuota(connID uint64) {
	if log.GetLevel() > zapcore.WarnLevel {
		return
	}
	// The out-of-memory SQL may be the internal SQL which is executed during
	// the bootstrap phase, and the `sm` is not set at this phase. This is
	// unlikely to happen except for testing. Thus we do not need to log
	// detailed message for it.
	v := eqh.sm.Load()
	if v == nil {
		logutil.BgLogger().Info("expensive_query during bootstrap phase", zap.Uint64("conn_id", connID))
		return
	}
	sm := v.(util.SessionManager)
	info, ok := sm.GetProcessInfo(connID)
	if !ok {
		return
	}
	logExpensiveQuery(time.Since(info.Time), info)
}

func genLogFields(costTime time.Duration, info *util.ProcessInfo) []zap.Field {
	logFields := make([]zap.Field, 0, 20)
	logFields = append(logFields, zap.String("cost_time", strconv.FormatFloat(costTime.Seconds(), 'f', -1, 64)+"s"))
	execDetail := info.StmtCtx.GetExecDetails()
	logFields = append(logFields, execDetail.ToZapFields()...)
	if copTaskInfo := info.StmtCtx.CopTasksDetails(); copTaskInfo != nil {
		logFields = append(logFields, copTaskInfo.ToZapFields()...)
	}
	if statsInfo := info.StatsInfo(info.Plan); len(statsInfo) > 0 {
		var buf strings.Builder
		firstComma := false
		vStr := ""
		for k, v := range statsInfo {
			if v == 0 {
				vStr = "pseudo"
			} else {
				vStr = strconv.FormatUint(v, 10)
			}
			if firstComma {
				buf.WriteString("," + k + ":" + vStr)
			} else {
				buf.WriteString(k + ":" + vStr)
				firstComma = true
			}
		}
		logFields = append(logFields, zap.String("stats", buf.String()))
	}
	if info.ID != 0 {
		logFields = append(logFields, zap.Uint64("conn_id", info.ID))
	}
	if len(info.User) > 0 {
		logFields = append(logFields, zap.String("user", info.User))
	}
	if len(info.DB) > 0 {
		logFields = append(logFields, zap.String("database", info.DB))
	}
	var tableIDs, indexNames string
	if len(info.StmtCtx.TableIDs) > 0 {
		tableIDs = strings.Replace(fmt.Sprintf("%v", info.StmtCtx.TableIDs), " ", ",", -1)
		logFields = append(logFields, zap.String("table_ids", tableIDs))
	}
	if len(info.StmtCtx.IndexNames) > 0 {
		indexNames = strings.Replace(fmt.Sprintf("%v", info.StmtCtx.IndexNames), " ", ",", -1)
		logFields = append(logFields, zap.String("index_names", indexNames))
	}
	logFields = append(logFields, zap.Uint64("txn_start_ts", info.CurTxnStartTS))
	if memTracker := info.StmtCtx.MemTracker; memTracker != nil {
		logFields = append(logFields, zap.String("mem_max", fmt.Sprintf("%d Bytes (%v)", memTracker.MaxConsumed(), memTracker.BytesToString(memTracker.MaxConsumed()))))
	}

	const logSQLLen = 1024 * 8
	var sql string
	if len(info.Info) > 0 {
		sql = info.Info
	}
	if len(sql) > logSQLLen {
		sql = fmt.Sprintf("%s len(%d)", sql[:logSQLLen], len(sql))
	}
	logFields = append(logFields, zap.String("sql", sql))
	return logFields
}

// logExpensiveQuery logs the queries which exceed the time threshold or memory threshold.
func logExpensiveQuery(costTime time.Duration, info *util.ProcessInfo) {
	logutil.BgLogger().Warn("expensive_query", genLogFields(costTime, info)...)
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package workqueue

import (
	"fmt"
	"math/rand"
	"reflect"
	"testing"
	"time"

	"k8s.io/apimachinery/pkg/util/wait"
	testingclock "k8s.io/utils/clock/testing"
)

func TestSimpleQueue(t *testing.T) {
	fakeClock := testingclock.NewFakeClock(time.Now())
	q := NewDelayingQueueWithCustomClock(fakeClock, "")

	first := "foo"

	q.AddAfter(first, 50*time.Millisecond)
	if err := waitForWaitingQueueToFill(q); err != nil {
		t.Fatalf("unexpected err: %v", err)
	}

	if q.Len() != 0 {
		t.Errorf("should not have added")
	}

	fakeClock.Step(60 * time.Millisecond)

	if err := waitForAdded(q, 1); err != nil {
		t.Errorf("should have added")
	}
	item, _ := q.Get()
	q.Done(item)

	// step past the next heartbeat
	fakeClock.Step(10 * time.Second)

	err := wait.Poll(1*time.Millisecond, 30*time.Millisecond, func() (done bool, err error) {
		if q.Len() > 0 {
			return false, fmt.Errorf("added to queue")
		}

		return false, nil
	})
	if err != wait.ErrWaitTimeout {
		t.Errorf("expected timeout, got: %v", err)
	}

	if q.Len() != 0 {
		t.Errorf("should not have added")
	}
}

func TestDeduping(t *testing.T) {
	fakeClock := testingclock.NewFakeClock(time.Now())
	q := NewDelayingQueueWithCustomClock(fakeClock, "")

	first := "foo"

	q.AddAfter(first, 50*time.Millisecond)
	if err := waitForWaitingQueueToFill(q); err != nil {
		t.Fatalf("unexpected err: %v", err)
	}
	q.AddAfter(first, 70*time.Millisecond)
	if err := waitForWaitingQueueToFill(q); err != nil {
		t.Fatalf("unexpected err: %v", err)
	}
	if q.Len() != 0 {
		t.Errorf("should not have added")
	}

	// step past the first block, we should receive now
	fakeClock.Step(60 * time.Millisecond)
	if err := waitForAdded(q, 1); err != nil {
		t.Errorf("should have added")
	}
	item, _ := q.Get()
	q.Done(item)

	// step past the second add
	fakeClock.Step(20 * time.Millisecond)
	if q.Len() != 0 {
		t.Errorf("should not have added")
	}

	// test again, but this time the earlier should override
	q.AddAfter(first, 50*time.Millisecond)
	q.AddAfter(first, 30*time.Millisecond)
	if err := waitForWaitingQueueToFill(q); err != nil {
		t.Fatalf("unexpected err: %v", err)
	}
	if q.Len() != 0 {
		t.Errorf("should not have added")
	}

	fakeClock.Step(40 * time.Millisecond)
	if err := waitForAdded(q, 1); err != nil {
		t.Errorf("should have added")
	}
	item, _ = q.Get()
	q.Done(item)

	// step past the second add
	fakeClock.Step(20 * time.Millisecond)
	if q.Len() != 0 {
		t.Errorf("should not have added")
	}
	if q.Len() != 0 {
		t.Errorf("should not have added")
	}
}

func TestAddTwoFireEarly(t *testing.T) {
	fakeClock := testingclock.NewFakeClock(time.Now())
	q := NewDelayingQueueWithCustomClock(fakeClock, "")

	first := "foo"
	second := "bar"
	third := "baz"

	q.AddAfter(first, 1*time.Second)
	q.AddAfter(second, 50*time.Millisecond)
	if err := waitForWaitingQueueToFill(q); err != nil {
		t.Fatalf("unexpected err: %v", err)
	}

	if q.Len() != 0 {
		t.Errorf("should not have added")
	}

	fakeClock.Step(60 * time.Millisecond)

	if err := waitForAdded(q, 1); err != nil {
		t.Fatalf("unexpected err: %v", err)
	}
	item, _ := q.Get()
	if !reflect.DeepEqual(item, second) {
		t.Errorf("expected %v, got %v", second, item)
	}

	q.AddAfter(third, 2*time.Second)

	fakeClock.Step(1 * time.Second)
	if err := waitForAdded(q, 1); err != nil {
		t.Fatalf("unexpected err: %v", err)
	}
	item, _ = q.Get()
	if !reflect.DeepEqual(item, first) {
		t.Errorf("expected %v, got %v", first, item)
	}

	fakeClock.Step(2 * time.Second)
	if err := waitForAdded(q, 1); err != nil {
		t.Fatalf("unexpected err: %v", err)
	}
	item, _ = q.Get()
	if !reflect.DeepEqual(item, third) {
		t.Errorf("expected %v, got %v", third, item)
	}
}

func TestCopyShifting(t *testing.T) {
	fakeClock := testingclock.NewFakeClock(time.Now())
	q := NewDelayingQueueWithCustomClock(fakeClock, "")

	first := "foo"
	second := "bar"
	third := "baz"

	q.AddAfter(first, 1*time.Second)
	q.AddAfter(second, 500*time.Millisecond)
	q.AddAfter(third, 250*time.Millisecond)
	if err := waitForWaitingQueueToFill(q); err != nil {
		t.Fatalf("unexpected err: %v", err)
	}

	if q.Len() != 0 {
		t.Errorf("should not have added")
	}

	fakeClock.Step(2 * time.Second)

	if err := waitForAdded(q, 3); err != nil {
		t.Fatalf("unexpected err: %v", err)
	}
	actualFirst, _ := q.Get()
	if !reflect.DeepEqual(actualFirst, third) {
		t.Errorf("expected %v, got %v", third, actualFirst)
	}
	actualSecond, _ := q.Get()
	if !reflect.DeepEqual(actualSecond, second) {
		t.Errorf("expected %v, got %v", second, actualSecond)
	}
	actualThird, _ := q.Get()
	if !reflect.DeepEqual(actualThird, first) {
		t.Errorf("expected %v, got %v", first, actualThird)
	}
}

func BenchmarkDelayingQueue_AddAfter(b *testing.B) {
	fakeClock := testingclock.NewFakeClock(time.Now())
	q := NewDelayingQueueWithCustomClock(fakeClock, "")

	// Add items
	for n := 0; n < b.N; n++ {
		data := fmt.Sprintf("%d", n)
		q.AddAfter(data, time.Duration(rand.Int63n(int64(10*time.Minute))))
	}

	// Exercise item removal as well
	fakeClock.Step(11 * time.Minute)
	for n := 0; n < b.N; n++ {
		_, _ = q.Get()
	}
}

func waitForAdded(q DelayingInterface, depth int) error {
	return wait.Poll(1*time.Millisecond, 10*time.Second, func() (done bool, err error) {
		if q.Len() == depth {
			return true, nil
		}

		return false, nil
	})
}

func waitForWaitingQueueToFill(q DelayingInterface) error {
	return wait.Poll(1*time.Millisecond, 10*time.Second, func() (done bool, err error) {
		if len(q.(*delayingType).waitingForAddCh) == 0 {
			return true, nil
		}

		return false, nil
	})
}
<commit_msg>Remove the duplicate code snippet in client-go delaying_queue tests<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package workqueue

import (
	"fmt"
	"math/rand"
	"reflect"
	"testing"
	"time"

	"k8s.io/apimachinery/pkg/util/wait"
	testingclock "k8s.io/utils/clock/testing"
)

func TestSimpleQueue(t *testing.T) {
	fakeClock := testingclock.NewFakeClock(time.Now())
	q := NewDelayingQueueWithCustomClock(fakeClock, "")

	first := "foo"

	q.AddAfter(first, 50*time.Millisecond)
	if err := waitForWaitingQueueToFill(q); err != nil {
		t.Fatalf("unexpected err: %v", err)
	}

	if q.Len() != 0 {
		t.Errorf("should not have added")
	}

	fakeClock.Step(60 * time.Millisecond)

	if err := waitForAdded(q, 1); err != nil {
		t.Errorf("should have added")
	}
	item, _ := q.Get()
	q.Done(item)

	// step past the next heartbeat
	fakeClock.Step(10 * time.Second)

	err := wait.Poll(1*time.Millisecond, 30*time.Millisecond, func() (done bool, err error) {
		if q.Len() > 0 {
			return false, fmt.Errorf("added to queue")
		}

		return false, nil
	})
	if err != wait.ErrWaitTimeout {
		t.Errorf("expected timeout, got: %v", err)
	}

	if q.Len() != 0 {
		t.Errorf("should not have added")
	}
}

func TestDeduping(t *testing.T) {
	fakeClock := testingclock.NewFakeClock(time.Now())
	q := NewDelayingQueueWithCustomClock(fakeClock, "")

	first := "foo"

	q.AddAfter(first, 50*time.Millisecond)
	if err := waitForWaitingQueueToFill(q); err != nil {
		t.Fatalf("unexpected err: %v", err)
	}
	q.AddAfter(first, 70*time.Millisecond)
	if err := waitForWaitingQueueToFill(q); err != nil {
		t.Fatalf("unexpected err: %v", err)
	}
	if q.Len() != 0 {
		t.Errorf("should not have added")
	}

	// step past the first block, we should receive now
	fakeClock.Step(60 * time.Millisecond)
	if err := waitForAdded(q, 1); err != nil {
		t.Errorf("should have added")
	}
	item, _ := q.Get()
	q.Done(item)

	// step past the second add
	fakeClock.Step(20 * time.Millisecond)
	if q.Len() != 0 {
		t.Errorf("should not have added")
	}

	// test again, but this time the earlier should override
	q.AddAfter(first, 50*time.Millisecond)
	q.AddAfter(first, 30*time.Millisecond)
	if err := waitForWaitingQueueToFill(q); err != nil {
		t.Fatalf("unexpected err: %v", err)
	}
	if q.Len() != 0 {
		t.Errorf("should not have added")
	}

	fakeClock.Step(40 * time.Millisecond)
	if err := waitForAdded(q, 1); err != nil {
		t.Errorf("should have added")
	}
	item, _ = q.Get()
	q.Done(item)

	// step past the second add
	fakeClock.Step(20 * time.Millisecond)
	if q.Len() != 0 {
		t.Errorf("should not have added")
	}
}

func TestAddTwoFireEarly(t *testing.T) {
	fakeClock := testingclock.NewFakeClock(time.Now())
	q := NewDelayingQueueWithCustomClock(fakeClock, "")

	first := "foo"
	second := "bar"
	third := "baz"

	q.AddAfter(first, 1*time.Second)
	q.AddAfter(second, 50*time.Millisecond)
	if err := waitForWaitingQueueToFill(q); err != nil {
		t.Fatalf("unexpected err: %v", err)
	}

	if q.Len() != 0 {
		t.Errorf("should not have added")
	}

	fakeClock.Step(60 * time.Millisecond)

	if err := waitForAdded(q, 1); err != nil {
		t.Fatalf("unexpected err: %v", err)
	}
	item, _ := q.Get()
	if !reflect.DeepEqual(item, second) {
		t.Errorf("expected %v, got %v", second, item)
	}

	q.AddAfter(third, 2*time.Second)

	fakeClock.Step(1 * time.Second)
	if err := waitForAdded(q, 1); err != nil {
		t.Fatalf("unexpected err: %v", err)
	}
	item, _ = q.Get()
	if !reflect.DeepEqual(item, first) {
		t.Errorf("expected %v, got %v", first, item)
	}

	fakeClock.Step(2 * time.Second)
	if err := waitForAdded(q, 1); err != nil {
		t.Fatalf("unexpected err: %v", err)
	}
	item, _ = q.Get()
	if !reflect.DeepEqual(item, third) {
		t.Errorf("expected %v, got %v", third, item)
	}
}

func TestCopyShifting(t *testing.T) {
	fakeClock := testingclock.NewFakeClock(time.Now())
	q := NewDelayingQueueWithCustomClock(fakeClock, "")

	first := "foo"
	second := "bar"
	third := "baz"

	q.AddAfter(first, 1*time.Second)
	q.AddAfter(second, 500*time.Millisecond)
	q.AddAfter(third, 250*time.Millisecond)
	if err := waitForWaitingQueueToFill(q); err != nil {
		t.Fatalf("unexpected err: %v", err)
	}

	if q.Len() != 0 {
		t.Errorf("should not have added")
	}

	fakeClock.Step(2 * time.Second)

	if err := waitForAdded(q, 3); err != nil {
		t.Fatalf("unexpected err: %v", err)
	}
	actualFirst, _ := q.Get()
	if !reflect.DeepEqual(actualFirst, third) {
		t.Errorf("expected %v, got %v", third, actualFirst)
	}
	actualSecond, _ := q.Get()
	if !reflect.DeepEqual(actualSecond, second) {
		t.Errorf("expected %v, got %v", second, actualSecond)
	}
	actualThird, _ := q.Get()
	if !reflect.DeepEqual(actualThird, first) {
		t.Errorf("expected %v, got %v", first, actualThird)
	}
}

func BenchmarkDelayingQueue_AddAfter(b *testing.B) {
	fakeClock := testingclock.NewFakeClock(time.Now())
	q := NewDelayingQueueWithCustomClock(fakeClock, "")

	// Add items
	for n := 0; n < b.N; n++ {
		data := fmt.Sprintf("%d", n)
		q.AddAfter(data, time.Duration(rand.Int63n(int64(10*time.Minute))))
	}

	// Exercise item removal as well
	fakeClock.Step(11 * time.Minute)
	for n := 0; n < b.N; n++ {
		_, _ = q.Get()
	}
}

func waitForAdded(q DelayingInterface, depth int) error {
	return wait.Poll(1*time.Millisecond, 10*time.Second, func() (done bool, err error) {
		if q.Len() == depth {
			return true, nil
		}

		return false, nil
	})
}

func waitForWaitingQueueToFill(q DelayingInterface) error {
	return wait.Poll(1*time.Millisecond, 10*time.Second, func() (done bool, err error) {
		if len(q.(*delayingType).waitingForAddCh) == 0 {
			return true, nil
		}

		return false, nil
	})
}
<|endoftext|>


<commit_before>package fetch

import (
	"context"
	"errors"
	"flag"
	"github.com/SlyMarbo/rss"
	log "github.com/golang/glog"
	"github.com/jrupac/goliath/models"
	"github.com/jrupac/goliath/storage"
	"github.com/jrupac/goliath/utils"
	"github.com/mat/besticon/besticon"
	"github.com/microcosm-cc/bluemonday"
	"io/ioutil"
	"net/url"
	"sync"
	"time"
)

var (
	sanitizeHTML      = flag.Bool("sanitizeHTML", false, "If true, sanitize HTML content with Bluemonday.")
	normalizeFavicons = flag.Bool("normalizeFavicons", true, "If true, resize favicons to 256x256 and encode as PNG.")
)

var (
	pauseChan             = make(chan struct{})
	pauseChanDone         = make(chan struct{})
	resumeChan            = make(chan struct{})
	bluemondayTitlePolicy = bluemonday.StrictPolicy()
	bluemondayBodyPolicy  = makeBodyPolicy()
)

type imagePair struct {
	id      int64
	mime    string
	favicon []byte
}

func makeBodyPolicy() *bluemonday.Policy {
	p := bluemonday.UGCPolicy()
	p.AllowAttrs("title", "alt").OnElements("img")
	return p
}

// Pause stops all continuous feed fetching in a way that is resume-able.
// This call will block until fetching is fully paused. If fetching has not
// started yet, this call will block indefinitely.
func Pause() {
	pauseChan <- struct{}{}
	<-pauseChanDone
}

// Resume resumes continuous feed fetching with a fresh read of feeds.
// If fetching has not started yet, this call will block indefinitely.
func Resume() {
	resumeChan <- struct{}{}
}

// Start starts continuous feed fetching and writes fetched articles to the
// database.
func Start(ctx context.Context, d *storage.Database) {
	log.Infof("Starting continuous feed fetching.")

	// Add additional time layouts that sometimes appear in feeds.
	rss.TimeLayouts = append(rss.TimeLayouts, "2006-01-02")
	rss.TimeLayouts = append(rss.TimeLayouts, "Monday, 02 Jan 2006 15:04:05 MST")
	rss.TimeLayouts = append(rss.TimeLayouts, "Mon, 02 Jan 2006")

	// Turn off logging of HTTP icon requests.
	besticon.SetLogOutput(ioutil.Discard)

	fctx, cancel := context.WithCancel(ctx)

	wg := &sync.WaitGroup{}
	wg.Add(1)
	go start(fctx, wg, d)

	for {
		select {
		case <-pauseChan:
			cancel()
			wg.Wait()
			log.Info("Fetcher paused.")
			pauseChanDone <- struct{}{}
		case <-resumeChan:
			fctx, cancel = context.WithCancel(ctx)
			wg.Add(1)
			go start(fctx, wg, d)
			log.Info("Fetcher resumed.")
		case <-ctx.Done():
			wg.Wait()
			cancel()
			return
		}
	}
}

func start(ctx context.Context, parent *sync.WaitGroup, d *storage.Database) {
	defer parent.Done()

	feeds, err := d.GetAllFeeds()
	if err != nil {
		log.Errorf("Failed to fetch all feeds: %s", err)
	}
	utils.DebugPrint("Feed list", feeds)

	wg := &sync.WaitGroup{}
	wg.Add(len(feeds))
	ac := make(chan models.Article)
	ic := make(chan imagePair)

	for _, f := range feeds {
		go func(f models.Feed) {
			defer wg.Done()
			fetchLoop(ctx, d, ac, ic, f)
		}(f)
	}

	for {
		select {
		case a := <-ac:
			utils.DebugPrint("Received a new article:", a)
			if err2 := d.InsertArticle(a); err2 != nil {
				log.Warningf("Failed to persist article: %+v: %s", a, err2)
			}
		case ip := <-ic:
			utils.DebugPrint("Received a new image:", ip)
			if err2 := d.InsertFavicon(ip.id, ip.mime, ip.favicon); err2 != nil {
				log.Warningf("Failed to persist icon for feed %d: %s", ip.id, err2)
			}
		case <-ctx.Done():
			log.Infof("Stopping fetching feeds...")
			wg.Wait()
			log.Infof("Stopped fetching feeds.")
			return
		}
	}
}

func fetchLoop(ctx context.Context, d *storage.Database, ac chan models.Article, ic chan imagePair, feed models.Feed) {
	log.Infof("Fetching URL '%s'", feed.URL)
	tick := make(<-chan time.Time)
	initalFetch := make(chan struct{})

	go func() {
		f, err := rss.Fetch(feed.URL)
		if err != nil {
			log.Warningf("Error for feed %d fetching URL '%s': %s", feed.ID, feed.URL, err)
			return
		}
		handleItems(ctx, &feed, d, f.Items, ac)
		handleImage(ctx, feed, f, ic)

		tick = time.After(time.Until(f.Refresh))
		log.Infof("Initial waiting to fetch %s until %s\n", feed.URL, f.Refresh)
		initalFetch <- struct{}{}
	}()

	for {
		select {
		case <-initalFetch:
			// Block on initial fetch here so that we can return early if needed
			continue
		case <-tick:
			log.Infof("Fetching feed %s", feed.URL)
			var refresh time.Time
			if f, err := rss.Fetch(feed.URL); err != nil {
				log.Warningf("Error fetching %s: %s", feed.URL, err)
				// If the request transiently fails, try again after a fixed interval.
				refresh = time.Now().Add(10 * time.Minute)
			} else {
				handleItems(ctx, &feed, d, f.Items, ac)
				refresh = f.Refresh
			}
			log.Infof("Waiting to fetch %s until %s\n", feed.URL, refresh)
			tick = time.After(time.Until(refresh))
		case <-ctx.Done():
			return
		}
	}
}

func handleItems(ctx context.Context, feed *models.Feed, d *storage.Database, items []*rss.Item, send chan models.Article) {
	latest := feed.Latest
	newLatest := latest

Loop:
	for _, item := range items {
		title := item.Title
		// Some feeds give back content that is HTML-escaped. When this happens,
		// sanitization makes the content appear as raw, escaped text. There's not
		// a canonical way of determining if the content is given here as escaped
		// or not, so we use a heuristic.
		content := maybeUnescapeHtml(item.Content)
		summary := maybeUnescapeHtml(item.Summary)

		parsed := maybeParseArticleContent(item.Link)

		if *sanitizeHTML {
			title = bluemondayTitlePolicy.Sanitize(title)
			content = bluemondayBodyPolicy.Sanitize(content)
			summary = bluemondayBodyPolicy.Sanitize(summary)
			parsed = bluemondayBodyPolicy.Sanitize(parsed)
		}

		summary = maybeRewriteImageSourceUrls(summary)

		a := models.Article{
			FeedID:    feed.ID,
			FolderID:  feed.FolderID,
			Title:     title,
			Summary:   summary,
			Content:   content,
			Parsed:    parsed,
			Link:      item.Link,
			Date:      item.Date,
			Read:      item.Read,
			Retrieved: time.Now(),
		}

		if a.Date.After(latest) {
			select {
			case send <- a:
				break
			case <-ctx.Done():
				// Break out of processing articles and just clean up.
				break Loop
			}
			if a.Date.After(newLatest) {
				newLatest = a.Date
			}
		} else {
			log.V(2).Infof("Not persisting too old article: %+v", a)
		}
	}

	err := d.UpdateLatestTimeForFeed(feed.ID, newLatest)
	if err != nil {
		log.Warningf("Failed to update latest feed time: %s", err)
	} else {
		feed.Latest = newLatest
	}
}

func handleImage(ctx context.Context, feed models.Feed, f *rss.Feed, send chan imagePair) {
	var icon besticon.Icon
	var feedHost string

	u, err := url.Parse(f.Link)
	if err == nil {
		feedHost = u.Hostname()
	}

	if i, err2 := tryIconFetch(f.Image.URL); err2 == nil {
		icon = i
	} else if i, err2 = tryIconFetch(f.Link); err2 == nil {
		icon = i
	} else if i, err2 = tryIconFetch(feedHost); err2 == nil {
		icon = i
	} else {
		return
	}

	select {
	case send <- maybeResizeImage(feed.ID, icon):
		break
	case <-ctx.Done():
		break
	}
}

func tryIconFetch(link string) (besticon.Icon, error) {
	icon := besticon.Icon{}

	if link == "" {
		return icon, errors.New("invalid URL")
	}

	finder := besticon.IconFinder{}

	icons, err := finder.FetchIcons(link)
	if err != nil {
		return icon, err
	}

	if len(icons) == 0 {
		return icon, errors.New("no icons found")
	}

	for _, i := range icons {
		if i.URL != "" && i.Format != "" {
			return i, nil
		}
	}

	return icon, errors.New("no suitable icons found")
}
<commit_msg>core/fetch: Minor error handling changes to icon fetching.<commit_after>package fetch

import (
	"context"
	"errors"
	"flag"
	"github.com/SlyMarbo/rss"
	log "github.com/golang/glog"
	"github.com/jrupac/goliath/models"
	"github.com/jrupac/goliath/storage"
	"github.com/jrupac/goliath/utils"
	"github.com/mat/besticon/besticon"
	"github.com/microcosm-cc/bluemonday"
	"io/ioutil"
	"net/url"
	"sync"
	"time"
)

var (
	sanitizeHTML      = flag.Bool("sanitizeHTML", false, "If true, sanitize HTML content with Bluemonday.")
	normalizeFavicons = flag.Bool("normalizeFavicons", true, "If true, resize favicons to 256x256 and encode as PNG.")
)

var (
	pauseChan             = make(chan struct{})
	pauseChanDone         = make(chan struct{})
	resumeChan            = make(chan struct{})
	bluemondayTitlePolicy = bluemonday.StrictPolicy()
	bluemondayBodyPolicy  = makeBodyPolicy()
)

type imagePair struct {
	id      int64
	mime    string
	favicon []byte
}

func makeBodyPolicy() *bluemonday.Policy {
	p := bluemonday.UGCPolicy()
	p.AllowAttrs("title", "alt").OnElements("img")
	return p
}

// Pause stops all continuous feed fetching in a way that is resume-able.
// This call will block until fetching is fully paused. If fetching has not
// started yet, this call will block indefinitely.
func Pause() {
	pauseChan <- struct{}{}
	<-pauseChanDone
}

// Resume resumes continuous feed fetching with a fresh read of feeds.
// If fetching has not started yet, this call will block indefinitely.
func Resume() {
	resumeChan <- struct{}{}
}

// Start starts continuous feed fetching and writes fetched articles to the
// database.
func Start(ctx context.Context, d *storage.Database) {
	log.Infof("Starting continuous feed fetching.")

	// Add additional time layouts that sometimes appear in feeds.
	rss.TimeLayouts = append(rss.TimeLayouts, "2006-01-02")
	rss.TimeLayouts = append(rss.TimeLayouts, "Monday, 02 Jan 2006 15:04:05 MST")
	rss.TimeLayouts = append(rss.TimeLayouts, "Mon, 02 Jan 2006")

	// Turn off logging of HTTP icon requests.
	besticon.SetLogOutput(ioutil.Discard)

	fctx, cancel := context.WithCancel(ctx)

	wg := &sync.WaitGroup{}
	wg.Add(1)
	go start(fctx, wg, d)

	for {
		select {
		case <-pauseChan:
			cancel()
			wg.Wait()
			log.Info("Fetcher paused.")
			pauseChanDone <- struct{}{}
		case <-resumeChan:
			fctx, cancel = context.WithCancel(ctx)
			wg.Add(1)
			go start(fctx, wg, d)
			log.Info("Fetcher resumed.")
		case <-ctx.Done():
			wg.Wait()
			cancel()
			return
		}
	}
}

func start(ctx context.Context, parent *sync.WaitGroup, d *storage.Database) {
	defer parent.Done()

	feeds, err := d.GetAllFeeds()
	if err != nil {
		log.Errorf("Failed to fetch all feeds: %s", err)
	}
	utils.DebugPrint("Feed list", feeds)

	wg := &sync.WaitGroup{}
	wg.Add(len(feeds))
	ac := make(chan models.Article)
	ic := make(chan imagePair)

	for _, f := range feeds {
		go func(f models.Feed) {
			defer wg.Done()
			fetchLoop(ctx, d, ac, ic, f)
		}(f)
	}

	for {
		select {
		case a := <-ac:
			utils.DebugPrint("Received a new article:", a)
			if err2 := d.InsertArticle(a); err2 != nil {
				log.Warningf("Failed to persist article: %+v: %s", a, err2)
			}
		case ip := <-ic:
			utils.DebugPrint("Received a new image:", ip)
			if err2 := d.InsertFavicon(ip.id, ip.mime, ip.favicon); err2 != nil {
				log.Warningf("Failed to persist icon for feed %d: %s", ip.id, err2)
			}
		case <-ctx.Done():
			log.Infof("Stopping fetching feeds...")
			wg.Wait()
			log.Infof("Stopped fetching feeds.")
			return
		}
	}
}

func fetchLoop(ctx context.Context, d *storage.Database, ac chan models.Article, ic chan imagePair, feed models.Feed) {
	log.Infof("Fetching URL '%s'", feed.URL)
	tick := make(<-chan time.Time)
	initalFetch := make(chan struct{})

	go func() {
		f, err := rss.Fetch(feed.URL)
		if err != nil {
			log.Warningf("Error for feed %d fetching URL '%s': %s", feed.ID, feed.URL, err)
			return
		}
		handleItems(ctx, &feed, d, f.Items, ac)
		handleImage(ctx, feed, f, ic)

		tick = time.After(time.Until(f.Refresh))
		log.Infof("Initial waiting to fetch %s until %s\n", feed.URL, f.Refresh)
		initalFetch <- struct{}{}
	}()

	for {
		select {
		case <-initalFetch:
			// Block on initial fetch here so that we can return early if needed
			continue
		case <-tick:
			log.Infof("Fetching feed %s", feed.URL)
			var refresh time.Time
			if f, err := rss.Fetch(feed.URL); err != nil {
				log.Warningf("Error fetching %s: %s", feed.URL, err)
				// If the request transiently fails, try again after a fixed interval.
				refresh = time.Now().Add(10 * time.Minute)
			} else {
				handleItems(ctx, &feed, d, f.Items, ac)
				refresh = f.Refresh
			}
			log.Infof("Waiting to fetch %s until %s\n", feed.URL, refresh)
			tick = time.After(time.Until(refresh))
		case <-ctx.Done():
			return
		}
	}
}

func handleItems(ctx context.Context, feed *models.Feed, d *storage.Database, items []*rss.Item, send chan models.Article) {
	latest := feed.Latest
	newLatest := latest

Loop:
	for _, item := range items {
		title := item.Title
		// Some feeds give back content that is HTML-escaped. When this happens,
		// sanitization makes the content appear as raw, escaped text. There's not
		// a canonical way of determining if the content is given here as escaped
		// or not, so we use a heuristic.
		content := maybeUnescapeHtml(item.Content)
		summary := maybeUnescapeHtml(item.Summary)

		parsed := maybeParseArticleContent(item.Link)

		if *sanitizeHTML {
			title = bluemondayTitlePolicy.Sanitize(title)
			content = bluemondayBodyPolicy.Sanitize(content)
			summary = bluemondayBodyPolicy.Sanitize(summary)
			parsed = bluemondayBodyPolicy.Sanitize(parsed)
		}

		summary = maybeRewriteImageSourceUrls(summary)

		a := models.Article{
			FeedID:    feed.ID,
			FolderID:  feed.FolderID,
			Title:     title,
			Summary:   summary,
			Content:   content,
			Parsed:    parsed,
			Link:      item.Link,
			Date:      item.Date,
			Read:      item.Read,
			Retrieved: time.Now(),
		}

		if a.Date.After(latest) {
			select {
			case send <- a:
				break
			case <-ctx.Done():
				// Break out of processing articles and just clean up.
				break Loop
			}
			if a.Date.After(newLatest) {
				newLatest = a.Date
			}
		} else {
			log.V(2).Infof("Not persisting too old article: %+v", a)
		}
	}

	err := d.UpdateLatestTimeForFeed(feed.ID, newLatest)
	if err != nil {
		log.Warningf("Failed to update latest feed time: %s", err)
	} else {
		feed.Latest = newLatest
	}
}

func handleImage(ctx context.Context, feed models.Feed, f *rss.Feed, send chan imagePair) {
	var icon besticon.Icon
	var feedHost string

	u, err := url.Parse(f.Link)
	if err == nil {
		feedHost = u.Hostname()
	}

	if i, err := tryIconFetch(f.Image.URL); err == nil {
		icon = i
	} else if i, err = tryIconFetch(f.Link); err == nil {
		icon = i
	} else if i, err = tryIconFetch(feedHost); err == nil {
		icon = i
	} else {
		log.V(2).Infof("Could not find suitable icon for feed: %s", feedHost)
		return
	}

	select {
	case send <- maybeResizeImage(feed.ID, icon):
		break
	case <-ctx.Done():
		break
	}
}

func tryIconFetch(link string) (besticon.Icon, error) {
	icon := besticon.Icon{}

	if link == "" {
		return icon, errors.New("invalid URL")
	}

	finder := besticon.IconFinder{}

	icons, err := finder.FetchIcons(link)
	if err != nil {
		return icon, err
	}

	if len(icons) == 0 {
		return icon, errors.New("no icons found")
	}

	for _, i := range icons {
		if i.URL != "" && i.Format != "" {
			return i, nil
		}
	}

	return icon, errors.New("no suitable icons found")
}
<|endoftext|>


<commit_before>package file

import "github.com/nsf/termbox-go"
import "strings"
import "regexp"
import "strconv"

func (file *File) EnforceColBounds() {
	for idx, cursor := range file.MultiCursor {
		if cursor.col > len(file.Buffer[cursor.row]) {
			file.MultiCursor[idx].col = len(file.Buffer[cursor.row])
		}
		if cursor.col < 0 {
			file.MultiCursor[idx].col = 0
		}
	}
}

func (file *File) EnforceRowBounds() {
	for idx, cursor := range file.MultiCursor {
		if cursor.row >= len(file.Buffer) {
			file.MultiCursor[idx].row = len(file.Buffer) - 1
		}
		if cursor.row < 0 {
			file.MultiCursor[idx].row = 0
		}
	}
}

func (file *File) MakeCursorNotAtTopBottom() {
	row := file.MultiCursor[0].row
	_, rows := termbox.Size()
	bottom := file.rowOffset + rows - 1
	if row >= bottom {
		file.rowOffset += (row - bottom) + rows/8
	}
}

func (file *File) CursorGoTo(row, col int) {
	file.MultiCursor[0].row = row
	file.MultiCursor[0].col = col
	file.EnforceRowBounds()
	file.EnforceColBounds()
	file.MakeCursorNotAtTopBottom()
}

func (file *File) PageDown() {
	_, rows := termbox.Size()
	file.CursorDown(rows/2 - 1)
}

func (file *File) PageUp() {
	_, rows := termbox.Size()
	file.CursorUp(rows/2 - 1)
}

func (file *File) CursorUp(n int) {
	file.MultiCursor[0].row -= n
	if file.MultiCursor[0].row < 0 {
		file.MultiCursor[0].row = 0
	}
	file.MultiCursor[0].col = file.MultiCursor[0].colwant
	file.EnforceColBounds()
}

func (file *File) CursorDown(n int) {
	file.MultiCursor[0].row += n
	if file.MultiCursor[0].row >= len(file.Buffer) {
		file.MultiCursor[0].row = len(file.Buffer) - 1
	}
	file.MultiCursor[0].col = file.MultiCursor[0].colwant
	file.EnforceColBounds()
}

func (file *File) CursorRight() {
	for idx, cursor := range file.MultiCursor {
		if cursor.col < len(file.Buffer[cursor.row]) {
			file.MultiCursor[idx].col += 1
		} else {
			if len(file.MultiCursor) > 1 {
				continue
			}
			if cursor.row < len(file.Buffer)-1 {
				file.MultiCursor[idx].row += 1
				file.MultiCursor[idx].col = 0
			}
		}
		file.MultiCursor[idx].colwant = file.MultiCursor[idx].col
	}
	file.EnforceRowBounds()
	file.EnforceColBounds()
}

func (file *File) CursorLeft() {
	for idx, cursor := range file.MultiCursor {
		if cursor.col > 0 {
			file.MultiCursor[idx].col -= 1
		} else {
			if len(file.MultiCursor) > 1 {
				continue
			}
			if cursor.row > 0 {
				file.MultiCursor[idx].row -= 1
				file.MultiCursor[idx].col = len(file.Buffer[file.MultiCursor[idx].row])
			}
		}
		file.MultiCursor[idx].colwant = file.MultiCursor[idx].col
	}
}

func (file *File) GetCursor(idx int) (int, int) {
	file.EnforceRowBounds()
	file.EnforceColBounds()
	line := file.Buffer[file.MultiCursor[idx].row][0:file.MultiCursor[idx].col]
	strLine := string(line)
	strLine = strings.Replace(strLine, "\t", "    ", -1)
	return file.MultiCursor[idx].row - file.rowOffset, len(strLine) - file.colOffset
}

func (file *File) ScrollLeft() {
	file.colOffset += 1
}

func (file *File) ScrollRight() {
	if file.colOffset > 0 {
		file.colOffset -= 1
	}
}

func (file *File) ScrollUp() {
	if file.rowOffset < len(file.Buffer)-1 {
		file.rowOffset += 1
	}
}

func (file *File) ScrollDown() {
	if file.rowOffset > 0 {
		file.rowOffset -= 1
	}
}

func (file *File) UpdateOffsets(nRows, nCols int) {

	if file.MultiCursor[0].row < file.rowOffset {
		file.rowOffset = file.MultiCursor[0].row
	}
	if file.MultiCursor[0].row >= file.rowOffset+nRows-1 {
		file.rowOffset = file.MultiCursor[0].row - nRows + 1
	}

	_, col := file.GetCursor(0)
	col += file.colOffset
	if col < file.colOffset {
		file.colOffset = col
	}
	if col >= file.colOffset+nCols-1 {
		file.colOffset = col - nCols + 1
	}

}

func (file *File) StartOfLine() {
	allAtZero := true
	for _, cursor := range file.MultiCursor {
		if cursor.col != 0 {
			allAtZero = false
			break
		}
	}
	if allAtZero {
		re := regexp.MustCompile("^[ \t]*")
		for idx, cursor := range file.MultiCursor {
			row := cursor.row
			line := file.Buffer[row]
			match := re.FindStringIndex(line.ToString())
			file.MultiCursor[idx].col = match[1]
			file.MultiCursor[idx].colwant = file.MultiCursor[idx].col
		}
	} else {
		for idx, _ := range file.MultiCursor {
			file.MultiCursor[idx].col = 0
			file.MultiCursor[idx].colwant = file.MultiCursor[idx].col
		}
	}
}

func (file *File) EndOfLine() {
	for idx, _ := range file.MultiCursor {
		row := file.MultiCursor[idx].row
		file.MultiCursor[idx].col = len(file.Buffer[row])
		file.MultiCursor[idx].colwant = file.MultiCursor[idx].col
	}
}

func (file *File) NextWord() {
	for idx, cursor := range file.MultiCursor {
		row := cursor.row
		line := file.Buffer[row]
		col := cursor.col
		re := regexp.MustCompile("[\t ][^\t ]")
		offset := re.FindStringIndex(line[col:].ToString())
		if offset == nil {
			col = len(line)
		} else {
			col += offset[0] + 1
		}
		file.MultiCursor[idx].col = col
		file.MultiCursor[idx].colwant = file.MultiCursor[idx].col
	}
}

func (file *File) PrevWord() {
	for idx, cursor := range file.MultiCursor {
		row := cursor.row
		line := file.Buffer[row]
		col := cursor.col
		re := regexp.MustCompile("[\t ][^\t ]")
		offsets := re.FindAllStringIndex(line[:col].ToString(), -1)
		if offsets == nil {
			col = 0
		} else {
			offset := offsets[len(offsets)-1]
			col = offset[0] + 1
		}
		file.MultiCursor[idx].col = col
		file.MultiCursor[idx].colwant = file.MultiCursor[idx].col
	}
}

func (file *File) GoToLine() {
	lineNo := file.screen.GetPromptAnswer("goto:", &file.gotoHist)
	if lineNo == "" {
		return
	}
	row, err := strconv.Atoi(lineNo)
	if err == nil {
		file.CursorGoTo(row, 0)
	}
}
<commit_msg>bugfix: colwant should work better now<commit_after>package file

import "github.com/nsf/termbox-go"
import "strings"
import "regexp"
import "strconv"

func (file *File) EnforceColBounds() {
	for idx, cursor := range file.MultiCursor {
		if cursor.col > len(file.Buffer[cursor.row]) {
			file.MultiCursor[idx].col = len(file.Buffer[cursor.row])
		}
		if cursor.col < 0 {
			file.MultiCursor[idx].col = 0
		}
	}
}

func (file *File) EnforceRowBounds() {
	for idx, cursor := range file.MultiCursor {
		if cursor.row >= len(file.Buffer) {
			file.MultiCursor[idx].row = len(file.Buffer) - 1
		}
		if cursor.row < 0 {
			file.MultiCursor[idx].row = 0
		}
	}
}

func (file *File) MakeCursorNotAtTopBottom() {
	row := file.MultiCursor[0].row
	_, rows := termbox.Size()
	bottom := file.rowOffset + rows - 1
	if row >= bottom {
		file.rowOffset += (row - bottom) + rows/8
	}
}

func (file *File) CursorGoTo(row, col int) {
	file.MultiCursor[0].row = row
	file.MultiCursor[0].col = col
	file.MultiCursor[0].colwant = col
	file.EnforceRowBounds()
	file.EnforceColBounds()
	file.MakeCursorNotAtTopBottom()
}

func (file *File) PageDown() {
	_, rows := termbox.Size()
	file.CursorDown(rows/2 - 1)
}

func (file *File) PageUp() {
	_, rows := termbox.Size()
	file.CursorUp(rows/2 - 1)
}

func (file *File) CursorUp(n int) {
	file.MultiCursor[0].row -= n
	if file.MultiCursor[0].row < 0 {
		file.MultiCursor[0].row = 0
	}
	file.MultiCursor[0].col = file.MultiCursor[0].colwant
	file.EnforceColBounds()
}

func (file *File) CursorDown(n int) {
	file.MultiCursor[0].row += n
	if file.MultiCursor[0].row >= len(file.Buffer) {
		file.MultiCursor[0].row = len(file.Buffer) - 1
	}
	file.MultiCursor[0].col = file.MultiCursor[0].colwant
	file.EnforceColBounds()
}

func (file *File) CursorRight() {
	for idx, cursor := range file.MultiCursor {
		if cursor.col < len(file.Buffer[cursor.row]) {
			file.MultiCursor[idx].col += 1
		} else {
			if len(file.MultiCursor) > 1 {
				continue
			}
			if cursor.row < len(file.Buffer)-1 {
				file.MultiCursor[idx].row += 1
				file.MultiCursor[idx].col = 0
			}
		}
		file.MultiCursor[idx].colwant = file.MultiCursor[idx].col
	}
	file.EnforceRowBounds()
	file.EnforceColBounds()
}

func (file *File) CursorLeft() {
	for idx, cursor := range file.MultiCursor {
		if cursor.col > 0 {
			file.MultiCursor[idx].col -= 1
		} else {
			if len(file.MultiCursor) > 1 {
				continue
			}
			if cursor.row > 0 {
				file.MultiCursor[idx].row -= 1
				file.MultiCursor[idx].col = len(file.Buffer[file.MultiCursor[idx].row])
			}
		}
		file.MultiCursor[idx].colwant = file.MultiCursor[idx].col
	}
}

func (file *File) GetCursor(idx int) (int, int) {
	file.EnforceRowBounds()
	file.EnforceColBounds()
	line := file.Buffer[file.MultiCursor[idx].row][0:file.MultiCursor[idx].col]
	strLine := string(line)
	strLine = strings.Replace(strLine, "\t", "    ", -1)
	return file.MultiCursor[idx].row - file.rowOffset, len(strLine) - file.colOffset
}

func (file *File) ScrollLeft() {
	file.colOffset += 1
}

func (file *File) ScrollRight() {
	if file.colOffset > 0 {
		file.colOffset -= 1
	}
}

func (file *File) ScrollUp() {
	if file.rowOffset < len(file.Buffer)-1 {
		file.rowOffset += 1
	}
}

func (file *File) ScrollDown() {
	if file.rowOffset > 0 {
		file.rowOffset -= 1
	}
}

func (file *File) UpdateOffsets(nRows, nCols int) {

	if file.MultiCursor[0].row < file.rowOffset {
		file.rowOffset = file.MultiCursor[0].row
	}
	if file.MultiCursor[0].row >= file.rowOffset+nRows-1 {
		file.rowOffset = file.MultiCursor[0].row - nRows + 1
	}

	_, col := file.GetCursor(0)
	col += file.colOffset
	if col < file.colOffset {
		file.colOffset = col
	}
	if col >= file.colOffset+nCols-1 {
		file.colOffset = col - nCols + 1
	}

}

func (file *File) StartOfLine() {
	allAtZero := true
	for _, cursor := range file.MultiCursor {
		if cursor.col != 0 {
			allAtZero = false
			break
		}
	}
	if allAtZero {
		re := regexp.MustCompile("^[ \t]*")
		for idx, cursor := range file.MultiCursor {
			row := cursor.row
			line := file.Buffer[row]
			match := re.FindStringIndex(line.ToString())
			file.MultiCursor[idx].col = match[1]
			file.MultiCursor[idx].colwant = file.MultiCursor[idx].col
		}
	} else {
		for idx, _ := range file.MultiCursor {
			file.MultiCursor[idx].col = 0
			file.MultiCursor[idx].colwant = file.MultiCursor[idx].col
		}
	}
}

func (file *File) EndOfLine() {
	for idx, _ := range file.MultiCursor {
		row := file.MultiCursor[idx].row
		file.MultiCursor[idx].col = len(file.Buffer[row])
		file.MultiCursor[idx].colwant = file.MultiCursor[idx].col
	}
}

func (file *File) NextWord() {
	for idx, cursor := range file.MultiCursor {
		row := cursor.row
		line := file.Buffer[row]
		col := cursor.col
		re := regexp.MustCompile("[\t ][^\t ]")
		offset := re.FindStringIndex(line[col:].ToString())
		if offset == nil {
			col = len(line)
		} else {
			col += offset[0] + 1
		}
		file.MultiCursor[idx].col = col
		file.MultiCursor[idx].colwant = file.MultiCursor[idx].col
	}
}

func (file *File) PrevWord() {
	for idx, cursor := range file.MultiCursor {
		row := cursor.row
		line := file.Buffer[row]
		col := cursor.col
		re := regexp.MustCompile("[\t ][^\t ]")
		offsets := re.FindAllStringIndex(line[:col].ToString(), -1)
		if offsets == nil {
			col = 0
		} else {
			offset := offsets[len(offsets)-1]
			col = offset[0] + 1
		}
		file.MultiCursor[idx].col = col
		file.MultiCursor[idx].colwant = file.MultiCursor[idx].col
	}
}

func (file *File) GoToLine() {
	lineNo := file.screen.GetPromptAnswer("goto:", &file.gotoHist)
	if lineNo == "" {
		return
	}
	row, err := strconv.Atoi(lineNo)
	if err == nil {
		file.CursorGoTo(row, 0)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Marc Berhault (marc@cockroachlabs.com)

package main

import (
	"database/sql"

	"bazil.org/fuse/fs"
)

const (
	fsSchema = `
CREATE DATABASE fs;

CREATE TABLE fs.namespace (
  parentID INT,
  name     STRING,
  id       INT,
  PRIMARY KEY (parentID, name)
);

CREATE TABLE fs.inode (
  id    INT PRIMARY KEY,
  inode STRING
);
`
)

// CFS implements a filesystem on top of cockroach.
type CFS struct {
	db *sql.DB
}

func (fs CFS) initSchema() error {
	_, err := fs.db.Exec(fsSchema)
	return err
}

func (fs CFS) create(parentID uint64, name, inode string) error {
	var id int64
	if err := fs.db.QueryRow(`SELECT experimental_unique_int()`).Scan(&id); err != nil {
		return err
	}
	tx, err := fs.db.Begin()
	if err != nil {
		return err
	}
	const sql = `
INSERT INTO fs.inode VALUES ($1, $2);
INSERT INTO fs.namespace VALUES ($3, $4, $1);
`
	if _, err := tx.Exec(sql, id, inode, parentID, name); err != nil {
		_ = tx.Rollback()
		return err
	}
	return tx.Commit()
}

func (fs CFS) lookup(parentID uint64, name string) (string, error) {
	const sql = `
SELECT inode FROM fs.inode WHERE id =
  (SELECT id FROM fs.namespace WHERE (parentID, name) = ($1, $2))
`
	var inode string
	if err := fs.db.QueryRow(`sql`, parentID, name).Scan(&inode); err != nil {
		return "", err
	}
	return inode, nil
}

func (fs CFS) list(parentID uint64) ([]string, error) {
	rows, err := fs.db.Query(`SELECT name, id FROM fs.namespace WHERE parentID = $1`, parentID)
	if err != nil {
		return nil, err
	}

	var results []string
	for rows.Next() {
		var name string
		var id int64
		if err := rows.Scan(&name, &id); err != nil {
			return nil, err
		}
		results = append(results, name)
	}
	if err := rows.Err(); err != nil {
		return nil, err
	}

	// TODO(pmattis): Lookup all of the inodes for all of the ids in single
	// "SELECT ... WHERE id IN" statement.
	return results, nil
}

// Root returns the filesystem's root node.
func (fs CFS) Root() (fs.Node, error) {
	return &Node{fs: fs, name: "", id: 0, isDir: true}, nil
}

// GenerateInode returns a new inode ID.
// TODO(marc): if not implemented, the fuse library auto-generates IDs
// from path hashes.
// func (CFS) GenerateInode(parentInode uint64, name string) uint64 {
// 	return 0
// }
<commit_msg>Implement FS.GenerateInode<commit_after>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Marc Berhault (marc@cockroachlabs.com)

package main

import (
	"database/sql"

	"bazil.org/fuse/fs"
)

const (
	fsSchema = `
CREATE DATABASE fs;

CREATE TABLE fs.namespace (
  parentID INT,
  name     STRING,
  id       INT,
  PRIMARY KEY (parentID, name)
);

CREATE TABLE fs.inode (
  id    INT PRIMARY KEY,
  inode STRING
);
`
)

// Root
var _ = fs.FS(&CFS{})

// GenerateInode
var _ = fs.FSInodeGenerator(&CFS{})

// CFS implements a filesystem on top of cockroach.
type CFS struct {
	db *sql.DB
}

func (fs CFS) initSchema() error {
	_, err := fs.db.Exec(fsSchema)
	return err
}

func (fs CFS) create(parentID uint64, name, inode string) error {
	var id int64
	if err := fs.db.QueryRow(`SELECT experimental_unique_int()`).Scan(&id); err != nil {
		return err
	}
	tx, err := fs.db.Begin()
	if err != nil {
		return err
	}
	const sql = `
INSERT INTO fs.inode VALUES ($1, $2);
INSERT INTO fs.namespace VALUES ($3, $4, $1);
`
	if _, err := tx.Exec(sql, id, inode, parentID, name); err != nil {
		_ = tx.Rollback()
		return err
	}
	return tx.Commit()
}

func (fs CFS) lookup(parentID uint64, name string) (string, error) {
	const sql = `
SELECT inode FROM fs.inode WHERE id =
  (SELECT id FROM fs.namespace WHERE (parentID, name) = ($1, $2))
`
	var inode string
	if err := fs.db.QueryRow(`sql`, parentID, name).Scan(&inode); err != nil {
		return "", err
	}
	return inode, nil
}

func (fs CFS) list(parentID uint64) ([]string, error) {
	rows, err := fs.db.Query(`SELECT name, id FROM fs.namespace WHERE parentID = $1`, parentID)
	if err != nil {
		return nil, err
	}

	var results []string
	for rows.Next() {
		var name string
		var id int64
		if err := rows.Scan(&name, &id); err != nil {
			return nil, err
		}
		results = append(results, name)
	}
	if err := rows.Err(); err != nil {
		return nil, err
	}

	// TODO(pmattis): Lookup all of the inodes for all of the ids in single
	// "SELECT ... WHERE id IN" statement.
	return results, nil
}

// Root returns the filesystem's root node.
func (fs CFS) Root() (fs.Node, error) {
	return &Node{fs: fs, name: "", id: 0, isDir: true}, nil
}

// GenerateInode returns a new inode ID.
func (fs CFS) GenerateInode(parentInode uint64, name string) uint64 {
	var id uint64
	if err := fs.db.QueryRow(`SELECT experimental_unique_int()`).Scan(&id); err != nil {
		panic(err)
	}
	return id
}
<|endoftext|>