<commit_before>package app

import (
	"bufio"
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/sclevine/agouti"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"golang.org/x/sync/errgroup"
)

type FileInfo struct {
	Filename string
	Line     uint64
	Start    time.Time
	Done     *sync.WaitGroup // just include parse, exclude write file
}

type URLRes struct {
	URL string
	Res map[string]interface{}
}

type URLInfo struct {
	URL      string
	JsFuncs  []string
	DumpHTML bool
	ResChan  chan URLRes
	FInfo    *FileInfo
}

var (
	fParallel   int
	fDataDir    string
	fScriptDir  string
	fOutputDir  string
	fSplitCount int
)

var CrawlCmd = &cobra.Command{
	Use:   "hamal",
	Short: "Hamal parse webpage based on scripts.",
	RunE: func(cmd *cobra.Command, args []string) error {
		if _, err := os.Stat(fDataDir); os.IsNotExist(err) {
			log.WithFields(log.Fields{
				"dataDir": fDataDir,
			}).Fatal("No script dir")
			return err
		}
		if _, err := os.Stat(fScriptDir); os.IsNotExist(err) {
			log.WithFields(log.Fields{
				"scriptDir": fScriptDir,
			}).Fatal("No script dir")
			return err
		}
		if _, err := os.Stat(fOutputDir); os.IsNotExist(err) {
			os.Mkdir(fOutputDir, os.ModePerm)
			log.WithFields(log.Fields{
				"outputDir": fOutputDir,
			}).Debug("Create output dir")
		}

		return mainFunc()
	},
}

func init() {
	flags := CrawlCmd.Flags()
	flags.IntVarP(&fParallel, "parallel", "p", 10, "max number of parallel exector")
	flags.StringVar(&fDataDir, "dataDir", "./data", "dir for storage url files")
	flags.StringVar(&fScriptDir, "scriptDir", "./script", "dir for storage scripts")
	flags.StringVar(&fOutputDir, "outputDir", "./output", "dir for storage parse result files")
	flags.IntVarP(&fSplitCount, "splitCount", "c", 10000, "max line count for one output file")
	viper.BindPFlag("parallel", flags.Lookup("parallel"))
	viper.BindPFlag("dataDir", flags.Lookup("dataDir"))
	viper.BindPFlag("scriptDir", flags.Lookup("scriptDir"))
	viper.BindPFlag("outputDir", flags.Lookup("outputDir"))
	viper.BindPFlag("splitCount", flags.Lookup("splitCount"))
}

func mainFunc() error {
	var infoeg errgroup.Group
	var retryeg errgroup.Group

	infoChan := make(chan URLInfo, fParallel)
	retryNum := fParallel/2 + 1
	retryChan := make(chan URLInfo, retryNum)
	for i := 0; i < fParallel; i++ {
		index := i
		infoeg.Go(func() error {
		RESTART:
			driver := agouti.PhantomJS()
			if err := driver.Start(); err != nil {
				log.WithFields(log.Fields{
					"index": index,
					"err":   err,
				}).Fatalf("Failed to start driver:%v", err)
				return err
			}
			log.WithField("index", index).Debug("Success to start worker")

			for {
				select {
				case info, ok := <-infoChan:
					if !ok {
						log.WithField("index", index).Debug("Worker exit")
						driver.Stop()
						return nil
					}

					err := parseURL(index, info, driver)
					if err != nil {
						// sometimes phantomjs crashed or just navigate timeout
						// we can't differentiate cause of errors
						// so we just restart the worker and push the *info* to retry queue
						log.WithFields(log.Fields{
							"index": index,
							"info":  info,
						}).Warn("Failed to parse, will retry later")
						go func(info URLInfo) {
							time.Sleep(10 * time.Second)
							retryChan <- info
						}(info)

						driver.Stop()
						goto RESTART
					}

					log.WithField("url", info.URL).Info("Success to parse")
					info.FInfo.Done.Done()
				}
			}
		})
	}
	for i := 0; i < retryNum; i++ {
		index := i
		retryeg.Go(func() error {
		RESTART:
			driver := agouti.PhantomJS()
			if err := driver.Start(); err != nil {
				log.WithFields(log.Fields{
					"index": index,
					"err":   err,
				}).Fatalf("Failed to start driver:%v", err)
				return err
			}
			log.WithField("index", index).Debug("Success to start retry worker")

			for {
				select {
				case info, ok := <-retryChan:
					if !ok {
						log.WithField("index", index).Debug("Retry worker exit")
						driver.Stop()
						return nil
					}
					err := parseURL(index, info, driver)
					if err != nil {
						// failed to retry, no more retry for this url, just mark completed
						log.WithField("url", info.URL).Info("Failed to retry")
						info.FInfo.Done.Done()

						// we need restart retry worker
						driver.Stop()
						goto RESTART
					}
					log.WithField("url", info.URL).Info("Success to parse")
					info.FInfo.Done.Done()
				}
			}
		})
	}

	log.WithField("dataDir", fDataDir).Debug("Start to traversal data files")
	err := filepath.Walk(fDataDir, walkFile(infoChan))
	close(infoChan)
	infoeg.Wait()
	close(retryChan)
	retryeg.Wait()
	log.Debug("Finish all tasks")

	if err != nil {
		return err
	}
	return nil
}

func parseURL(index int, info URLInfo, driver *agouti.WebDriver) error {
	page, err := driver.NewPage(agouti.Browser("phantomjs"))
	if err != nil {
		log.WithFields(log.Fields{
			"index": index,
			"err":   err,
		}).Warn("Failed to create session")
		return err
	}
	defer page.Destroy()

	page.Session().SetPageLoad(300000)
	page.Session().SetScriptTimeout(30000)
	page.Session().SetImplicitWait(0)

	url := info.URL
	log.WithFields(log.Fields{
		"index": index,
		"url":   url,
	}).Debug("Get target url")

	// this step may be blocked until 'page load' timeout
	if err := page.Navigate(url); err != nil {
		log.WithFields(log.Fields{
			"index": index,
			"url":   url,
			"err":   err,
		}).Warn("Failed to navigate to target url")
		return err
	}
	log.WithFields(log.Fields{
		"index": index,
		"url":   url,
	}).Debug("Success to open url")

	res := make(map[string]interface{})
	for i, jsFunc := range info.JsFuncs {
		err := page.RunScript(jsFunc, res, &res)
		if err != nil {
			log.WithFields(log.Fields{
				"index":       index,
				"url":         url,
				"scriptIndex": i,
				"err":         err,
			}).Warn("Failed to run script")
			return err
		}
		log.WithFields(log.Fields{
			"index":       index,
			"url":         url,
			"scriptIndex": i,
			"res":         res,
		}).Debug("Get parse result")

		if val, ok := res["stop"]; ok && val.(bool) {
			return nil
		}

		if val, ok := res["waitTime"]; ok {
			delete(res, "stop")
			delete(res, "waitTime")
			d := time.Duration(val.(float64)) * time.Millisecond
			time.Sleep(d)
		}
	}
	log.WithFields(log.Fields{
		"index": index,
		"url":   url,
	}).Debug("Parse finished")

	if info.DumpHTML {
		res["html"], err = page.HTML()
		if err != nil {
			log.WithFields(log.Fields{
				"index": index,
				"url":   url,
			}).Warn("Failed to get html")
			return err
		}
	}

	if len(res) == 0 {
		log.WithFields(log.Fields{
			"index": index,
			"url":   url,
		}).Debug("Get empty response")
		return nil
	}

	info.ResChan <- URLRes{URL: url, Res: res}

	return nil
}

func walkFile(infoChan chan<- URLInfo) func(path string, f os.FileInfo, err error) error {
	info := new(URLInfo)
	return func(path string, f os.FileInfo, err error) error {
		if f.IsDir() {
			return nil
		}
		log.WithFields(log.Fields{
			"path":     path,
			"fileName": f.Name(),
		}).Debug("Get crawler data file")

		// find script
		filename := f.Name()
		if !viper.IsSet(filename) {
			log.WithFields(log.Fields{
				"filename": filename,
			}).Warn("No conf item for data file")
			return nil
		}
		conf := viper.GetStringMap(filename)
		log.WithFields(log.Fields{
			"filename": filename,
			"conf":     conf,
		}).Debug("Read conf for data file")
		if val, ok := conf["ignore"]; ok && val.(bool) {
			log.WithFields(log.Fields{
				"conf": conf,
			}).Warn("Data file is ignored by conf")
			return nil
		}

		// get 'dump' setting
		if val, ok := conf["dump_html"]; ok && val.(bool) {
			info.DumpHTML = val.(bool)
			log.WithField("dumpHTML", info.DumpHTML).Debug("Read dump_html conf")
		}

		// we can have multi scripts for one page
		if _, ok := conf["script_name"]; !ok {
			log.WithFields(log.Fields{
				"conf": conf,
			}).Warn("Data file's conf has no 'script_name' item")
			return nil
		}
		var scriptPath []string
		switch scripts := conf["script_name"].(type) {
		case string:
			scriptPath = append(scriptPath, filepath.Join(fScriptDir, scripts))
		case []interface{}:
			for _, v := range scripts {
				scriptPath = append(scriptPath, filepath.Join(fScriptDir, v.(string)))
			}
		default:
			log.WithFields(log.Fields{
				"script_name": scripts,
			}).Warn("Conf[script_name] is not string or array of string")
			return nil
		}

		// load scripts
		for _, v := range scriptPath {
			data, err := ioutil.ReadFile(v)
			if err != nil {
				log.WithFields(log.Fields{
					"script_path": v,
					"err":         err,
				}).Warn("Failed to read script")
				return nil
			}
			info.JsFuncs = append(info.JsFuncs, string(data))
		}

		ctx, cancel := context.WithCancel(context.Background())
		// write output file routine
		resChan := make(chan URLRes, fParallel+fParallel/2+1)
		// TODO: make sure finish to write output file before exit
		go func() {
			// create output file
			var resFilename string
			if val, ok := conf["output_file"]; ok {
				resFilename = val.(string)
			} else {
				resFilename = filename
			}
			noSuffix := strings.TrimSuffix(resFilename, filepath.Ext(resFilename))
			resPath := filepath.Join(fOutputDir, noSuffix+".txt")
			resFile, err := os.Create(resPath)
			if err != nil {
				log.WithFields(log.Fields{
					"res_path": resPath,
					"err":      err,
				}).Warn("Failed to create output file")
				cancel()
			}

			line := 0
			index := 0
			for res := range resChan {
				data, _ := json.Marshal(res.Res)
				resFile.WriteString(fmt.Sprintf("%s\t%s\n", res.URL, string(data)))
				line++
				if line >= fSplitCount {
					resFile.Close()
					line = 0
					index++
					resPath = filepath.Join(fOutputDir, noSuffix+"_"+strconv.Itoa(index)+".txt")
					resFile, err = os.Create(resPath)
					if err != nil {
						log.WithFields(log.Fields{
							"res_path": resPath,
							"err":      err,
						}).Warn("Failed to create output file")
						cancel()
					}
				}
			}
			resFile.Close()
		}()

		// read url from data file
		inFile, err := os.Open(path)
		if err != nil {
			log.WithFields(log.Fields{
				"path": path,
				"err":  err,
			}).Fatal("Failed to open data file")
			return nil
		}
		defer inFile.Close()

		fi := FileInfo{
			Filename: filename,
			Start:    time.Now(),
			Done:     new(sync.WaitGroup),
		}
		info.FInfo = &fi
		log.WithFields(log.Fields{
			"filename": filename,
			"start":    info.FInfo.Start,
		}).Info("Start to crawler urls in one file")

		sc := bufio.NewScanner(inFile)
		for sc.Scan() {
			select {
			case <-ctx.Done():
				log.WithFields(log.Fields{
					"filename": filename,
					"line":     atomic.LoadUint64(&info.FInfo.Line),
					"elapsed":  time.Since(info.FInfo.Start),
				}).Info("Partial finished to crawler urls in one file")

				return ctx.Err()
			default:
				info.FInfo.Done.Add(1)
				info.URL = sc.Text()
				info.ResChan = resChan
				atomic.AddUint64(&info.FInfo.Line, 1)
				// log.WithField("info", info).Debug("Create one info")
				infoChan <- *info
			}
		}

		go func() {
			info.FInfo.Done.Wait()
			close(resChan)
			log.WithFields(log.Fields{
				"filename": filename,
				"line":     atomic.LoadUint64(&info.FInfo.Line),
				"elapsed":  time.Since(info.FInfo.Start),
			}).Info("Finished to crawler urls in one file")
		}()

		return nil
	}
}
<commit_msg>fix crawl subcommand<commit_after>package app

import (
	"bufio"
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/sclevine/agouti"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"golang.org/x/sync/errgroup"
)

type FileInfo struct {
	Filename string
	Line     uint64
	Start    time.Time
	Done     *sync.WaitGroup // just include parse, exclude write file
}

type URLRes struct {
	URL string
	Res map[string]interface{}
}

type URLInfo struct {
	URL      string
	JsFuncs  []string
	DumpHTML bool
	ResChan  chan URLRes
	FInfo    *FileInfo
}

var (
	fParallel   int
	fDataDir    string
	fScriptDir  string
	fOutputDir  string
	fSplitCount int
)

var CrawlCmd = &cobra.Command{
	Use:   "crawl",
	Short: "Crawl parse webpage based on javascripts, then save parse results.",
	PreRunE: func(cmd *cobra.Command, args []string) error {
		if _, err := os.Stat(fDataDir); os.IsNotExist(err) {
			log.WithFields(log.Fields{
				"dataDir": fDataDir,
			}).Fatal("No data dir")
			return err
		}
		if _, err := os.Stat(fScriptDir); os.IsNotExist(err) {
			log.WithFields(log.Fields{
				"scriptDir": fScriptDir,
			}).Fatal("No script dir")
			return err
		}
		if _, err := os.Stat(fOutputDir); os.IsNotExist(err) {
			os.Mkdir(fOutputDir, os.ModePerm)
			log.WithFields(log.Fields{
				"outputDir": fOutputDir,
			}).Debug("Create output dir")
		}

		return mainFunc()
	},
}

func init() {
	flags := CrawlCmd.Flags()
	flags.IntVarP(&fParallel, "parallel", "p", 10, "max number of parallel exector")
	flags.StringVar(&fDataDir, "dataDir", "./data", "dir for storage url files")
	flags.StringVar(&fScriptDir, "scriptDir", "./script", "dir for storage scripts")
	flags.StringVar(&fOutputDir, "outputDir", "./output", "dir for storage parse result files")
	flags.IntVarP(&fSplitCount, "splitCount", "c", 10000, "max line count for one output file")
	viper.BindPFlag("parallel", flags.Lookup("parallel"))
	viper.BindPFlag("dataDir", flags.Lookup("dataDir"))
	viper.BindPFlag("scriptDir", flags.Lookup("scriptDir"))
	viper.BindPFlag("outputDir", flags.Lookup("outputDir"))
	viper.BindPFlag("splitCount", flags.Lookup("splitCount"))
}

func mainFunc() error {
	var infoeg errgroup.Group
	var retryeg errgroup.Group

	infoChan := make(chan URLInfo, fParallel)
	retryNum := fParallel/2 + 1
	retryChan := make(chan URLInfo, retryNum)
	for i := 0; i < fParallel; i++ {
		index := i
		infoeg.Go(func() error {
		RESTART:
			driver := agouti.PhantomJS()
			if err := driver.Start(); err != nil {
				log.WithFields(log.Fields{
					"index": index,
					"err":   err,
				}).Fatalf("Failed to start driver:%v", err)
				return err
			}
			log.WithField("index", index).Debug("Success to start worker")

			for {
				select {
				case info, ok := <-infoChan:
					if !ok {
						log.WithField("index", index).Debug("Worker exit")
						driver.Stop()
						return nil
					}

					err := parseURL(index, info, driver)
					if err != nil {
						// sometimes phantomjs crashed or just navigate timeout
						// we can't differentiate cause of errors
						// so we just restart the worker and push the *info* to retry queue
						log.WithFields(log.Fields{
							"index": index,
							"info":  info,
						}).Warn("Failed to parse, will retry later")
						go func(info URLInfo) {
							time.Sleep(10 * time.Second)
							retryChan <- info
						}(info)

						driver.Stop()
						goto RESTART
					}

					log.WithField("url", info.URL).Info("Success to parse")
					info.FInfo.Done.Done()
				}
			}
		})
	}
	for i := 0; i < retryNum; i++ {
		index := i
		retryeg.Go(func() error {
		RESTART:
			driver := agouti.PhantomJS()
			if err := driver.Start(); err != nil {
				log.WithFields(log.Fields{
					"index": index,
					"err":   err,
				}).Fatalf("Failed to start driver:%v", err)
				return err
			}
			log.WithField("index", index).Debug("Success to start retry worker")

			for {
				select {
				case info, ok := <-retryChan:
					if !ok {
						log.WithField("index", index).Debug("Retry worker exit")
						driver.Stop()
						return nil
					}
					err := parseURL(index, info, driver)
					if err != nil {
						// failed to retry, no more retry for this url, just mark completed
						log.WithField("url", info.URL).Info("Failed to retry")
						info.FInfo.Done.Done()

						// we need restart retry worker
						driver.Stop()
						goto RESTART
					}
					log.WithField("url", info.URL).Info("Success to parse")
					info.FInfo.Done.Done()
				}
			}
		})
	}

	log.WithField("dataDir", fDataDir).Debug("Start to traversal data files")
	err := filepath.Walk(fDataDir, walkFile(infoChan))
	close(infoChan)
	infoeg.Wait()
	close(retryChan)
	retryeg.Wait()
	log.Debug("Finish all tasks")

	if err != nil {
		return err
	}
	return nil
}

func parseURL(index int, info URLInfo, driver *agouti.WebDriver) error {
	page, err := driver.NewPage(agouti.Browser("phantomjs"))
	if err != nil {
		log.WithFields(log.Fields{
			"index": index,
			"err":   err,
		}).Warn("Failed to create session")
		return err
	}
	defer page.Destroy()

	page.Session().SetPageLoad(300000)
	page.Session().SetScriptTimeout(30000)
	page.Session().SetImplicitWait(0)

	url := info.URL
	log.WithFields(log.Fields{
		"index": index,
		"url":   url,
	}).Debug("Get target url")

	// this step may be blocked until 'page load' timeout
	if err := page.Navigate(url); err != nil {
		log.WithFields(log.Fields{
			"index": index,
			"url":   url,
			"err":   err,
		}).Warn("Failed to navigate to target url")
		return err
	}
	log.WithFields(log.Fields{
		"index": index,
		"url":   url,
	}).Debug("Success to open url")

	res := make(map[string]interface{})
	for i, jsFunc := range info.JsFuncs {
		err := page.RunScript(jsFunc, res, &res)
		if err != nil {
			log.WithFields(log.Fields{
				"index":       index,
				"url":         url,
				"scriptIndex": i,
				"err":         err,
			}).Warn("Failed to run script")
			return err
		}
		log.WithFields(log.Fields{
			"index":       index,
			"url":         url,
			"scriptIndex": i,
			"res":         res,
		}).Debug("Get parse result")

		if val, ok := res["stop"]; ok && val.(bool) {
			return nil
		}

		if val, ok := res["waitTime"]; ok {
			delete(res, "stop")
			delete(res, "waitTime")
			d := time.Duration(val.(float64)) * time.Millisecond
			time.Sleep(d)
		}
	}
	log.WithFields(log.Fields{
		"index": index,
		"url":   url,
	}).Debug("Parse finished")

	if info.DumpHTML {
		res["html"], err = page.HTML()
		if err != nil {
			log.WithFields(log.Fields{
				"index": index,
				"url":   url,
			}).Warn("Failed to get html")
			return err
		}
	}

	if len(res) == 0 {
		log.WithFields(log.Fields{
			"index": index,
			"url":   url,
		}).Debug("Get empty response")
		return nil
	}

	info.ResChan <- URLRes{URL: url, Res: res}

	return nil
}

func walkFile(infoChan chan<- URLInfo) func(path string, f os.FileInfo, err error) error {
	info := new(URLInfo)
	return func(path string, f os.FileInfo, err error) error {
		if f.IsDir() {
			return nil
		}
		log.WithFields(log.Fields{
			"path":     path,
			"fileName": f.Name(),
		}).Debug("Get crawler data file")

		// find script
		filename := f.Name()
		if !viper.IsSet(filename) {
			log.WithFields(log.Fields{
				"filename": filename,
			}).Warn("No conf item for data file")
			return nil
		}
		conf := viper.GetStringMap(filename)
		log.WithFields(log.Fields{
			"filename": filename,
			"conf":     conf,
		}).Debug("Read conf for data file")
		if val, ok := conf["ignore"]; ok && val.(bool) {
			log.WithFields(log.Fields{
				"conf": conf,
			}).Warn("Data file is ignored by conf")
			return nil
		}

		// get 'dump' setting
		if val, ok := conf["dump_html"]; ok && val.(bool) {
			info.DumpHTML = val.(bool)
			log.WithField("dumpHTML", info.DumpHTML).Debug("Read dump_html conf")
		}

		// we can have multi scripts for one page
		if _, ok := conf["script_name"]; !ok {
			log.WithFields(log.Fields{
				"conf": conf,
			}).Warn("Data file's conf has no 'script_name' item")
			return nil
		}
		var scriptPath []string
		switch scripts := conf["script_name"].(type) {
		case string:
			scriptPath = append(scriptPath, filepath.Join(fScriptDir, scripts))
		case []interface{}:
			for _, v := range scripts {
				scriptPath = append(scriptPath, filepath.Join(fScriptDir, v.(string)))
			}
		default:
			log.WithFields(log.Fields{
				"script_name": scripts,
			}).Warn("Conf[script_name] is not string or array of string")
			return nil
		}

		// load scripts
		for _, v := range scriptPath {
			data, err := ioutil.ReadFile(v)
			if err != nil {
				log.WithFields(log.Fields{
					"script_path": v,
					"err":         err,
				}).Warn("Failed to read script")
				return nil
			}
			info.JsFuncs = append(info.JsFuncs, string(data))
		}

		ctx, cancel := context.WithCancel(context.Background())
		// write output file routine
		resChan := make(chan URLRes, fParallel+fParallel/2+1)
		// TODO: make sure finish to write output file before exit
		go func() {
			// create output file
			var resFilename string
			if val, ok := conf["output_file"]; ok {
				resFilename = val.(string)
			} else {
				resFilename = filename
			}
			noSuffix := strings.TrimSuffix(resFilename, filepath.Ext(resFilename))
			resPath := filepath.Join(fOutputDir, noSuffix+".txt")
			resFile, err := os.Create(resPath)
			if err != nil {
				log.WithFields(log.Fields{
					"res_path": resPath,
					"err":      err,
				}).Warn("Failed to create output file")
				cancel()
			}

			line := 0
			index := 0
			for res := range resChan {
				data, _ := json.Marshal(res.Res)
				resFile.WriteString(fmt.Sprintf("%s\t%s\n", res.URL, string(data)))
				line++
				if line >= fSplitCount {
					resFile.Close()
					line = 0
					index++
					resPath = filepath.Join(fOutputDir, noSuffix+"_"+strconv.Itoa(index)+".txt")
					resFile, err = os.Create(resPath)
					if err != nil {
						log.WithFields(log.Fields{
							"res_path": resPath,
							"err":      err,
						}).Warn("Failed to create output file")
						cancel()
					}
				}
			}
			resFile.Close()
		}()

		// read url from data file
		inFile, err := os.Open(path)
		if err != nil {
			log.WithFields(log.Fields{
				"path": path,
				"err":  err,
			}).Fatal("Failed to open data file")
			return nil
		}
		defer inFile.Close()

		fi := FileInfo{
			Filename: filename,
			Start:    time.Now(),
			Done:     new(sync.WaitGroup),
		}
		info.FInfo = &fi
		log.WithFields(log.Fields{
			"filename": filename,
			"start":    info.FInfo.Start,
		}).Info("Start to crawler urls in one file")

		sc := bufio.NewScanner(inFile)
		for sc.Scan() {
			select {
			case <-ctx.Done():
				log.WithFields(log.Fields{
					"filename": filename,
					"line":     atomic.LoadUint64(&info.FInfo.Line),
					"elapsed":  time.Since(info.FInfo.Start),
				}).Info("Partial finished to crawler urls in one file")

				return ctx.Err()
			default:
				info.FInfo.Done.Add(1)
				info.URL = sc.Text()
				info.ResChan = resChan
				atomic.AddUint64(&info.FInfo.Line, 1)
				// log.WithField("info", info).Debug("Create one info")
				infoChan <- *info
			}
		}

		go func() {
			info.FInfo.Done.Wait()
			close(resChan)
			log.WithFields(log.Fields{
				"filename": filename,
				"line":     atomic.LoadUint64(&info.FInfo.Line),
				"elapsed":  time.Since(info.FInfo.Start),
			}).Info("Finished to crawler urls in one file")
		}()

		return nil
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package md

import (
	"bytes"
	"strings"

	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"
)

var (
	// headerLevel maps HTML tags to their level in parser.HeaderNode.
	// we -1 as H2 is a new step
	headerLevel = map[atom.Atom]int{
		atom.H3: 2,
		atom.H4: 3,
		atom.H5: 4,
		atom.H6: 5,
	}
)

// isHeader returns true if hn is one of secondary headers.
// Step header is not one of them.
func isHeader(hn *html.Node) bool {
	_, ok := headerLevel[hn.DataAtom]
	return ok
}

func isMeta(hn *html.Node) bool {
	elem := strings.ToLower(hn.Data)
	return strings.HasPrefix(elem, metaDuration+metaSep) || strings.HasPrefix(elem, metaEnvironment+metaSep)
}

func isBold(hn *html.Node) bool {
	if hn.Type == html.TextNode {
		hn = hn.Parent
	} else if hn.DataAtom == atom.Code {
		// Look up as many as 2 levels, to handle the case of e.g. <bold>
		for i:= 0; i < 2; i++ {
			hn = hn.Parent
			if hn.DataAtom == atom.Strong || hn.DataAtom == atom.B {
				return true
			}
		}
		return false
	}
	return hn.DataAtom == atom.Strong ||
		hn.DataAtom == atom.B
}

func isItalic(hn *html.Node) bool {
	if hn.Type == html.TextNode {
		hn = hn.Parent
	} else if hn.DataAtom == atom.Code {
		// Look up as many as 2 levels, to handle the case of e.g. <bold>
		for i:= 0; i < 2; i++ {
			hn = hn.Parent
			if hn.DataAtom == atom.Em || hn.DataAtom == atom.I {
				return true
			}
		}
		return false
	}
	return hn.DataAtom == atom.Em ||
		hn.DataAtom == atom.I
}

// This is different to calling isBold and isItalic seperately as we must look
// up an extra level in the tree
func isBoldAndItalic(hn *html.Node) bool {
	if hn.Parent == nil || hn.Parent.Parent == nil {
		return false
	}
	if hn.Type == html.TextNode {
		hn = hn.Parent
	}
	return (isItalic(hn) && isBold(hn.Parent)) || (isItalic(hn.Parent) && isBold(hn))

}

func isConsole(hn *html.Node) bool {
    if hn.Type == html.TextNode {
        hn = hn.Parent
    }
    if (hn.DataAtom == atom.Code) {
        for _, a := range hn.Attr {
            if (a.Key == "class" && a.Val == "language-console") {
                return true;
            }
        }
    }
    return false;
}

func isCode(hn *html.Node) bool {
	if hn.Type == html.TextNode {
		hn = hn.Parent
	}
	return hn.DataAtom == atom.Code && !isConsole(hn)
}

func isButton(hn *html.Node) bool {
	return hn.DataAtom == atom.Button
}

func isAside(hn *html.Node) bool {
	return hn.DataAtom == atom.Aside
}

func isNewAside(hn *html.Node) bool {
	if hn.FirstChild == nil ||
	   hn.FirstChild.NextSibling == nil ||
	   hn.FirstChild.NextSibling.FirstChild == nil {
		return false
	}

	bq := hn.DataAtom == atom.Blockquote
	apn := strings.HasPrefix(strings.ToLower(hn.FirstChild.NextSibling.FirstChild.Data), "aside positive") ||
	       strings.HasPrefix(strings.ToLower(hn.FirstChild.NextSibling.FirstChild.Data), "aside negative")
	return bq && apn
}

func isInfobox(hn *html.Node) bool {
	if hn.DataAtom != atom.Dt {
		return false
	}
	return strings.ToLower(hn.FirstChild.Data) == "positive" || isInfoboxNegative(hn)
}

func isInfoboxNegative(hn *html.Node) bool {
	if hn.DataAtom != atom.Dt {
		return false
	}
	return strings.ToLower(hn.FirstChild.Data) == "negative"
}

func isSurvey(hn *html.Node) bool {
	if hn.DataAtom != atom.Form {
		return false
	}
	if findAtom(hn, atom.Name) == nil {
		return false
	}
	if len(findChildAtoms(hn, atom.Input)) == 0 {
		return false
	}
	return true
}

func isTable(hn *html.Node) bool {
	if hn.DataAtom != atom.Table {
		return false
	}
	return countTwo(hn, atom.Tr) >= 1 || countTwo(hn, atom.Td) >= 1
}

func isList(hn *html.Node) bool {
	return hn.DataAtom == atom.Ul || hn.DataAtom == atom.Ol
}

func isYoutube(hn *html.Node) bool {
	return hn.DataAtom == atom.Video
}

func isFragmentImport(hn *html.Node) bool {
	return hn.DataAtom == 0 && strings.HasPrefix(hn.Data, convertedImportsDataPrefix)
}

// countTwo starts counting the number of a Atom children in hn.
// It returns as soon as the count exceeds 1, so the returned value is inexact.
//
// The callers can test for > 1 to verify whether a node contains two
// or more children of the Atom a.
func countTwo(hn *html.Node, a atom.Atom) int {
	var count int
	for c := hn.FirstChild; c != nil; c = c.NextSibling {
		if c.DataAtom == a {
			count++
		} else {
			count += countTwo(c, a)
		}
		if count > 1 {
			break
		}
	}
	return count
}

// countDirect returns the number of immediate children of hn.
func countDirect(hn *html.Node) int {
	var count int
	for c := hn.FirstChild; c != nil; c = c.NextSibling {
		count++
	}
	return count
}

// findAtom returns first child of root which matches a, nil otherwise.
// It returns root if it is the same Atom as a.
func findAtom(root *html.Node, a atom.Atom) *html.Node {
	if root.DataAtom == a {
		return root
	}
	for c := root.FirstChild; c != nil; c = c.NextSibling {
		if v := findAtom(c, a); v != nil {
			return v
		}
	}
	return nil
}

func findChildAtoms(root *html.Node, a atom.Atom) []*html.Node {
	var nodes []*html.Node
	for hn := root.FirstChild; hn != nil; hn = hn.NextSibling {
		if hn.DataAtom == a {
			nodes = append(nodes, hn)
		}
		nodes = append(nodes, findChildAtoms(hn, a)...)
	}
	return nodes
}

// findParent is like findAtom but search is in the opposite direction.
// It is faster to look for parent than child lookup in findAtom.
func findParent(root *html.Node, a atom.Atom) *html.Node {
	if root.DataAtom == a {
		return root
	}
	for c := root.Parent; c != nil; c = c.Parent {
		if c.DataAtom == a {
			return c
		}
	}
	return nil
}

var blockParents = map[atom.Atom]struct{}{
	atom.H1:  {},
	atom.H2:  {},
	atom.H3:  {},
	atom.H4:  {},
	atom.H5:  {},
	atom.H6:  {},
	atom.Li:  {},
	atom.P:   {},
	atom.Div: {},
}

// findBlockParent looks up nearest block parent node of hn.
// For instance, block parent of "text" in

text

 is
,
// while block parent of "text" in
text
 is
.
func findBlockParent(hn *html.Node) *html.Node {
	for p := hn.Parent; p != nil; p = p.Parent {
		if _, ok := blockParents[p.DataAtom]; ok {
			return p
		}
	}
	return nil
}

// nodeAttr returns node attribute value of the key name.
// Attribute keys are case insensitive.
func nodeAttr(n *html.Node, name string) string {
	name = strings.ToLower(name)
	for _, a := range n.Attr {
		if strings.ToLower(a.Key) == name {
			return a.Val
		}
	}
	return ""
}

// stringifyNode extracts and concatenates all text nodes starting with root.
// Line breaks are inserted at
 and any non- elements.
func stringifyNode(root *html.Node, trim bool) string {
	if root.Type == html.TextNode {
		s := textCleaner.Replace(root.Data)
		s = strings.Replace(s, "\n", " ", -1)
		if !trim {
			return s
		}
		return strings.TrimSpace(s)
	}
	if root.DataAtom == atom.Br && !trim {
		return "\n"
	}
	var buf bytes.Buffer
	for c := root.FirstChild; c != nil; c = c.NextSibling {
		if c.DataAtom == atom.Br {
			buf.WriteRune('\n')
			continue
		}
		if c.Type == html.TextNode {
			buf.WriteString(c.Data)
			continue
		}
		if c.DataAtom != atom.Span && c.DataAtom != atom.A {
			buf.WriteRune('\n')
		}
		buf.WriteString(stringifyNode(c, false))
	}
	s := textCleaner.Replace(buf.String())
	if !trim {
		return s
	}
	return strings.TrimSpace(s)
}
<commit_msg>Fix typo<commit_after>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package md

import (
	"bytes"
	"strings"

	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"
)

var (
	// headerLevel maps HTML tags to their level in parser.HeaderNode.
	// we -1 as H2 is a new step
	headerLevel = map[atom.Atom]int{
		atom.H3: 2,
		atom.H4: 3,
		atom.H5: 4,
		atom.H6: 5,
	}
)

// isHeader returns true if hn is one of secondary headers.
// Step header is not one of them.
func isHeader(hn *html.Node) bool {
	_, ok := headerLevel[hn.DataAtom]
	return ok
}

func isMeta(hn *html.Node) bool {
	elem := strings.ToLower(hn.Data)
	return strings.HasPrefix(elem, metaDuration+metaSep) || strings.HasPrefix(elem, metaEnvironment+metaSep)
}

func isBold(hn *html.Node) bool {
	if hn.Type == html.TextNode {
		hn = hn.Parent
	} else if hn.DataAtom == atom.Code {
		// Look up as many as 2 levels, to handle the case of e.g. <bold>
		for i := 0; i < 2; i++ {
			hn = hn.Parent
			if hn.DataAtom == atom.Strong || hn.DataAtom == atom.B {
				return true
			}
		}
		return false
	}
	return hn.DataAtom == atom.Strong ||
		hn.DataAtom == atom.B
}

func isItalic(hn *html.Node) bool {
	if hn.Type == html.TextNode {
		hn = hn.Parent
	} else if hn.DataAtom == atom.Code {
		// Look up as many as 2 levels, to handle the case of e.g. <bold>
		for i := 0; i < 2; i++ {
			hn = hn.Parent
			if hn.DataAtom == atom.Em || hn.DataAtom == atom.I {
				return true
			}
		}
		return false
	}
	return hn.DataAtom == atom.Em ||
		hn.DataAtom == atom.I
}

// This is different to calling isBold and isItalic separately as we must look
// up an extra level in the tree
func isBoldAndItalic(hn *html.Node) bool {
	if hn.Parent == nil || hn.Parent.Parent == nil {
		return false
	}
	if hn.Type == html.TextNode {
		hn = hn.Parent
	}
	return (isItalic(hn) && isBold(hn.Parent)) || (isItalic(hn.Parent) && isBold(hn))

}

func isConsole(hn *html.Node) bool {
	if hn.Type == html.TextNode {
		hn = hn.Parent
	}
	if hn.DataAtom == atom.Code {
		for _, a := range hn.Attr {
			if a.Key == "class" && a.Val == "language-console" {
				return true
			}
		}
	}
	return false
}

func isCode(hn *html.Node) bool {
	if hn.Type == html.TextNode {
		hn = hn.Parent
	}
	return hn.DataAtom == atom.Code && !isConsole(hn)
}

func isButton(hn *html.Node) bool {
	return hn.DataAtom == atom.Button
}

func isAside(hn *html.Node) bool {
	return hn.DataAtom == atom.Aside
}

func isNewAside(hn *html.Node) bool {
	if hn.FirstChild == nil ||
		hn.FirstChild.NextSibling == nil ||
		hn.FirstChild.NextSibling.FirstChild == nil {
		return false
	}

	bq := hn.DataAtom == atom.Blockquote
	apn := strings.HasPrefix(strings.ToLower(hn.FirstChild.NextSibling.FirstChild.Data), "aside positive") ||
		strings.HasPrefix(strings.ToLower(hn.FirstChild.NextSibling.FirstChild.Data), "aside negative")
	return bq && apn
}

func isInfobox(hn *html.Node) bool {
	if hn.DataAtom != atom.Dt {
		return false
	}
	return strings.ToLower(hn.FirstChild.Data) == "positive" || isInfoboxNegative(hn)
}

func isInfoboxNegative(hn *html.Node) bool {
	if hn.DataAtom != atom.Dt {
		return false
	}
	return strings.ToLower(hn.FirstChild.Data) == "negative"
}

func isSurvey(hn *html.Node) bool {
	if hn.DataAtom != atom.Form {
		return false
	}
	if findAtom(hn, atom.Name) == nil {
		return false
	}
	if len(findChildAtoms(hn, atom.Input)) == 0 {
		return false
	}
	return true
}

func isTable(hn *html.Node) bool {
	if hn.DataAtom != atom.Table {
		return false
	}
	return countTwo(hn, atom.Tr) >= 1 || countTwo(hn, atom.Td) >= 1
}

func isList(hn *html.Node) bool {
	return hn.DataAtom == atom.Ul || hn.DataAtom == atom.Ol
}

func isYoutube(hn *html.Node) bool {
	return hn.DataAtom == atom.Video
}

func isFragmentImport(hn *html.Node) bool {
	return hn.DataAtom == 0 && strings.HasPrefix(hn.Data, convertedImportsDataPrefix)
}

// countTwo starts counting the number of a Atom children in hn.
// It returns as soon as the count exceeds 1, so the returned value is inexact.
//
// The callers can test for > 1 to verify whether a node contains two
// or more children of the Atom a.
func countTwo(hn *html.Node, a atom.Atom) int {
	var count int
	for c := hn.FirstChild; c != nil; c = c.NextSibling {
		if c.DataAtom == a {
			count++
		} else {
			count += countTwo(c, a)
		}
		if count > 1 {
			break
		}
	}
	return count
}

// countDirect returns the number of immediate children of hn.
func countDirect(hn *html.Node) int {
	var count int
	for c := hn.FirstChild; c != nil; c = c.NextSibling {
		count++
	}
	return count
}

// findAtom returns first child of root which matches a, nil otherwise.
// It returns root if it is the same Atom as a.
func findAtom(root *html.Node, a atom.Atom) *html.Node {
	if root.DataAtom == a {
		return root
	}
	for c := root.FirstChild; c != nil; c = c.NextSibling {
		if v := findAtom(c, a); v != nil {
			return v
		}
	}
	return nil
}

func findChildAtoms(root *html.Node, a atom.Atom) []*html.Node {
	var nodes []*html.Node
	for hn := root.FirstChild; hn != nil; hn = hn.NextSibling {
		if hn.DataAtom == a {
			nodes = append(nodes, hn)
		}
		nodes = append(nodes, findChildAtoms(hn, a)...)
	}
	return nodes
}

// findParent is like findAtom but search is in the opposite direction.
// It is faster to look for parent than child lookup in findAtom.
func findParent(root *html.Node, a atom.Atom) *html.Node {
	if root.DataAtom == a {
		return root
	}
	for c := root.Parent; c != nil; c = c.Parent {
		if c.DataAtom == a {
			return c
		}
	}
	return nil
}

var blockParents = map[atom.Atom]struct{}{
	atom.H1:  {},
	atom.H2:  {},
	atom.H3:  {},
	atom.H4:  {},
	atom.H5:  {},
	atom.H6:  {},
	atom.Li:  {},
	atom.P:   {},
	atom.Div: {},
}

// findBlockParent looks up nearest block parent node of hn.
// For instance, block parent of "text" in

text

 is
,
// while block parent of "text" in
text
 is
.
func findBlockParent(hn *html.Node) *html.Node {
	for p := hn.Parent; p != nil; p = p.Parent {
		if _, ok := blockParents[p.DataAtom]; ok {
			return p
		}
	}
	return nil
}

// nodeAttr returns node attribute value of the key name.
// Attribute keys are case insensitive.
func nodeAttr(n *html.Node, name string) string {
	name = strings.ToLower(name)
	for _, a := range n.Attr {
		if strings.ToLower(a.Key) == name {
			return a.Val
		}
	}
	return ""
}

// stringifyNode extracts and concatenates all text nodes starting with root.
// Line breaks are inserted at
 and any non- elements.
func stringifyNode(root *html.Node, trim bool) string {
	if root.Type == html.TextNode {
		s := textCleaner.Replace(root.Data)
		s = strings.Replace(s, "\n", " ", -1)
		if !trim {
			return s
		}
		return strings.TrimSpace(s)
	}
	if root.DataAtom == atom.Br && !trim {
		return "\n"
	}
	var buf bytes.Buffer
	for c := root.FirstChild; c != nil; c = c.NextSibling {
		if c.DataAtom == atom.Br {
			buf.WriteRune('\n')
			continue
		}
		if c.Type == html.TextNode {
			buf.WriteString(c.Data)
			continue
		}
		if c.DataAtom != atom.Span && c.DataAtom != atom.A {
			buf.WriteRune('\n')
		}
		buf.WriteString(stringifyNode(c, false))
	}
	s := textCleaner.Replace(buf.String())
	if !trim {
		return s
	}
	return strings.TrimSpace(s)
}
<|endoftext|>


<commit_before>/* Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import (
	"fmt"
	"log"
	"regexp"
	"strings"

	"github.com/mvdan/fdroidcl"
	"github.com/mvdan/fdroidcl/adb"
)

var cmdSearch = &Command{
	UsageLine: "search <regexp...>",
	Short:     "Search available apps",
}

var (
	quiet     = cmdSearch.Flag.Bool("q", false, "Print package names only")
	installed = cmdSearch.Flag.Bool("i", false, "Filter installed packages")
)

func init() {
	cmdSearch.Run = runSearch
}

func runSearch(args []string) {
	index := mustLoadIndex()
	apps := filterAppsSearch(index.Apps, args)
	if *installed {
		device := oneDevice()
		installed := mustInstalled(device)
		apps = filterAppsInstalled(apps, installed)
	}
	if *quiet {
		for _, app := range apps {
			fmt.Println(app.ID)
		}
	} else {
		printApps(apps)
	}
}

func filterAppsSearch(apps []fdroidcl.App, terms []string) []fdroidcl.App {
	regexes := make([]*regexp.Regexp, len(terms))
	for i, term := range terms {
		regexes[i] = regexp.MustCompile(term)
	}
	var result []fdroidcl.App
	for _, app := range apps {
		fields := []string{
			strings.ToLower(app.ID),
			strings.ToLower(app.Name),
			strings.ToLower(app.Summary),
			strings.ToLower(app.Desc),
		}
		if !appMatches(fields, regexes) {
			continue
		}
		result = append(result, app)
	}
	return result
}

func appMatches(fields []string, regexes []*regexp.Regexp) bool {
fieldLoop:
	for _, field := range fields {
		for _, regex := range regexes {
			if !regex.MatchString(field) {
				continue fieldLoop
			}
		}
		return true
	}
	return false
}

func printApps(apps []fdroidcl.App) {
	maxIDLen := 0
	for _, app := range apps {
		if len(app.ID) > maxIDLen {
			maxIDLen = len(app.ID)
		}
	}
	for _, app := range apps {
		printApp(app, maxIDLen)
	}
}

func printApp(app fdroidcl.App, IDLen int) {
	fmt.Printf("%s%s %s %s\n", app.ID, strings.Repeat(" ", IDLen-len(app.ID)),
		app.Name, app.CurApk.VName)
	fmt.Printf("    %s\n", app.Summary)
}

func mustInstalled(device adb.Device) []adb.Package {
	installed, err := device.Installed()
	if err != nil {
		log.Fatalf("Could not get installed packages: %v", err)
	}
	return installed
}

func filterAppsInstalled(apps []fdroidcl.App, installed []adb.Package) []fdroidcl.App {
	instMap := make(map[string]struct{}, len(installed))
	for _, p := range installed {
		instMap[p.ID] = struct{}{}
	}
	var result []fdroidcl.App
	for _, app := range apps {
		if _, e := instMap[app.ID]; !e {
			continue
		}
		result = append(result, app)
	}
	return result
}
<commit_msg>Add -u flag to search to filter updates<commit_after>/* Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import (
	"fmt"
	"log"
	"regexp"
	"strings"

	"github.com/mvdan/fdroidcl"
	"github.com/mvdan/fdroidcl/adb"
)

var cmdSearch = &Command{
	UsageLine: "search <regexp...>",
	Short:     "Search available apps",
}

var (
	quiet     = cmdSearch.Flag.Bool("q", false, "Print package names only")
	installed = cmdSearch.Flag.Bool("i", false, "Filter installed apps")
	updates   = cmdSearch.Flag.Bool("u", false, "Filter apps with updates")
)

func init() {
	cmdSearch.Run = runSearch
}

func runSearch(args []string) {
	index := mustLoadIndex()
	apps := filterAppsSearch(index.Apps, args)
	if *installed && *updates {
		fmt.Println("-i and -u at the same time don't make sense")
		cmdSearch.Flag.Usage()
	}
	if *installed {
		device := oneDevice()
		installed := mustInstalled(device)
		apps = filterAppsInstalled(apps, installed)
	}
	if *updates {
		device := oneDevice()
		installed := mustInstalled(device)
		apps = filterAppsUpdates(apps, installed)
	}
	if *quiet {
		for _, app := range apps {
			fmt.Println(app.ID)
		}
	} else {
		printApps(apps)
	}
}

func filterAppsSearch(apps []fdroidcl.App, terms []string) []fdroidcl.App {
	regexes := make([]*regexp.Regexp, len(terms))
	for i, term := range terms {
		regexes[i] = regexp.MustCompile(term)
	}
	var result []fdroidcl.App
	for _, app := range apps {
		fields := []string{
			strings.ToLower(app.ID),
			strings.ToLower(app.Name),
			strings.ToLower(app.Summary),
			strings.ToLower(app.Desc),
		}
		if !appMatches(fields, regexes) {
			continue
		}
		result = append(result, app)
	}
	return result
}

func appMatches(fields []string, regexes []*regexp.Regexp) bool {
fieldLoop:
	for _, field := range fields {
		for _, regex := range regexes {
			if !regex.MatchString(field) {
				continue fieldLoop
			}
		}
		return true
	}
	return false
}

func printApps(apps []fdroidcl.App) {
	maxIDLen := 0
	for _, app := range apps {
		if len(app.ID) > maxIDLen {
			maxIDLen = len(app.ID)
		}
	}
	for _, app := range apps {
		printApp(app, maxIDLen)
	}
}

func printApp(app fdroidcl.App, IDLen int) {
	fmt.Printf("%s%s %s %s\n", app.ID, strings.Repeat(" ", IDLen-len(app.ID)),
		app.Name, app.CurApk.VName)
	fmt.Printf("    %s\n", app.Summary)
}

func mustInstalled(device adb.Device) []adb.Package {
	installed, err := device.Installed()
	if err != nil {
		log.Fatalf("Could not get installed packages: %v", err)
	}
	return installed
}

func filterAppsInstalled(apps []fdroidcl.App, installed []adb.Package) []fdroidcl.App {
	instMap := make(map[string]struct{}, len(installed))
	for _, p := range installed {
		instMap[p.ID] = struct{}{}
	}
	var result []fdroidcl.App
	for _, app := range apps {
		if _, e := instMap[app.ID]; !e {
			continue
		}
		result = append(result, app)
	}
	return result
}

func filterAppsUpdates(apps []fdroidcl.App, installed []adb.Package) []fdroidcl.App {
	instMap := make(map[string]*adb.Package, len(installed))
	for i := range installed {
		p := &installed[i]
		instMap[p.ID] = p
	}
	var result []fdroidcl.App
	for _, app := range apps {
		p, e := instMap[app.ID]
		if !e {
			continue
		}
		if p.VCode >= app.CurApk.VCode {
			continue
		}
		result = append(result, app)
	}
	return result
}
<|endoftext|>


<commit_before>package main

import (
	"expvar"
	"fmt"
	"io"
	"log"
	"os"
	"os/signal"
	"sync/atomic"
	"syscall"
	"time"

	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"

	"github.com/thecodearchive/gitarchive/git"
	"github.com/thecodearchive/gitarchive/index"
	"github.com/thecodearchive/gitarchive/queue"
	"github.com/thecodearchive/gitarchive/weekmap"
)

type Fetcher struct {
	q        *queue.Queue
	i        *index.Index
	bucket   *storage.BucketHandle
	schedule *weekmap.WeekMap

	exp *expvar.Map

	closing uint32
}

func (f *Fetcher) Run() error {
	f.exp.Set("fetchbytes", &expvar.Int{})
	for atomic.LoadUint32(&f.closing) == 0 {
		if !f.schedule.Get(time.Now()) {
			f.exp.Add("sleep", 1)
			interruptableSleep(5 * time.Minute)
			continue
		}

		name, parent, err := f.q.Pop()
		if err != nil {
			return err
		}

		if name == "" {
			f.exp.Add("emptyqueue", 1)
			interruptableSleep(30 * time.Second)
			continue
		}

		if err := f.Fetch(name, parent); err != nil {
			return err
		}
	}
	return nil
}

func (f *Fetcher) Fetch(name, parent string) error {
	f.exp.Add("fetches", 1)

	name = "github.com/" + name
	haves, deps, err := f.i.GetHaves(name)
	if err != nil {
		return err
	}

	if haves == nil {
		f.exp.Add("new", 1)
	}
	if parent != "" {
		f.exp.Add("forks", 1)
	}

	logVerb, logFork := "Cloning", ""
	if haves != nil {
		logVerb = "Fetching"
	}
	if parent != "" {
		logFork = fmt.Sprintf(" (fork of %s)", parent)
	}
	log.Printf("[+] %s %s%s...", logVerb, name, logFork)

	start := time.Now()
	bw := f.exp.Get("fetchbytes").(*expvar.Int)
	refs, r, err := git.Fetch("git://"+name+".git", haves, os.Stderr, bw)
	if err == git.RepoNotFoundError {
		log.Println("[-] Repository vanished :(")
		f.exp.Add("vanished", 1)
		return nil
	}
	if err != nil {
		return err
	}

	packRefName := fmt.Sprintf("%s/%d", name, time.Now().UnixNano())
	if r != nil {
		w := f.bucket.Object(packRefName).NewWriter(context.Background())

		bytesFetched, err := io.Copy(w, r)
		if err != nil {
			return err
		}
		w.Close()
		r.Close()
		f.exp.Add("fetchtime", int64(time.Since(start)))
		log.Printf("[+] Got %d refs, %d bytes in %s.", len(refs), bytesFetched, time.Since(start))
	} else {
		// Empty packfile.
		packRefName = "EMPTY|" + packRefName
		f.exp.Add("emptypack", 1)
		log.Printf("[+] Got %d refs, and a empty packfile.", len(refs))
	}

	if parent != "" {
		parent = "github.com/" + parent
	}

	return f.i.AddFetch(name, parent, time.Now(), refs, packRefName, deps)
}

func (f *Fetcher) Stop() {
	atomic.StoreUint32(&f.closing, 1)
}

func interruptableSleep(d time.Duration) bool {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
	defer signal.Stop(c)
	t := time.NewTimer(d)
	select {
	case <-c:
		return false
	case <-t.C:
		return true
	}
}
<commit_msg>fetcher: bail on large repos<commit_after>package main

import (
	"errors"
	"expvar"
	"fmt"
	"io"
	"log"
	"os"
	"os/signal"
	"sync/atomic"
	"syscall"
	"time"

	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"

	"github.com/thecodearchive/gitarchive/git"
	"github.com/thecodearchive/gitarchive/index"
	"github.com/thecodearchive/gitarchive/queue"
	"github.com/thecodearchive/gitarchive/weekmap"
)

const maxSize = 100 << 20

type Fetcher struct {
	q        *queue.Queue
	i        *index.Index
	bucket   *storage.BucketHandle
	schedule *weekmap.WeekMap

	exp *expvar.Map

	closing uint32
}

func (f *Fetcher) Run() error {
	f.exp.Set("fetchbytes", &expvar.Int{})
	for atomic.LoadUint32(&f.closing) == 0 {
		if !f.schedule.Get(time.Now()) {
			f.exp.Add("sleep", 1)
			interruptableSleep(5 * time.Minute)
			continue
		}

		name, parent, err := f.q.Pop()
		if err != nil {
			return err
		}

		if name == "" {
			f.exp.Add("emptyqueue", 1)
			interruptableSleep(30 * time.Second)
			continue
		}

		if err := f.Fetch(name, parent); err != nil {
			return err
		}
	}
	return nil
}

func (f *Fetcher) Fetch(name, parent string) error {
	f.exp.Add("fetches", 1)

	name = "github.com/" + name
	haves, deps, err := f.i.GetHaves(name)
	if err != nil {
		return err
	}

	if haves == nil {
		f.exp.Add("new", 1)
	}
	if parent != "" {
		f.exp.Add("forks", 1)
	}

	logVerb, logFork := "Cloning", ""
	if haves != nil {
		logVerb = "Fetching"
	}
	if parent != "" {
		logFork = fmt.Sprintf(" (fork of %s)", parent)
	}
	log.Printf("[+] %s %s%s...", logVerb, name, logFork)

	start := time.Now()
	bw := f.exp.Get("fetchbytes").(*expvar.Int)
	refs, r, err := git.Fetch("git://"+name+".git", haves, os.Stderr, bw)
	if err == git.RepoNotFoundError {
		log.Println("[-] Repository vanished :(")
		f.exp.Add("vanished", 1)
		return nil
	}
	if err != nil {
		return err
	}

	packRefName := fmt.Sprintf("%s/%d", name, time.Now().UnixNano())
	if r != nil {
		w := f.bucket.Object(packRefName).NewWriter(context.Background())

		lr := &io.LimitedReader{R: r, N: maxSize}
		bytesFetched, err := io.Copy(w, lr)
		if err != nil {
			return err
		}
		r.Close()
		if lr.N <= 0 {
			w.CloseWithError(errors.New("too big"))
			log.Printf("[-] Repository too big :(")
			f.exp.Add("toobig", 1)
			return nil
		}
		w.Close()
		f.exp.Add("fetchtime", int64(time.Since(start)))
		log.Printf("[+] Got %d refs, %d bytes in %s.", len(refs), bytesFetched, time.Since(start))
	} else {
		// Empty packfile.
		packRefName = "EMPTY|" + packRefName
		f.exp.Add("emptypack", 1)
		log.Printf("[+] Got %d refs, and a empty packfile.", len(refs))
	}

	if parent != "" {
		parent = "github.com/" + parent
	}

	return f.i.AddFetch(name, parent, time.Now(), refs, packRefName, deps)
}

func (f *Fetcher) Stop() {
	atomic.StoreUint32(&f.closing, 1)
}

func interruptableSleep(d time.Duration) bool {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
	defer signal.Stop(c)
	t := time.NewTimer(d)
	select {
	case <-c:
		return false
	case <-t.C:
		return true
	}
}
<|endoftext|>


<commit_before>/*
Copyright The Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"io"
	"os"
	"path/filepath"

	"github.com/spf13/cobra"

	"helm.sh/helm/v3/cmd/helm/require"
)

const completionDesc = `
Generate autocompletion scripts for Helm for the specified shell.
`
const bashCompDesc = `
Generate the autocompletion script for Helm for the bash shell.

To load completions in your current shell session:

    source <(helm completion bash)

To load completions for every new session, execute once:
- Linux:

      helm completion bash > /etc/bash_completion.d/helm

- MacOS:

      helm completion bash > /usr/local/etc/bash_completion.d/helm
`

const zshCompDesc = `
Generate the autocompletion script for Helm for the zsh shell.

To load completions in your current shell session:

    source <(helm completion zsh)

To load completions for every new session, execute once:

    helm completion zsh > "${fpath[1]}/_helm"
`

const fishCompDesc = `
Generate the autocompletion script for Helm for the fish shell.

To load completions in your current shell session:

    helm completion fish | source

To load completions for every new session, execute once:

    helm completion fish > ~/.config/fish/completions/helm.fish

You will need to start a new shell for this setup to take effect.
`

const (
	noDescFlagName = "no-descriptions"
	noDescFlagText = "disable completion descriptions"
)

var disableCompDescriptions bool

func newCompletionCmd(out io.Writer) *cobra.Command {
	cmd := &cobra.Command{
		Use:   "completion",
		Short: "generate autocompletion scripts for the specified shell",
		Long:  completionDesc,
		Args:  require.NoArgs,
	}

	bash := &cobra.Command{
		Use:                   "bash",
		Short:                 "generate autocompletion script for bash",
		Long:                  bashCompDesc,
		Args:                  require.NoArgs,
		DisableFlagsInUseLine: true,
		ValidArgsFunction:     noCompletions,
		RunE: func(cmd *cobra.Command, args []string) error {
			return runCompletionBash(out, cmd)
		},
	}

	zsh := &cobra.Command{
		Use:               "zsh",
		Short:             "generate autocompletion script for zsh",
		Long:              zshCompDesc,
		Args:              require.NoArgs,
		ValidArgsFunction: noCompletions,
		RunE: func(cmd *cobra.Command, args []string) error {
			return runCompletionZsh(out, cmd)
		},
	}
	zsh.Flags().BoolVar(&disableCompDescriptions, noDescFlagName, false, noDescFlagText)

	fish := &cobra.Command{
		Use:               "fish",
		Short:             "generate autocompletion script for fish",
		Long:              fishCompDesc,
		Args:              require.NoArgs,
		ValidArgsFunction: noCompletions,
		RunE: func(cmd *cobra.Command, args []string) error {
			return runCompletionFish(out, cmd)
		},
	}
	fish.Flags().BoolVar(&disableCompDescriptions, noDescFlagName, false, noDescFlagText)

	cmd.AddCommand(bash, zsh, fish)

	return cmd
}

func runCompletionBash(out io.Writer, cmd *cobra.Command) error {
	err := cmd.Root().GenBashCompletion(out)

	// In case the user renamed the helm binary (e.g., to be able to run
	// both helm2 and helm3), we hook the new binary name to the completion function
	if binary := filepath.Base(os.Args[0]); binary != "helm" {
		renamedBinaryHook := `
# Hook the command used to generate the completion script
# to the helm completion function to handle the case where
# the user renamed the helm binary
if [[ $(type -t compopt) = "builtin" ]]; then
    complete -o default -F __start_helm %[1]s
else
    complete -o default -o nospace -F __start_helm %[1]s
fi
`
		fmt.Fprintf(out, renamedBinaryHook, binary)
	}

	return err
}

func runCompletionZsh(out io.Writer, cmd *cobra.Command) error {
	var err error
	if disableCompDescriptions {
		err = cmd.Root().GenZshCompletionNoDesc(out)
	} else {
		err = cmd.Root().GenZshCompletion(out)
	}

	// In case the user renamed the helm binary (e.g., to be able to run
	// both helm2 and helm3), we hook the new binary name to the completion function
	if binary := filepath.Base(os.Args[0]); binary != "helm" {
		renamedBinaryHook := `
# Hook the command used to generate the completion script
# to the helm completion function to handle the case where
# the user renamed the helm binary
compdef _helm %[1]s
`
		fmt.Fprintf(out, renamedBinaryHook, binary)
	}

	// Cobra doesn't source zsh completion file, explicitly doing it here
	fmt.Fprintf(out, "compdef _helm helm")

	return err
}

func runCompletionFish(out io.Writer, cmd *cobra.Command) error {
	return cmd.Root().GenFishCompletion(out, !disableCompDescriptions)
}

// Function to disable file completion
func noCompletions(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
	return nil, cobra.ShellCompDirectiveNoFileComp
}
<commit_msg>feat(comp): Add support for powershell completion<commit_after>/*
Copyright The Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"io"
	"os"
	"path/filepath"

	"github.com/spf13/cobra"

	"helm.sh/helm/v3/cmd/helm/require"
)

const completionDesc = `
Generate autocompletion scripts for Helm for the specified shell.
`
const bashCompDesc = `
Generate the autocompletion script for Helm for the bash shell.

To load completions in your current shell session:

    source <(helm completion bash)

To load completions for every new session, execute once:
- Linux:

      helm completion bash > /etc/bash_completion.d/helm

- MacOS:

      helm completion bash > /usr/local/etc/bash_completion.d/helm
`

const zshCompDesc = `
Generate the autocompletion script for Helm for the zsh shell.

To load completions in your current shell session:

    source <(helm completion zsh)

To load completions for every new session, execute once:

    helm completion zsh > "${fpath[1]}/_helm"
`

const fishCompDesc = `
Generate the autocompletion script for Helm for the fish shell.

To load completions in your current shell session:

    helm completion fish | source

To load completions for every new session, execute once:

    helm completion fish > ~/.config/fish/completions/helm.fish

You will need to start a new shell for this setup to take effect.
`

const powershellCompDesc = `
Generate the autocompletion script for powershell.

To load completions in your current shell session:
PS C:\> helm completion powershell | Out-String | Invoke-Expression

To load completions for every new session, add the output of the above command
to your powershell profile.
`

const (
	noDescFlagName = "no-descriptions"
	noDescFlagText = "disable completion descriptions"
)

var disableCompDescriptions bool

func newCompletionCmd(out io.Writer) *cobra.Command {
	cmd := &cobra.Command{
		Use:   "completion",
		Short: "generate autocompletion scripts for the specified shell",
		Long:  completionDesc,
		Args:  require.NoArgs,
	}

	bash := &cobra.Command{
		Use:                   "bash",
		Short:                 "generate autocompletion script for bash",
		Long:                  bashCompDesc,
		Args:                  require.NoArgs,
		DisableFlagsInUseLine: true,
		ValidArgsFunction:     noCompletions,
		RunE: func(cmd *cobra.Command, args []string) error {
			return runCompletionBash(out, cmd)
		},
	}

	zsh := &cobra.Command{
		Use:               "zsh",
		Short:             "generate autocompletion script for zsh",
		Long:              zshCompDesc,
		Args:              require.NoArgs,
		ValidArgsFunction: noCompletions,
		RunE: func(cmd *cobra.Command, args []string) error {
			return runCompletionZsh(out, cmd)
		},
	}
	zsh.Flags().BoolVar(&disableCompDescriptions, noDescFlagName, false, noDescFlagText)

	fish := &cobra.Command{
		Use:               "fish",
		Short:             "generate autocompletion script for fish",
		Long:              fishCompDesc,
		Args:              require.NoArgs,
		ValidArgsFunction: noCompletions,
		RunE: func(cmd *cobra.Command, args []string) error {
			return runCompletionFish(out, cmd)
		},
	}
	fish.Flags().BoolVar(&disableCompDescriptions, noDescFlagName, false, noDescFlagText)

	powershell := &cobra.Command{
		Use:               "powershell",
		Short:             "generate autocompletion script for powershell",
		Long:              powershellCompDesc,
		Args:              require.NoArgs,
		ValidArgsFunction: noCompletions,
		RunE: func(cmd *cobra.Command, args []string) error {
			return runCompletionPowershell(out, cmd)
		},
	}
	powershell.Flags().BoolVar(&disableCompDescriptions, noDescFlagName, false, noDescFlagText)

	cmd.AddCommand(bash, zsh, fish, powershell)

	return cmd
}

func runCompletionBash(out io.Writer, cmd *cobra.Command) error {
	err := cmd.Root().GenBashCompletion(out)

	// In case the user renamed the helm binary (e.g., to be able to run
	// both helm2 and helm3), we hook the new binary name to the completion function
	if binary := filepath.Base(os.Args[0]); binary != "helm" {
		renamedBinaryHook := `
# Hook the command used to generate the completion script
# to the helm completion function to handle the case where
# the user renamed the helm binary
if [[ $(type -t compopt) = "builtin" ]]; then
    complete -o default -F __start_helm %[1]s
else
    complete -o default -o nospace -F __start_helm %[1]s
fi
`
		fmt.Fprintf(out, renamedBinaryHook, binary)
	}

	return err
}

func runCompletionZsh(out io.Writer, cmd *cobra.Command) error {
	var err error
	if disableCompDescriptions {
		err = cmd.Root().GenZshCompletionNoDesc(out)
	} else {
		err = cmd.Root().GenZshCompletion(out)
	}

	// In case the user renamed the helm binary (e.g., to be able to run
	// both helm2 and helm3), we hook the new binary name to the completion function
	if binary := filepath.Base(os.Args[0]); binary != "helm" {
		renamedBinaryHook := `
# Hook the command used to generate the completion script
# to the helm completion function to handle the case where
# the user renamed the helm binary
compdef _helm %[1]s
`
		fmt.Fprintf(out, renamedBinaryHook, binary)
	}

	// Cobra doesn't source zsh completion file, explicitly doing it here
	fmt.Fprintf(out, "compdef _helm helm")

	return err
}

func runCompletionFish(out io.Writer, cmd *cobra.Command) error {
	return cmd.Root().GenFishCompletion(out, !disableCompDescriptions)
}

func runCompletionPowershell(out io.Writer, cmd *cobra.Command) error {
	if disableCompDescriptions {
		return cmd.Root().GenPowerShellCompletion(out)
	}
	return cmd.Root().GenPowerShellCompletionWithDesc(out)
}

// Function to disable file completion
func noCompletions(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
	return nil, cobra.ShellCompDirectiveNoFileComp
}
<|endoftext|>


<commit_before>package internal

import (
	"errors"
	"fmt"
	"strings"

	"github.com/ready-steady/numeric/basis/linhat"
	"github.com/ready-steady/numeric/grid/newcot"
	"github.com/ready-steady/numeric/interpolation/adhier"
)

type Solver struct {
	adhier.Interpolator
}

type Solution struct {
	adhier.Surrogate
	Expectation []float64
}

func NewSolver(problem *Problem, target Target) (*Solver, error) {
	ni, _ := target.Dimensions()

	var grid adhier.Grid
	var basis adhier.Basis

	switch strings.ToLower(problem.Config.Interpolation.Rule) {
	case "open":
		grid, basis = newcot.NewOpen(ni), linhat.NewOpen(ni)
	case "closed":
		grid, basis = newcot.NewClosed(ni), linhat.NewClosed(ni)
	default:
		return nil, errors.New("the interpolation rule is unknown")
	}

	interpolator := adhier.New(grid, basis,
		(*adhier.Config)(&problem.Config.Interpolation.Config))

	return &Solver{*interpolator}, nil
}

func (s *Solver) Compute(target Target) *Solution {
	surrogate := s.Interpolator.Compute(target)
	target.Monitor(surrogate.Level, 0, surrogate.Nodes)
	return &Solution{
		Surrogate:   *surrogate,
		Expectation: s.Interpolator.Integrate(surrogate),
	}
}

func (s *Solver) Evaluate(solution *Solution, nodes []float64) []float64 {
	return s.Interpolator.Evaluate(&solution.Surrogate, nodes)
}

func (s *Solution) String() string {
	return fmt.Sprintf("Solution{inputs: %d, outputs: %d, level: %d, nodes: %d}",
		s.Inputs, s.Outputs, s.Level, s.Nodes)
}
<commit_msg>Fix a typo in solver<commit_after>package internal

import (
	"errors"
	"fmt"
	"strings"

	"github.com/ready-steady/numeric/basis/linhat"
	"github.com/ready-steady/numeric/grid/newcot"
	"github.com/ready-steady/numeric/interpolation/adhier"
)

type Solver struct {
	adhier.Interpolator
}

type Solution struct {
	adhier.Surrogate
	Expectation []float64
}

func NewSolver(problem *Problem, target Target) (*Solver, error) {
	ni, _ := target.Dimensions()

	var grid adhier.Grid
	var basis adhier.Basis

	switch strings.ToLower(problem.Config.Interpolation.Rule) {
	case "open":
		grid, basis = newcot.NewOpen(ni), linhat.NewOpen(ni)
	case "closed":
		grid, basis = newcot.NewClosed(ni), linhat.NewClosed(ni)
	default:
		return nil, errors.New("the interpolation rule is unknown")
	}

	interpolator := adhier.New(grid, basis,
		(*adhier.Config)(&problem.Config.Interpolation.Config))

	return &Solver{*interpolator}, nil
}

func (s *Solver) Compute(target Target) *Solution {
	surrogate := s.Interpolator.Compute(target)
	target.Monitor(surrogate.Level, surrogate.Nodes, 0)
	return &Solution{
		Surrogate:   *surrogate,
		Expectation: s.Interpolator.Integrate(surrogate),
	}
}

func (s *Solver) Evaluate(solution *Solution, nodes []float64) []float64 {
	return s.Interpolator.Evaluate(&solution.Surrogate, nodes)
}

func (s *Solution) String() string {
	return fmt.Sprintf("Solution{inputs: %d, outputs: %d, level: %d, nodes: %d}",
		s.Inputs, s.Outputs, s.Level, s.Nodes)
}
<|endoftext|>


<commit_before>// Copyright 2017 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"errors"
	"fmt"
	"io"
	"os"

	"istio.io/manager/cmd"
	"istio.io/manager/cmd/version"
	"istio.io/manager/platform/kube/inject"

	"github.com/spf13/cobra"
)

var (
	hub              string
	tag              string
	managerAddr      string
	mixerAddr        string
	sidecarProxyUID  int64
	sidecarProxyPort int
	runtimeVerbosity int
	versionStr       string // override build version

	inFilename  string
	outFilename string
)

var (
	injectCmd = &cobra.Command{
		Use:   "kube-inject",
		Short: "Inject istio runtime into kubernete resources",
		RunE: func(_ *cobra.Command, _ []string) (err error) {
			if inFilename == "" {
				return errors.New("filename not specified (see --filename or -f)")
			}
			var reader io.Reader
			if inFilename == "-" {
				reader = os.Stdin
			} else {
				reader, err = os.Open(inFilename)
				if err != nil {
					return err
				}
			}

			var writer io.Writer
			if outFilename == "" {
				writer = os.Stdout
			} else {
				file, err := os.Create(outFilename)
				if err != nil {
					return err
				}
				writer = file
				defer func() { err = file.Close() }()
			}

			if versionStr == "" {
				versionStr = fmt.Sprintf("%v@%v-%v-%v",
					version.Info.User,
					version.Info.Host,
					version.Info.Version,
					version.Info.GitRevision)
			}
			params := &inject.Params{
				InitImage:        inject.InitImageName(hub, tag),
				RuntimeImage:     inject.RuntimeImageName(hub, tag),
				RuntimeVerbosity: runtimeVerbosity,
				ManagerAddr:      managerAddr,
				MixerAddr:        mixerAddr,
				SidecarProxyUID:  sidecarProxyUID,
				SidecarProxyPort: sidecarProxyPort,
				Version:          versionStr,
			}
			return inject.IntoResourceFile(params, reader, writer)
		},
	}
)

func init() {
	injectCmd.PersistentFlags().StringVar(&hub, "hub",
		inject.DefaultHub, "Docker hub")
	injectCmd.PersistentFlags().StringVar(&tag, "tag",
		inject.DefaultTag, "Docker tag")
	injectCmd.PersistentFlags().StringVarP(&inFilename, "filename", "f",
		"", "Input kubernetes resource filename")
	injectCmd.PersistentFlags().StringVarP(&outFilename, "output", "o",
		"", "Modified output kubernetes resource filename")
	injectCmd.PersistentFlags().StringVar(&managerAddr, "managerAddr",
		inject.DefaultManagerAddr, "Manager service DNS address")
	injectCmd.PersistentFlags().StringVar(&mixerAddr, "mixerAddr",
		inject.DefaultMixerAddr, "Mixer DNS address")
	injectCmd.PersistentFlags().IntVar(&runtimeVerbosity, "verbosity",
		inject.DefaultRuntimeVerbosity, "Runtime verbosity")
	injectCmd.PersistentFlags().Int64Var(&sidecarProxyUID, "sidecarProxyUID",
		inject.DefaultSidecarProxyUID, "Sidecar proxy UID")
	injectCmd.PersistentFlags().IntVar(&sidecarProxyPort, "sidecarProxyPort",
		inject.DefaultSidecarProxyPort, "Sidecar proxy Port")
	injectCmd.PersistentFlags().StringVar(&versionStr, "setVersionString",
		"", "Override version info injected into resource")
	cmd.RootCmd.AddCommand(injectCmd)
}
<commit_msg>add long help text and example usage to kube-inject command (#393)<commit_after>// Copyright 2017 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"errors"
	"fmt"
	"io"
	"os"

	"istio.io/manager/cmd"
	"istio.io/manager/cmd/version"
	"istio.io/manager/platform/kube/inject"

	"github.com/spf13/cobra"
)

var (
	hub              string
	tag              string
	managerAddr      string
	mixerAddr        string
	sidecarProxyUID  int64
	sidecarProxyPort int
	runtimeVerbosity int
	versionStr       string // override build version

	inFilename  string
	outFilename string
)

var (
	injectCmd = &cobra.Command{
		Use:   "kube-inject",
		Short: "Inject istio sidecar proxy into kubernetes resources",
		Long: `
Use kube-inject to manually inject istio sidecar proxy into kubernetes
resource files. Unsupported resources are left unmodified so it is
safe to run kube-inject over a single file that contains multiple
Service, ConfigMap, Deployment, etc. definitions for a complex
application. Its best to do this when the resource is initially
created.

Example usage:

	kubectl apply -f <(istioctl kube-inject -f <resource.yaml>)
`,
		RunE: func(_ *cobra.Command, _ []string) (err error) {
			if inFilename == "" {
				return errors.New("filename not specified (see --filename or -f)")
			}
			var reader io.Reader
			if inFilename == "-" {
				reader = os.Stdin
			} else {
				reader, err = os.Open(inFilename)
				if err != nil {
					return err
				}
			}

			var writer io.Writer
			if outFilename == "" {
				writer = os.Stdout
			} else {
				file, err := os.Create(outFilename)
				if err != nil {
					return err
				}
				writer = file
				defer func() { err = file.Close() }()
			}

			if versionStr == "" {
				versionStr = fmt.Sprintf("%v@%v-%v-%v",
					version.Info.User,
					version.Info.Host,
					version.Info.Version,
					version.Info.GitRevision)
			}
			params := &inject.Params{
				InitImage:        inject.InitImageName(hub, tag),
				RuntimeImage:     inject.RuntimeImageName(hub, tag),
				RuntimeVerbosity: runtimeVerbosity,
				ManagerAddr:      managerAddr,
				MixerAddr:        mixerAddr,
				SidecarProxyUID:  sidecarProxyUID,
				SidecarProxyPort: sidecarProxyPort,
				Version:          versionStr,
			}
			return inject.IntoResourceFile(params, reader, writer)
		},
	}
)

func init() {
	injectCmd.PersistentFlags().StringVar(&hub, "hub",
		inject.DefaultHub, "Docker hub")
	injectCmd.PersistentFlags().StringVar(&tag, "tag",
		inject.DefaultTag, "Docker tag")
	injectCmd.PersistentFlags().StringVarP(&inFilename, "filename", "f",
		"", "Input kubernetes resource filename")
	injectCmd.PersistentFlags().StringVarP(&outFilename, "output", "o",
		"", "Modified output kubernetes resource filename")
	injectCmd.PersistentFlags().StringVar(&managerAddr, "managerAddr",
		inject.DefaultManagerAddr, "Manager service DNS address")
	injectCmd.PersistentFlags().StringVar(&mixerAddr, "mixerAddr",
		inject.DefaultMixerAddr, "Mixer DNS address")
	injectCmd.PersistentFlags().IntVar(&runtimeVerbosity, "verbosity",
		inject.DefaultRuntimeVerbosity, "Runtime verbosity")
	injectCmd.PersistentFlags().Int64Var(&sidecarProxyUID, "sidecarProxyUID",
		inject.DefaultSidecarProxyUID, "Sidecar proxy UID")
	injectCmd.PersistentFlags().IntVar(&sidecarProxyPort, "sidecarProxyPort",
		inject.DefaultSidecarProxyPort, "Sidecar proxy Port")
	injectCmd.PersistentFlags().StringVar(&versionStr, "setVersionString",
		"", "Override version info injected into resource")
	cmd.RootCmd.AddCommand(injectCmd)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"log"
	"net/http"
	"runtime"
	"sync/atomic"
	"time"
)

var rc *rateCalculator

func statusService(addr string) {
	rc = newRateCalculator(360, 10*time.Second, &bytesProxied)

	http.HandleFunc("/status", getStatus)
	if err := http.ListenAndServe(addr, nil); err != nil {
		log.Fatal(err)
	}
}

func getStatus(w http.ResponseWriter, r *http.Request) {
	status := make(map[string]interface{})

	sessionMut.Lock()
	// This can potentially be double the number of pending sessions, as each session has two keys, one for each side.
	status["startTime"] = rc.startTime
	status["uptime"] = time.Since(rc.startTime)
	status["numPendingSessionKeys"] = len(pendingSessions)
	status["numActiveSessions"] = len(activeSessions)
	sessionMut.Unlock()
	status["numConnections"] = atomic.LoadInt64(&numConnections)
	status["numProxies"] = atomic.LoadInt64(&numProxies)
	status["bytesProxied"] = atomic.LoadInt64(&bytesProxied)
	status["goVersion"] = runtime.Version()
	status["goOS"] = runtime.GOOS
	status["goAarch"] = runtime.GOARCH
	status["goMaxProcs"] = runtime.GOMAXPROCS(-1)
	status["kbps10s1m5m15m30m60m"] = []int64{
		rc.rate(10/10) * 8 / 1000,
		rc.rate(60/10) * 8 / 1000,
		rc.rate(5*60/10) * 8 / 1000,
		rc.rate(15*60/10) * 8 / 1000,
		rc.rate(30*60/10) * 8 / 1000,
		rc.rate(60*60/10) * 8 / 1000,
	}
	status["options"] = map[string]interface{}{
		"network-timeout":  networkTimeout,
		"ping-interval":    pingInterval,
		"message-timeout":  messageTimeout,
		"per-session-rate": sessionLimitBps,
		"global-rate":      globalLimitBps,
		"pools":            defaultPoolAddrs,
	}

	bs, err := json.MarshalIndent(status, "", "    ")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-Type", "application/json")
	w.Write(bs)
}

type rateCalculator struct {
	rates     []int64
	prev      int64
	counter   *int64
	startTime time.Time
}

func newRateCalculator(keepIntervals int, interval time.Duration, counter *int64) *rateCalculator {
	r := &rateCalculator{
		rates:     make([]int64, keepIntervals),
		counter:   counter,
		startTime: time.Now(),
	}

	go r.updateRates(interval)

	return r
}

func (r *rateCalculator) updateRates(interval time.Duration) {
	for {
		now := time.Now()
		next := now.Truncate(interval).Add(interval)
		time.Sleep(next.Sub(now))

		cur := atomic.LoadInt64(r.counter)
		rate := int64(float64(cur-r.prev) / interval.Seconds())
		copy(r.rates[1:], r.rates)
		r.rates[0] = rate
		r.prev = cur
	}
}

func (r *rateCalculator) rate(periods int) int64 {
	var tot int64
	for i := 0; i < periods; i++ {
		tot += r.rates[i]
	}
	return tot / int64(periods)
}
<commit_msg>Expose provided by in status endpoint<commit_after>package main

import (
	"encoding/json"
	"log"
	"net/http"
	"runtime"
	"sync/atomic"
	"time"
)

var rc *rateCalculator

func statusService(addr string) {
	rc = newRateCalculator(360, 10*time.Second, &bytesProxied)

	http.HandleFunc("/status", getStatus)
	if err := http.ListenAndServe(addr, nil); err != nil {
		log.Fatal(err)
	}
}

func getStatus(w http.ResponseWriter, r *http.Request) {
	status := make(map[string]interface{})

	sessionMut.Lock()
	// This can potentially be double the number of pending sessions, as each session has two keys, one for each side.
	status["startTime"] = rc.startTime
	status["uptime"] = time.Since(rc.startTime)
	status["numPendingSessionKeys"] = len(pendingSessions)
	status["numActiveSessions"] = len(activeSessions)
	sessionMut.Unlock()
	status["numConnections"] = atomic.LoadInt64(&numConnections)
	status["numProxies"] = atomic.LoadInt64(&numProxies)
	status["bytesProxied"] = atomic.LoadInt64(&bytesProxied)
	status["goVersion"] = runtime.Version()
	status["goOS"] = runtime.GOOS
	status["goAarch"] = runtime.GOARCH
	status["goMaxProcs"] = runtime.GOMAXPROCS(-1)
	status["kbps10s1m5m15m30m60m"] = []int64{
		rc.rate(10/10) * 8 / 1000,
		rc.rate(60/10) * 8 / 1000,
		rc.rate(5*60/10) * 8 / 1000,
		rc.rate(15*60/10) * 8 / 1000,
		rc.rate(30*60/10) * 8 / 1000,
		rc.rate(60*60/10) * 8 / 1000,
	}
	status["options"] = map[string]interface{}{
		"network-timeout":  networkTimeout,
		"ping-interval":    pingInterval,
		"message-timeout":  messageTimeout,
		"per-session-rate": sessionLimitBps,
		"global-rate":      globalLimitBps,
		"pools":            defaultPoolAddrs,
		"provided-by":      providedBy,
	}

	bs, err := json.MarshalIndent(status, "", "    ")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-Type", "application/json")
	w.Write(bs)
}

type rateCalculator struct {
	rates     []int64
	prev      int64
	counter   *int64
	startTime time.Time
}

func newRateCalculator(keepIntervals int, interval time.Duration, counter *int64) *rateCalculator {
	r := &rateCalculator{
		rates:     make([]int64, keepIntervals),
		counter:   counter,
		startTime: time.Now(),
	}

	go r.updateRates(interval)

	return r
}

func (r *rateCalculator) updateRates(interval time.Duration) {
	for {
		now := time.Now()
		next := now.Truncate(interval).Add(interval)
		time.Sleep(next.Sub(now))

		cur := atomic.LoadInt64(r.counter)
		rate := int64(float64(cur-r.prev) / interval.Seconds())
		copy(r.rates[1:], r.rates)
		r.rates[0] = rate
		r.prev = cur
	}
}

func (r *rateCalculator) rate(periods int) int64 {
	var tot int64
	for i := 0; i < periods; i++ {
		tot += r.rates[i]
	}
	return tot / int64(periods)
}
<|endoftext|>


<commit_before>// Copyright (c) 2015-2022 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package cmd

import (
	"fmt"
	"sort"
	"strings"
	"sync/atomic"
	"time"

	"github.com/charmbracelet/bubbles/spinner"
	tea "github.com/charmbracelet/bubbletea"
	"github.com/charmbracelet/lipgloss"
	"github.com/dustin/go-humanize"
	"github.com/minio/madmin-go"
	"github.com/olekukonko/tablewriter"
)

// TODO: Add ART (Average Response Time) latency
type topAPIStats struct {
	TotalCalls   uint64
	TotalBytesRX uint64
	TotalBytesTX uint64
}

func (s *topAPIStats) addAPICall(n int) {
	atomic.AddUint64(&s.TotalCalls, uint64(n))
}

func (s *topAPIStats) addAPIBytesRX(n int) {
	atomic.AddUint64(&s.TotalBytesRX, uint64(n))
}

func (s *topAPIStats) addAPIBytesTX(n int) {
	atomic.AddUint64(&s.TotalBytesTX, uint64(n))
}

func (s *topAPIStats) loadAPICall() uint64 {
	return atomic.LoadUint64(&s.TotalCalls)
}

func (s *topAPIStats) loadAPIBytesRX() uint64 {
	return atomic.LoadUint64(&s.TotalBytesRX)
}

func (s *topAPIStats) loadAPIBytesTX() uint64 {
	return atomic.LoadUint64(&s.TotalBytesTX)
}

type traceUI struct {
	spinner     spinner.Model
	quitting    bool
	startTime   time.Time
	result      topAPIResult
	lastResult  topAPIResult
	apiStatsMap map[string]*topAPIStats
}

type topAPIResult struct {
	final       bool
	apiCallInfo madmin.ServiceTraceInfo
}

func initTraceUI() *traceUI {
	s := spinner.New()
	s.Spinner = spinner.Points
	s.Style = lipgloss.NewStyle().Foreground(lipgloss.Color("205"))
	return &traceUI{
		spinner:     s,
		apiStatsMap: make(map[string]*topAPIStats),
	}
}

func (m *traceUI) Init() tea.Cmd {
	return m.spinner.Tick
}

func (m *traceUI) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
	switch msg := msg.(type) {
	case tea.KeyMsg:
		switch msg.String() {
		case "ctrl+c":
			m.quitting = true
			return m, tea.Quit
		default:
			return m, nil
		}
	case topAPIResult:
		m.result = msg
		if m.result.apiCallInfo.Trace.FuncName != "" {
			m.lastResult = m.result
		}
		if msg.final {
			m.quitting = true
			return m, tea.Quit
		}
		return m, nil

	case spinner.TickMsg:
		var cmd tea.Cmd
		m.spinner, cmd = m.spinner.Update(msg)
		return m, cmd
	default:
		return m, nil
	}
}

func (m *traceUI) View() string {
	var s strings.Builder
	s.WriteString("\n")

	// Set table header
	table := tablewriter.NewWriter(&s)
	table.SetAutoWrapText(false)
	table.SetAutoFormatHeaders(true)
	table.SetHeaderAlignment(tablewriter.ALIGN_LEFT)
	table.SetAlignment(tablewriter.ALIGN_LEFT)
	table.SetCenterSeparator("")
	table.SetColumnSeparator("")
	table.SetRowSeparator("")
	table.SetHeaderLine(false)
	table.SetBorder(false)
	table.SetTablePadding("\t") // pad with tabs
	table.SetNoWhiteSpace(true)

	res := m.result.apiCallInfo
	if m.startTime.IsZero() && !res.Trace.Time.IsZero() {
		m.startTime = res.Trace.Time
	}
	if res.Trace.FuncName != "" && res.Trace.FuncName != "errorResponseHandler" {
		traceSt, ok := m.apiStatsMap[res.Trace.FuncName]
		if !ok {
			traceSt = &topAPIStats{}
		}
		traceSt.addAPICall(1)
		if res.Trace.HTTP != nil {
			traceSt.addAPIBytesRX(res.Trace.HTTP.CallStats.InputBytes)
			traceSt.addAPIBytesTX(res.Trace.HTTP.CallStats.OutputBytes)
		}
		m.apiStatsMap[res.Trace.FuncName] = traceSt
	}

	table.SetHeader([]string{"API", "CALLS", "RX", "TX"})
	data := make([][]string, 0, len(m.apiStatsMap))

	for k, stats := range m.apiStatsMap {
		data = append(data, []string{
			k,
			whiteStyle.Render(fmt.Sprintf("%d", stats.loadAPICall())),
			whiteStyle.Render(humanize.IBytes(stats.loadAPIBytesRX())),
			whiteStyle.Render(humanize.IBytes(stats.loadAPIBytesTX())),
		})
	}
	sort.Slice(data, func(i, j int) bool {
		return data[i][0] < data[j][0]
	})

	table.AppendBulk(data)
	table.Render()

	if !m.quitting {
		s.WriteString(fmt.Sprintf("\nTopAPI: %s", m.spinner.View()))
	} else {
		var totalTX, totalRX, totalCalls uint64
		lastReqTime := m.lastResult.apiCallInfo.Trace.Time
		if m.lastResult.apiCallInfo.Trace.Time.IsZero() {
			lastReqTime = time.Now()
		}
		for _, stats := range m.apiStatsMap {
			totalRX += stats.loadAPIBytesRX()
			totalTX += stats.loadAPIBytesTX()
			totalCalls += stats.loadAPICall()
		}
		msg := fmt.Sprintf("\n========\nTotal: %d CALLS, %s RX, %s TX - in %.02fs",
			totalCalls,
			humanize.IBytes(totalRX),
			humanize.IBytes(totalTX),
			lastReqTime.Sub(m.startTime).Seconds(),
		)
		s.WriteString(msg)
		s.WriteString("\n")
	}
	return s.String()
}
<commit_msg>add errors to admin top api command (#4176)<commit_after>// Copyright (c) 2015-2022 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package cmd

import (
	"fmt"
	"sort"
	"strings"
	"sync/atomic"
	"time"

	"github.com/charmbracelet/bubbles/spinner"
	tea "github.com/charmbracelet/bubbletea"
	"github.com/charmbracelet/lipgloss"
	"github.com/dustin/go-humanize"
	"github.com/minio/madmin-go"
	"github.com/olekukonko/tablewriter"
)

// TODO: Add ART (Average Response Time) latency
type topAPIStats struct {
	TotalCalls   uint64
	TotalBytesRX uint64
	TotalBytesTX uint64
	TotalErrors  uint64
}

func (s *topAPIStats) addAPICall(n int) {
	atomic.AddUint64(&s.TotalCalls, uint64(n))
}

func (s *topAPIStats) addAPIBytesRX(n int) {
	atomic.AddUint64(&s.TotalBytesRX, uint64(n))
}

func (s *topAPIStats) addAPIBytesTX(n int) {
	atomic.AddUint64(&s.TotalBytesTX, uint64(n))
}

func (s *topAPIStats) addAPIErrors(n int) {
	atomic.AddUint64(&s.TotalErrors, uint64(n))
}

func (s *topAPIStats) loadAPICall() uint64 {
	return atomic.LoadUint64(&s.TotalCalls)
}

func (s *topAPIStats) loadAPIBytesRX() uint64 {
	return atomic.LoadUint64(&s.TotalBytesRX)
}

func (s *topAPIStats) loadAPIBytesTX() uint64 {
	return atomic.LoadUint64(&s.TotalBytesTX)
}

func (s *topAPIStats) loadAPIErrors() uint64 {
	return atomic.LoadUint64(&s.TotalErrors)
}

type traceUI struct {
	spinner     spinner.Model
	quitting    bool
	startTime   time.Time
	result      topAPIResult
	lastResult  topAPIResult
	apiStatsMap map[string]*topAPIStats
}

type topAPIResult struct {
	final       bool
	apiCallInfo madmin.ServiceTraceInfo
}

func initTraceUI() *traceUI {
	s := spinner.New()
	s.Spinner = spinner.Points
	s.Style = lipgloss.NewStyle().Foreground(lipgloss.Color("205"))
	return &traceUI{
		spinner:     s,
		apiStatsMap: make(map[string]*topAPIStats),
	}
}

func (m *traceUI) Init() tea.Cmd {
	return m.spinner.Tick
}

func (m *traceUI) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
	switch msg := msg.(type) {
	case tea.KeyMsg:
		switch msg.String() {
		case "ctrl+c":
			m.quitting = true
			return m, tea.Quit
		default:
			return m, nil
		}
	case topAPIResult:
		m.result = msg
		if m.result.apiCallInfo.Trace.FuncName != "" {
			m.lastResult = m.result
		}
		if msg.final {
			m.quitting = true
			return m, tea.Quit
		}
		return m, nil

	case spinner.TickMsg:
		var cmd tea.Cmd
		m.spinner, cmd = m.spinner.Update(msg)
		return m, cmd
	default:
		return m, nil
	}
}

func (m *traceUI) View() string {
	var s strings.Builder
	s.WriteString("\n")

	// Set table header
	table := tablewriter.NewWriter(&s)
	table.SetAutoWrapText(false)
	table.SetAutoFormatHeaders(true)
	table.SetHeaderAlignment(tablewriter.ALIGN_LEFT)
	table.SetAlignment(tablewriter.ALIGN_LEFT)
	table.SetCenterSeparator("")
	table.SetColumnSeparator("")
	table.SetRowSeparator("")
	table.SetHeaderLine(false)
	table.SetBorder(false)
	table.SetTablePadding("\t") // pad with tabs
	table.SetNoWhiteSpace(true)

	res := m.result.apiCallInfo
	if m.startTime.IsZero() && !res.Trace.Time.IsZero() {
		m.startTime = res.Trace.Time
	}
	if res.Trace.FuncName != "" && res.Trace.FuncName != "errorResponseHandler" {
		traceSt, ok := m.apiStatsMap[res.Trace.FuncName]
		if !ok {
			traceSt = &topAPIStats{}
		}
		traceSt.addAPICall(1)
		if res.Trace.HTTP != nil {
			traceSt.addAPIBytesRX(res.Trace.HTTP.CallStats.InputBytes)
			traceSt.addAPIBytesTX(res.Trace.HTTP.CallStats.OutputBytes)
		}
		if res.Trace.HTTP.RespInfo.StatusCode >= 499 {
			traceSt.addAPIErrors(1)
		}
		m.apiStatsMap[res.Trace.FuncName] = traceSt
	}

	table.SetHeader([]string{"API", "RX", "TX", "CALLS", "ERRORS"})
	data := make([][]string, 0, len(m.apiStatsMap))

	for k, stats := range m.apiStatsMap {
		data = append(data, []string{
			k,
			whiteStyle.Render(humanize.IBytes(stats.loadAPIBytesRX())),
			whiteStyle.Render(humanize.IBytes(stats.loadAPIBytesTX())),
			whiteStyle.Render(fmt.Sprintf("%d", stats.loadAPICall())),
			whiteStyle.Render(fmt.Sprintf("%d", stats.loadAPIErrors())),
		})
	}
	sort.Slice(data, func(i, j int) bool {
		return data[i][0] < data[j][0]
	})

	table.AppendBulk(data)
	table.Render()

	if !m.quitting {
		s.WriteString(fmt.Sprintf("\nTopAPI: %s", m.spinner.View()))
	} else {
		var totalTX, totalRX, totalCalls uint64
		lastReqTime := m.lastResult.apiCallInfo.Trace.Time
		if m.lastResult.apiCallInfo.Trace.Time.IsZero() {
			lastReqTime = time.Now()
		}
		for _, stats := range m.apiStatsMap {
			totalRX += stats.loadAPIBytesRX()
			totalTX += stats.loadAPIBytesTX()
			totalCalls += stats.loadAPICall()
		}
		msg := fmt.Sprintf("\n========\nTotal: %d CALLS, %s RX, %s TX - in %.02fs",
			totalCalls,
			humanize.IBytes(totalRX),
			humanize.IBytes(totalTX),
			lastReqTime.Sub(m.startTime).Seconds(),
		)
		s.WriteString(msg)
		s.WriteString("\n")
	}
	return s.String()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"github.com/andrewcopp/coup"
)

func init() {

}

func main() {

	wins := 0
	losses := 0
	for i := 0; i < 1; i++ {
		var chooser coup.Chooser
		chooser = coup.NewAgent(i, 0.8)
		one := coup.NewPlayer("Player One", chooser, false)
		for j := 0; j < 2000; j++ {
			chooser = coup.NewRandom()
			two := coup.NewPlayer("Player Two", chooser, false)
			three := coup.NewPlayer("Player Three", chooser, true)
			four := coup.NewPlayer("Player Four", chooser, true)
			five := coup.NewPlayer("Player Five", chooser, true)

			game := coup.NewGame([]*coup.Player{one, two, three, four, five})

			game.Setup()
			winner := game.Play()
			if winner == one {
				wins++
			} else {
				losses++
			}
		}
	}

	fmt.Println()
	fmt.Println(float64(wins)/float64((wins+losses))*100.0, "%")
	fmt.Println()

}
<commit_msg>Properly Training (Minus TensorFlow)<commit_after>package main

import (
	"fmt"
	"math/rand"
	"time"

	"github.com/andrewcopp/coup"
)

func init() {

}

func main() {

	for i := 0; i < 5; i++ {
		wins := 0
		losses := 0
		var chooser coup.Chooser
		chooser = coup.NewAgent(i, 0.8)
		one := coup.NewPlayer("Player One", chooser, false)
		for j := 0; j < 1000; j++ {
			rand.Seed(time.Now().UnixNano())
			if i > 0 {
				chooser = coup.NewAgent(rand.Intn(i), 0.0)
			} else {
				chooser = coup.NewRandom()
			}
			two := coup.NewPlayer("Player Two", chooser, false)
			chooser = coup.NewRandom()
			three := coup.NewPlayer("Player Three", chooser, true)
			four := coup.NewPlayer("Player Four", chooser, true)
			five := coup.NewPlayer("Player Five", chooser, true)

			game := coup.NewGame([]*coup.Player{one, two, three, four, five})

			game.Setup()
			winner := game.Play()
			if winner == one {
				wins++
			} else {
				losses++
			}
		}
		fmt.Println()
		fmt.Println(float64(wins)/float64((wins+losses))*100.0, "%")
		fmt.Println()
	}

}
<|endoftext|>