<commit_before>package main

import (
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/jeffail/gabs"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/log"
)

const namespace = "marathon"

type Exporter struct {
	uri            *url.URL
	metricsVersion string
	duration       prometheus.Gauge
	scrapeError    prometheus.Gauge
	totalErrors    prometheus.Counter
	totalScrapes   prometheus.Counter
	Counters       *CounterContainer
	Gauges         *GaugeContainer
}

// Describe implements prometheus.Collector.
func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
	log.Debugln("Describing metrics")
	metricCh := make(chan prometheus.Metric)
	doneCh := make(chan struct{})

	go func() {
		for m := range metricCh {
			ch <- m.Desc()
		}
		close(doneCh)
	}()

	e.Collect(metricCh)
	close(metricCh)
	<-doneCh
}

// Collect implements prometheus.Collector.
func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
	log.Debugln("Collecting metrics")
	e.scrape(ch)

	ch <- e.duration
	ch <- e.totalScrapes
	ch <- e.totalErrors
	ch <- e.scrapeError
}

func (e *Exporter) scrape(ch chan<- prometheus.Metric) {
	e.totalScrapes.Inc()

	var err error
	defer func(begin time.Time) {
		e.duration.Set(time.Since(begin).Seconds())
		if err == nil {
			e.scrapeError.Set(0)
		} else {
			e.totalErrors.Inc()
			e.scrapeError.Set(1)
		}
	}(time.Now())

	client := &http.Client{
		Timeout: 10 * time.Second,
		Transport: &http.Transport{
			Dial: (&net.Dialer{
				Timeout: 10 * time.Second,
			}).Dial,
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: true,
			},
		},
	}

	response, err := client.Get(fmt.Sprintf("%v/metrics", e.uri))
	if err != nil {
		log.Debugf("Problem connecting to metrics endpoint: %v\n", err)
		return
	}

	body, err := ioutil.ReadAll(response.Body)
	response.Body.Close()
	if err != nil {
		log.Debugf("Problem reading metrics response body: %v\n", err)
		return
	}

	json, err := gabs.ParseJSON(body)
	if err != nil {
		log.Debugf("Problem parsing metrics response body: %v\n", err)
		return
	}

	elements, err := json.ChildrenMap()
	for key, element := range elements {
		switch key {
		case "message":
			log.Debugf("Problem collecting metrics: %s\n", element.Data().(string))
			return
		case "version":
			if e.metricsVersion == "" {
				e.metricsVersion = element.Data().(string)
				log.Infof("Collecting Marathon metrics version: %s\n", e.metricsVersion)
			}
		case "counters":
			e.scrapeCounters(element, ch)
		case "gauges":
			e.scrapeGauges(element, ch)
		case "histograms":
			log.Debugln("Found historgrams")
		case "meters":
			e.scrapeMeters(element, ch)
		case "timers":
			log.Debugln("Found timers")
		}
	}
}

func (e *Exporter) scrapeCounters(json *gabs.Container, ch chan<- prometheus.Metric) {
	elements, _ := json.ChildrenMap()
	for key, element := range elements {
		log.Debugf("Found counter metric %s\n", key)
		name := metricName(key)

		data := element.Path("count").Data()
		count, ok := data.(float64)
		if !ok {
			log.Debugf("Bad conversion! Skipping counter %s with count %v\n", name, data)
			continue
		}

		log.Debugf("Adding counter %s with count %v\n", name, count)
		counter := e.Counters.GetOrCreate(name)
		counter.WithLabelValues().Set(count)
		counter.Collect(ch)
	}
}

func (e *Exporter) scrapeGauges(json *gabs.Container, ch chan<- prometheus.Metric) {
	elements, _ := json.ChildrenMap()
	for key, element := range elements {
		log.Debugf("Found gauge metric %s\n", key)
		name := metricName(key)

		data := element.Path("value").Data()
		value, ok := data.(float64)
		if !ok {
			log.Debugf("Bad conversion! Skipping gauge %s with value %v\n", name, data)
			continue
		}

		log.Debugf("Adding gauge %s with value %v\n", name, value)
		gauge := e.Gauges.GetOrCreate(name)
		gauge.WithLabelValues().Set(value)
		gauge.Collect(ch)
	}
}

func (e *Exporter) scrapeMeters(json *gabs.Container, ch chan<- prometheus.Metric) {
	elements, _ := json.ChildrenMap()
	for key, element := range elements {
		log.Debugf("Found meter metric %s\n", key)
		name := metricName(key)
		e.scrapeMeter(name, element, ch)
	}
}

func (e *Exporter) scrapeMeter(name string, json *gabs.Container, ch chan<- prometheus.Metric) {
	count, ok := json.Path("count").Data().(float64)
	if !ok {
		log.Debugf("Bad meter! %s has no count\n", name)
		return
	}

	log.Debugf("Adding meter %s with count %v\n", name, count)
	counter := e.Counters.GetOrCreate(name + "_count")
	counter.WithLabelValues().Set(count)
	counter.Collect(ch)

	gauge := e.Gauges.GetOrCreate(name, "rate")
	properties, _ := json.ChildrenMap()
	for propName, property := range properties {
		if !strings.Contains(propName, "rate") {
			continue
		}

		if value, ok := property.Data().(float64); ok {
			gauge.WithLabelValues(
				rateName(propName),
			).Set(value)
		}
	}

	gauge.Collect(ch)
}

func rateName(originalRate string) (name string) {
	switch originalRate {
	case "m1_rate":
		name = "1m"
	case "m5_rate":
		name = "5m"
	case "m15_rate":
		name = "15m"
	default:
		name = strings.TrimSuffix(originalRate, "_rate")
	}
	return
}

func metricName(originalName string) (name string) {
	name = strings.ToLower(originalName)
	name = strings.Replace(name, ".", "_", -1)
	name = strings.Replace(name, "-", "_", -1)
	name = strings.Replace(name, "$", "_", -1)
	return
}

func NewExporter(uri *url.URL) *Exporter {
	return &Exporter{
		uri:      uri,
		Counters: NewCounterContainer(),
		Gauges:   NewGaugeContainer(),
		duration: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Subsystem: "exporter",
			Name:      "last_scrape_duration_seconds",
			Help:      "Duration of the last scrape of metrics from Marathon.",
		}),
		scrapeError: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Subsystem: "exporter",
			Name:      "last_scrape_error",
			Help:      "Whether the last scrape of metrics from Marathon resulted in an error (1 for error, 0 for success).",
		}),
		totalScrapes: prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: namespace,
			Subsystem: "exporter",
			Name:      "scrapes_total",
			Help:      "Total number of times Marathon was scraped for metrics.",
		}),
		totalErrors: prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: namespace,
			Subsystem: "exporter",
			Name:      "errors_total",
			Help:      "Total number of times the exporter experienced errors collecting Marathon metrics.",
		}),
	}
}
<commit_msg>Scrape Marathon histograms<commit_after>package main

import (
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/jeffail/gabs"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/log"
)

const namespace = "marathon"

type Exporter struct {
	uri            *url.URL
	metricsVersion string
	duration       prometheus.Gauge
	scrapeError    prometheus.Gauge
	totalErrors    prometheus.Counter
	totalScrapes   prometheus.Counter
	Counters       *CounterContainer
	Gauges         *GaugeContainer
}

// Describe implements prometheus.Collector.
func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
	log.Debugln("Describing metrics")
	metricCh := make(chan prometheus.Metric)
	doneCh := make(chan struct{})

	go func() {
		for m := range metricCh {
			ch <- m.Desc()
		}
		close(doneCh)
	}()

	e.Collect(metricCh)
	close(metricCh)
	<-doneCh
}

// Collect implements prometheus.Collector.
func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
	log.Debugln("Collecting metrics")
	e.scrape(ch)

	ch <- e.duration
	ch <- e.totalScrapes
	ch <- e.totalErrors
	ch <- e.scrapeError
}

func (e *Exporter) scrape(ch chan<- prometheus.Metric) {
	e.totalScrapes.Inc()

	var err error
	defer func(begin time.Time) {
		e.duration.Set(time.Since(begin).Seconds())
		if err == nil {
			e.scrapeError.Set(0)
		} else {
			e.totalErrors.Inc()
			e.scrapeError.Set(1)
		}
	}(time.Now())

	client := &http.Client{
		Timeout: 10 * time.Second,
		Transport: &http.Transport{
			Dial: (&net.Dialer{
				Timeout: 10 * time.Second,
			}).Dial,
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: true,
			},
		},
	}

	response, err := client.Get(fmt.Sprintf("%v/metrics", e.uri))
	if err != nil {
		log.Debugf("Problem connecting to metrics endpoint: %v\n", err)
		return
	}

	body, err := ioutil.ReadAll(response.Body)
	response.Body.Close()
	if err != nil {
		log.Debugf("Problem reading metrics response body: %v\n", err)
		return
	}

	json, err := gabs.ParseJSON(body)
	if err != nil {
		log.Debugf("Problem parsing metrics response body: %v\n", err)
		return
	}

	elements, err := json.ChildrenMap()
	for key, element := range elements {
		switch key {
		case "message":
			log.Debugf("Problem collecting metrics: %s\n", element.Data().(string))
			return
		case "version":
			if e.metricsVersion == "" {
				e.metricsVersion = element.Data().(string)
				log.Infof("Collecting Marathon metrics version: %s\n", e.metricsVersion)
			}
		case "counters":
			e.scrapeCounters(element, ch)
		case "gauges":
			e.scrapeGauges(element, ch)
		case "histograms":
			e.scrapeHistograms(element, ch)
		case "meters":
			e.scrapeMeters(element, ch)
		case "timers":
			log.Debugln("Found timers")
		}
	}
}

func (e *Exporter) scrapeCounters(json *gabs.Container, ch chan<- prometheus.Metric) {
	elements, _ := json.ChildrenMap()
	for key, element := range elements {
		log.Debugf("Found counter metric %s\n", key)
		name := metricName(key)

		data := element.Path("count").Data()
		count, ok := data.(float64)
		if !ok {
			log.Debugf("Bad conversion! Skipping counter %s with count %v\n", name, data)
			continue
		}

		log.Debugf("Adding counter %s with count %v\n", name, count)
		counter := e.Counters.GetOrCreate(name)
		counter.WithLabelValues().Set(count)
		counter.Collect(ch)
	}
}

func (e *Exporter) scrapeGauges(json *gabs.Container, ch chan<- prometheus.Metric) {
	elements, _ := json.ChildrenMap()
	for key, element := range elements {
		log.Debugf("Found gauge metric %s\n", key)
		name := metricName(key)

		data := element.Path("value").Data()
		value, ok := data.(float64)
		if !ok {
			log.Debugf("Bad conversion! Skipping gauge %s with value %v\n", name, data)
			continue
		}

		log.Debugf("Adding gauge %s with value %v\n", name, value)
		gauge := e.Gauges.GetOrCreate(name)
		gauge.WithLabelValues().Set(value)
		gauge.Collect(ch)
	}
}

func (e *Exporter) scrapeMeters(json *gabs.Container, ch chan<- prometheus.Metric) {
	elements, _ := json.ChildrenMap()
	for key, element := range elements {
		log.Debugf("Found meter metric %s\n", key)
		name := metricName(key)
		e.scrapeMeter(name, element, ch)
	}
}

func (e *Exporter) scrapeMeter(name string, json *gabs.Container, ch chan<- prometheus.Metric) {
	count, ok := json.Path("count").Data().(float64)
	if !ok {
		log.Debugf("Bad meter! %s has no count\n", name)
		return
	}

	log.Debugf("Adding meter %s with count %v\n", name, count)
	counter := e.Counters.GetOrCreate(name + "_count")
	counter.WithLabelValues().Set(count)
	counter.Collect(ch)

	gauge := e.Gauges.GetOrCreate(name, "rate")
	properties, _ := json.ChildrenMap()
	for propName, property := range properties {
		if !strings.Contains(propName, "rate") {
			continue
		}

		if value, ok := property.Data().(float64); ok {
			gauge.WithLabelValues(
				rateName(propName),
			).Set(value)
		}
	}

	gauge.Collect(ch)
}

func (e *Exporter) scrapeHistograms(json *gabs.Container, ch chan<- prometheus.Metric) {
	elements, _ := json.ChildrenMap()
	for key, element := range elements {
		log.Debugf("Found histogram metric %s\n", key)
		name := metricName(key)
		e.scrapeHistogram(name, element, ch)
	}
}

func (e *Exporter) scrapeHistogram(name string, json *gabs.Container, ch chan<- prometheus.Metric) {
	count, ok := json.Path("count").Data().(float64)
	if !ok {
		log.Debugf("Bad histogram! %s has no count\n", name)
		return
	}

	log.Debugf("Adding histogram %s with count %v\n", name, count)
	counter := e.Counters.GetOrCreate(name + "_count")
	counter.WithLabelValues().Set(count)
	counter.Collect(ch)

	percentiles := e.Gauges.GetOrCreate(name, "percentile")
	properties, _ := json.ChildrenMap()
	for propName, property := range properties {
		switch propName {
		case "p50", "p75", "p95", "p98", "p99", "p999":
			if value, ok := property.Data().(float64); ok {
				percentiles.WithLabelValues(
					"0." + propName[1:],
				).Set(value)
			}
		case "max", "min", "mean", "stdev":
			if value, ok := property.Data().(float64); ok {
				gauge := e.Gauges.GetOrCreate(name + "_" + propName)
				gauge.WithLabelValues().Set(value)
				gauge.Collect(ch)
			}
		}

	}

	percentiles.Collect(ch)
}

func rateName(originalRate string) (name string) {
	switch originalRate {
	case "m1_rate":
		name = "1m"
	case "m5_rate":
		name = "5m"
	case "m15_rate":
		name = "15m"
	default:
		name = strings.TrimSuffix(originalRate, "_rate")
	}
	return
}

func metricName(originalName string) (name string) {
	name = strings.ToLower(originalName)
	name = strings.Replace(name, ".", "_", -1)
	name = strings.Replace(name, "-", "_", -1)
	name = strings.Replace(name, "$", "_", -1)
	return
}

func NewExporter(uri *url.URL) *Exporter {
	return &Exporter{
		uri:      uri,
		Counters: NewCounterContainer(),
		Gauges:   NewGaugeContainer(),
		duration: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Subsystem: "exporter",
			Name:      "last_scrape_duration_seconds",
			Help:      "Duration of the last scrape of metrics from Marathon.",
		}),
		scrapeError: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Subsystem: "exporter",
			Name:      "last_scrape_error",
			Help:      "Whether the last scrape of metrics from Marathon resulted in an error (1 for error, 0 for success).",
		}),
		totalScrapes: prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: namespace,
			Subsystem: "exporter",
			Name:      "scrapes_total",
			Help:      "Total number of times Marathon was scraped for metrics.",
		}),
		totalErrors: prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: namespace,
			Subsystem: "exporter",
			Name:      "errors_total",
			Help:      "Total number of times the exporter experienced errors collecting Marathon metrics.",
		}),
	}
}
<|endoftext|>


<commit_before>/*
	This file is part of go-ethereum

	go-ethereum is free software: you can redistribute it and/or modify
	it under the terms of the GNU Lesser General Public License as published by
	the Free Software Foundation, either version 3 of the License, or
	(at your option) any later version.

	go-ethereum is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
	GNU General Public License for more details.

	You should have received a copy of the GNU Lesser General Public License
	along with go-ethereum.  If not, see <http://www.gnu.org/licenses/>.
*/
/**
 * @authors
 * 	Gustav Simonsson <gustav.simonsson@gmail.com>
 * @date 2015
 *
 */

package main

import (
	"bytes"
	"encoding/hex"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math/big"
	"os"
	"runtime"
	"strings"

	"github.com/ethereum/go-ethereum/cmd/utils"
	"github.com/ethereum/go-ethereum/core"
	types "github.com/ethereum/go-ethereum/core/types"
	"github.com/ethereum/go-ethereum/ethdb"
	"github.com/ethereum/go-ethereum/event"
	"github.com/ethereum/go-ethereum/logger"
	"github.com/ethereum/go-ethereum/rlp"
)

type Account struct {
	Balance string
	Code    string
	Nonce   string
	Storage map[string]string
}

type BlockHeader struct {
	Bloom            string
	Coinbase         string
	Difficulty       string
	ExtraData        string
	GasLimit         string
	GasUsed          string
	MixHash          string
	Nonce            string
	Number           string
	ParentHash       string
	ReceiptTrie      string
	SeedHash         string
	StateRoot        string
	Timestamp        string
	TransactionsTrie string
	UncleHash        string
}

type Tx struct {
	Data     string
	GasLimit string
	GasPrice string
	Nonce    string
	R        string
	S        string
	To       string
	V        string
	Value    string
}

type Block struct {
	BlockHeader  BlockHeader
	Rlp          string
	Transactions []Tx
	UncleHeaders []string
}

type Test struct {
	Blocks             []Block
	GenesisBlockHeader BlockHeader
	Pre                map[string]Account
}

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "%s <testfile>\n", os.Args[0])
		flag.PrintDefaults()
	}
	flag.Parse()

	runtime.GOMAXPROCS(runtime.NumCPU())
	logger.AddLogSystem(logger.NewStdLogSystem(os.Stderr, log.LstdFlags, logger.DebugDetailLevel))
	defer func() { logger.Flush() }()

	if len(os.Args) < 2 {
		utils.Fatalf("Please specify a test file as the first argument.")
	}
	blocks, err := loadBlocksFromTestFile(os.Args[1])
	if err != nil {
		utils.Fatalf("Could not load blocks: %v", err)
	}

	chain := memchain()
	chain.ResetWithGenesisBlock(blocks[0])
	if err = chain.InsertChain(types.Blocks{blocks[1]}); err != nil {
		utils.Fatalf("Error: %v", err)
	} else {
		fmt.Println("PASS")
	}
}

func memchain() *core.ChainManager {
	blockdb, err := ethdb.NewMemDatabase()
	if err != nil {
		utils.Fatalf("Could not create in-memory database: %v", err)
	}
	statedb, err := ethdb.NewMemDatabase()
	if err != nil {
		utils.Fatalf("Could not create in-memory database: %v", err)
	}
	return core.NewChainManager(blockdb, statedb, new(event.TypeMux))
}

func loadBlocksFromTestFile(filePath string) (blocks types.Blocks, err error) {
	fileContent, err := ioutil.ReadFile(filePath)
	if err != nil {
		return
	}
	bt := make(map[string]Test)
	if err = json.Unmarshal(fileContent, &bt); err != nil {
		return
	}

	// TODO: support multiple blocks; loop over all blocks
	gbh := new(types.Header)

	// Let's use slighlty different namings for the same things, because that's awesome.
	gbh.ParentHash, err = hex_decode(bt["SimpleTx"].GenesisBlockHeader.ParentHash)
	gbh.UncleHash, err = hex_decode(bt["SimpleTx"].GenesisBlockHeader.UncleHash)
	gbh.Coinbase, err = hex_decode(bt["SimpleTx"].GenesisBlockHeader.Coinbase)
	gbh.Root, err = hex_decode(bt["SimpleTx"].GenesisBlockHeader.StateRoot)
	gbh.TxHash, err = hex_decode(bt["SimpleTx"].GenesisBlockHeader.TransactionsTrie)
	gbh.ReceiptHash, err = hex_decode(bt["SimpleTx"].GenesisBlockHeader.ReceiptTrie)
	gbh.Bloom, err = hex_decode(bt["SimpleTx"].GenesisBlockHeader.Bloom)

	gbh.MixDigest, err = hex_decode(bt["SimpleTx"].GenesisBlockHeader.MixHash)
	gbh.SeedHash, err = hex_decode(bt["SimpleTx"].GenesisBlockHeader.SeedHash)

	d, _ := new(big.Int).SetString(bt["SimpleTx"].GenesisBlockHeader.Difficulty, 10)
	gbh.Difficulty = d

	n, _ := new(big.Int).SetString(bt["SimpleTx"].GenesisBlockHeader.Number, 10)
	gbh.Number = n

	gl, _ := new(big.Int).SetString(bt["SimpleTx"].GenesisBlockHeader.GasLimit, 10)
	gbh.GasLimit = gl

	gu, _ := new(big.Int).SetString(bt["SimpleTx"].GenesisBlockHeader.GasUsed, 10)
	gbh.GasUsed = gu

	ts, _ := new(big.Int).SetString(bt["SimpleTx"].GenesisBlockHeader.Timestamp, 0)
	gbh.Time = ts.Uint64()

	extra, err := hex_decode(bt["SimpleTx"].GenesisBlockHeader.ExtraData)
	gbh.Extra = string(extra) // TODO: change ExtraData to byte array

	nonce, _ := hex_decode(bt["SimpleTx"].GenesisBlockHeader.Nonce)
	gbh.Nonce = nonce

	if err != nil {
		return
	}

	gb := types.NewBlockWithHeader(gbh)
	//gb.uncles = *new([]*types.Header)
	//gb.transactions = *new(types.Transactions)
	gb.Td = new(big.Int)
	gb.Reward = new(big.Int)

	testBlock := new(types.Block)

	rlpBytes, err := hex_decode(bt["SimpleTx"].Blocks[0].Rlp)
	err = rlp.Decode(bytes.NewReader(rlpBytes), &testBlock)
	if err != nil {
		return
	}

	blocks = types.Blocks{
		gb,
		testBlock,
	}

	return
}

func hex_decode(s string) (res []byte, err error) {
	return hex.DecodeString(strings.TrimPrefix(s, "0x"))
}
<commit_msg>removed seed hash<commit_after>/*
	This file is part of go-ethereum

	go-ethereum is free software: you can redistribute it and/or modify
	it under the terms of the GNU Lesser General Public License as published by
	the Free Software Foundation, either version 3 of the License, or
	(at your option) any later version.

	go-ethereum is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
	GNU General Public License for more details.

	You should have received a copy of the GNU Lesser General Public License
	along with go-ethereum.  If not, see <http://www.gnu.org/licenses/>.
*/
/**
 * @authors
 * 	Gustav Simonsson <gustav.simonsson@gmail.com>
 * @date 2015
 *
 */

package main

import (
	"bytes"
	"encoding/hex"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math/big"
	"os"
	"runtime"
	"strings"

	"github.com/ethereum/go-ethereum/cmd/utils"
	"github.com/ethereum/go-ethereum/core"
	types "github.com/ethereum/go-ethereum/core/types"
	"github.com/ethereum/go-ethereum/ethdb"
	"github.com/ethereum/go-ethereum/event"
	"github.com/ethereum/go-ethereum/logger"
	"github.com/ethereum/go-ethereum/rlp"
)

type Account struct {
	Balance string
	Code    string
	Nonce   string
	Storage map[string]string
}

type BlockHeader struct {
	Bloom            string
	Coinbase         string
	Difficulty       string
	ExtraData        string
	GasLimit         string
	GasUsed          string
	MixHash          string
	Nonce            string
	Number           string
	ParentHash       string
	ReceiptTrie      string
	SeedHash         string
	StateRoot        string
	Timestamp        string
	TransactionsTrie string
	UncleHash        string
}

type Tx struct {
	Data     string
	GasLimit string
	GasPrice string
	Nonce    string
	R        string
	S        string
	To       string
	V        string
	Value    string
}

type Block struct {
	BlockHeader  BlockHeader
	Rlp          string
	Transactions []Tx
	UncleHeaders []string
}

type Test struct {
	Blocks             []Block
	GenesisBlockHeader BlockHeader
	Pre                map[string]Account
}

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "%s <testfile>\n", os.Args[0])
		flag.PrintDefaults()
	}
	flag.Parse()

	runtime.GOMAXPROCS(runtime.NumCPU())
	logger.AddLogSystem(logger.NewStdLogSystem(os.Stderr, log.LstdFlags, logger.DebugDetailLevel))
	defer func() { logger.Flush() }()

	if len(os.Args) < 2 {
		utils.Fatalf("Please specify a test file as the first argument.")
	}
	blocks, err := loadBlocksFromTestFile(os.Args[1])
	if err != nil {
		utils.Fatalf("Could not load blocks: %v", err)
	}

	chain := memchain()
	chain.ResetWithGenesisBlock(blocks[0])
	if err = chain.InsertChain(types.Blocks{blocks[1]}); err != nil {
		utils.Fatalf("Error: %v", err)
	} else {
		fmt.Println("PASS")
	}
}

func memchain() *core.ChainManager {
	blockdb, err := ethdb.NewMemDatabase()
	if err != nil {
		utils.Fatalf("Could not create in-memory database: %v", err)
	}
	statedb, err := ethdb.NewMemDatabase()
	if err != nil {
		utils.Fatalf("Could not create in-memory database: %v", err)
	}
	return core.NewChainManager(blockdb, statedb, new(event.TypeMux))
}

func loadBlocksFromTestFile(filePath string) (blocks types.Blocks, err error) {
	fileContent, err := ioutil.ReadFile(filePath)
	if err != nil {
		return
	}
	bt := make(map[string]Test)
	if err = json.Unmarshal(fileContent, &bt); err != nil {
		return
	}

	// TODO: support multiple blocks; loop over all blocks
	gbh := new(types.Header)

	// Let's use slighlty different namings for the same things, because that's awesome.
	gbh.ParentHash, err = hex_decode(bt["SimpleTx"].GenesisBlockHeader.ParentHash)
	gbh.UncleHash, err = hex_decode(bt["SimpleTx"].GenesisBlockHeader.UncleHash)
	gbh.Coinbase, err = hex_decode(bt["SimpleTx"].GenesisBlockHeader.Coinbase)
	gbh.Root, err = hex_decode(bt["SimpleTx"].GenesisBlockHeader.StateRoot)
	gbh.TxHash, err = hex_decode(bt["SimpleTx"].GenesisBlockHeader.TransactionsTrie)
	gbh.ReceiptHash, err = hex_decode(bt["SimpleTx"].GenesisBlockHeader.ReceiptTrie)
	gbh.Bloom, err = hex_decode(bt["SimpleTx"].GenesisBlockHeader.Bloom)

	gbh.MixDigest, err = hex_decode(bt["SimpleTx"].GenesisBlockHeader.MixHash)
	//gbh.SeedHash, err = hex_decode(bt["SimpleTx"].GenesisBlockHeader.SeedHash)

	d, _ := new(big.Int).SetString(bt["SimpleTx"].GenesisBlockHeader.Difficulty, 10)
	gbh.Difficulty = d

	n, _ := new(big.Int).SetString(bt["SimpleTx"].GenesisBlockHeader.Number, 10)
	gbh.Number = n

	gl, _ := new(big.Int).SetString(bt["SimpleTx"].GenesisBlockHeader.GasLimit, 10)
	gbh.GasLimit = gl

	gu, _ := new(big.Int).SetString(bt["SimpleTx"].GenesisBlockHeader.GasUsed, 10)
	gbh.GasUsed = gu

	ts, _ := new(big.Int).SetString(bt["SimpleTx"].GenesisBlockHeader.Timestamp, 0)
	gbh.Time = ts.Uint64()

	extra, err := hex_decode(bt["SimpleTx"].GenesisBlockHeader.ExtraData)
	gbh.Extra = string(extra) // TODO: change ExtraData to byte array

	nonce, _ := hex_decode(bt["SimpleTx"].GenesisBlockHeader.Nonce)
	gbh.Nonce = nonce

	if err != nil {
		return
	}

	gb := types.NewBlockWithHeader(gbh)
	//gb.uncles = *new([]*types.Header)
	//gb.transactions = *new(types.Transactions)
	gb.Td = new(big.Int)
	gb.Reward = new(big.Int)

	testBlock := new(types.Block)

	rlpBytes, err := hex_decode(bt["SimpleTx"].Blocks[0].Rlp)
	err = rlp.Decode(bytes.NewReader(rlpBytes), &testBlock)
	if err != nil {
		return
	}

	blocks = types.Blocks{
		gb,
		testBlock,
	}

	return
}

func hex_decode(s string) (res []byte, err error) {
	return hex.DecodeString(strings.TrimPrefix(s, "0x"))
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"os"
	"syscall"
	"time"

	"github.com/containers/buildah"
	"github.com/containers/buildah/pkg/unshare"
	"github.com/containers/buildah/util"
	is "github.com/containers/image/storage"
	"github.com/containers/image/types"
	"github.com/containers/storage"
	digest "github.com/opencontainers/go-digest"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
)

var needToShutdownStore = false

func getStore(c *cobra.Command) (storage.Store, error) {
	options, err := storage.DefaultStoreOptions(unshare.IsRootless(), unshare.GetRootlessUID())
	if err != nil {
		return nil, err
	}
	if c.Flag("root").Changed || c.Flag("runroot").Changed {
		options.GraphRoot = globalFlagResults.Root
		options.RunRoot = globalFlagResults.RunRoot
	}
	if err := os.Setenv("XDG_RUNTIME_DIR", options.RunRoot); err != nil {
		return nil, errors.New("could not set XDG_RUNTIME_DIR")
	}

	if c.Flag("storage-driver").Changed {
		options.GraphDriverName = globalFlagResults.StorageDriver
		// If any options setup in config, these should be dropped if user overrode the driver
		options.GraphDriverOptions = []string{}
	}
	if c.Flag("storage-opt").Changed {
		if len(globalFlagResults.StorageOpts) > 0 {
			options.GraphDriverOptions = globalFlagResults.StorageOpts
		}
	}

	// Do not allow to mount a graphdriver that is not vfs if we are creating the userns as part
	// of the mount command.
	// Differently, allow the mount if we are already in a userns, as the mount point will still
	// be accessible once "buildah mount" exits.
	if os.Geteuid() != 0 && options.GraphDriverName != "vfs" {
		return nil, fmt.Errorf("cannot mount using driver %s in rootless mode. You need to run it in a `buildah unshare` session", options.GraphDriverName)
	}

	// For uid/gid mappings, first we check the global definitions
	if len(globalFlagResults.UserNSUID) > 0 || len(globalFlagResults.UserNSGID) > 0 {
		if !(len(globalFlagResults.UserNSUID) > 0 && len(globalFlagResults.UserNSGID) > 0) {
			return nil, errors.Errorf("--userns-uid-map and --userns-gid-map must be used together")
		}
		uopts := globalFlagResults.UserNSUID
		gopts := globalFlagResults.UserNSGID
		if len(uopts) == 0 {
			return nil, errors.New("--userns-uid-map used with no mappings?")
		}
		if len(gopts) == 0 {
			return nil, errors.New("--userns-gid-map used with no mappings?")
		}
		uidmap, gidmap, err := util.ParseIDMappings(uopts, gopts)
		if err != nil {
			return nil, err
		}
		options.UIDMap = uidmap
		options.GIDMap = gidmap
	}

	// If a subcommand has the flags, check if they are set; if so, override the global values
	localUIDMapFlag := c.Flags().Lookup("userns-uid-map")
	localGIDMapFlag := c.Flags().Lookup("userns-gid-map")
	if localUIDMapFlag != nil && localGIDMapFlag != nil && (localUIDMapFlag.Changed || localGIDMapFlag.Changed) {
		if !(localUIDMapFlag.Changed && localGIDMapFlag.Changed) {
			return nil, errors.Errorf("--userns-uid-map and --userns-gid-map must be used together")
		}
		// We know that the flags are both !nil and have been changed (i.e. have values)
		uopts, _ := c.Flags().GetStringSlice("userns-uid-map")
		gopts, _ := c.Flags().GetStringSlice("userns-gid-map")
		if len(uopts) == 0 {
			return nil, errors.New("--userns-uid-map used with no mappings?")
		}
		if len(gopts) == 0 {
			return nil, errors.New("--userns-gid-map used with no mappings?")
		}
		uidmap, gidmap, err := util.ParseIDMappings(uopts, gopts)
		if err != nil {
			return nil, err
		}
		options.UIDMap = uidmap
		options.GIDMap = gidmap
	}

	oldUmask := syscall.Umask(0022)
	if (oldUmask & ^0022) != 0 {
		logrus.Debugf("umask value too restrictive.  Forcing it to 022")
	}

	store, err := storage.GetStore(options)
	if store != nil {
		is.Transport.SetStore(store)
	}
	needToShutdownStore = true
	return store, err
}

func openBuilder(ctx context.Context, store storage.Store, name string) (builder *buildah.Builder, err error) {
	if name != "" {
		builder, err = buildah.OpenBuilder(store, name)
		if os.IsNotExist(err) {
			options := buildah.ImportOptions{
				Container: name,
			}
			builder, err = buildah.ImportBuilder(ctx, store, options)
		}
	}
	if err != nil {
		return nil, errors.Wrapf(err, "error reading build container")
	}
	if builder == nil {
		return nil, errors.Errorf("error finding build container")
	}
	return builder, nil
}

func openBuilders(store storage.Store) (builders []*buildah.Builder, err error) {
	return buildah.OpenAllBuilders(store)
}

func openImage(ctx context.Context, sc *types.SystemContext, store storage.Store, name string) (builder *buildah.Builder, err error) {
	options := buildah.ImportFromImageOptions{
		Image:         name,
		SystemContext: sc,
	}
	builder, err = buildah.ImportBuilderFromImage(ctx, store, options)
	if err != nil {
		return nil, errors.Wrapf(err, "error reading image")
	}
	if builder == nil {
		return nil, errors.Errorf("error mocking up build configuration")
	}
	return builder, nil
}

func getDateAndDigestAndSize(ctx context.Context, image storage.Image, store storage.Store) (time.Time, string, int64, error) {
	created := time.Time{}
	is.Transport.SetStore(store)
	storeRef, err := is.Transport.ParseStoreReference(store, image.ID)
	if err != nil {
		return created, "", -1, err
	}
	img, err := storeRef.NewImage(ctx, nil)
	if err != nil {
		return created, "", -1, err
	}
	defer img.Close()
	imgSize, sizeErr := img.Size()
	if sizeErr != nil {
		imgSize = -1
	}
	manifest, _, manifestErr := img.Manifest(ctx)
	manifestDigest := ""
	if manifestErr == nil && len(manifest) > 0 {
		manifestDigest = digest.Canonical.FromBytes(manifest).String()
	}
	inspectInfo, inspectErr := img.Inspect(ctx)
	if inspectErr == nil && inspectInfo != nil {
		created = *inspectInfo.Created
	}
	if sizeErr != nil {
		err = sizeErr
	} else if manifestErr != nil {
		err = manifestErr
	} else if inspectErr != nil {
		err = inspectErr
	}
	return created, manifestDigest, imgSize, err
}

// getContext returns a context.TODO
func getContext() context.Context {
	return context.TODO()
}

func getUserFlags() pflag.FlagSet {
	fs := pflag.FlagSet{}
	fs.String("user", "", "`user[:group]` to run the command as")
	return fs
}

func defaultFormat() string {
	format := os.Getenv("BUILDAH_FORMAT")
	if format != "" {
		return format
	}
	return buildah.OCI
}

// imageIsParent goes through the layers in the store and checks if i.TopLayer is
// the parent of any other layer in store. Double check that image with that
// layer exists as well.
func imageIsParent(store storage.Store, topLayer string) (bool, error) {
	children, err := getChildren(store, topLayer)
	if err != nil {
		return false, err
	}
	return len(children) > 0, nil
}

// getParent returns the image ID of the parent. Return nil if a parent is not found.
func getParent(store storage.Store, topLayer string) (*storage.Image, error) {
	images, err := store.Images()
	if err != nil {
		return nil, errors.Wrapf(err, "unable to retrieve images from store")
	}
	layer, err := store.Layer(topLayer)
	if err != nil {
		return nil, errors.Wrapf(err, "unable to retrieve layers from store")
	}
	for _, img := range images {
		if img.TopLayer == layer.Parent {
			return &img, nil
		}
	}
	return nil, nil
}

// getChildren returns a list of the imageIDs that depend on the image
func getChildren(store storage.Store, topLayer string) ([]string, error) {
	var children []string
	images, err := store.Images()
	if err != nil {
		return nil, errors.Wrapf(err, "unable to retrieve images from store")
	}
	layers, err := store.Layers()
	if err != nil {
		return nil, errors.Wrapf(err, "unable to retrieve layers from store")
	}

	for _, layer := range layers {
		if layer.Parent == topLayer {
			if imageID := getImageOfTopLayer(images, layer.ID); len(imageID) > 0 {
				children = append(children, imageID...)
			}
		}
	}
	return children, nil
}

// getImageOfTopLayer returns the image ID where layer is the top layer of the image
func getImageOfTopLayer(images []storage.Image, layer string) []string {
	var matches []string
	for _, img := range images {
		if img.TopLayer == layer {
			matches = append(matches, img.ID)
		}
	}
	return matches
}

func getFormat(format string) (string, error) {
	switch format {
	case buildah.OCI:
		return buildah.OCIv1ImageManifest, nil
	case buildah.DOCKER:
		return buildah.Dockerv2ImageManifest, nil
	default:
		return "", errors.Errorf("unrecognized image type %q", format)
	}
}

// Tail returns a string slice after the first element unless there are
// not enough elements, then it returns an empty slice.  This is to replace
// the urfavecli Tail method for args
func Tail(a []string) []string {
	if len(a) >= 2 {
		return a[1:]
	}
	return []string{}
}

// UsageTemplate returns the usage template for podman commands
// This blocks the desplaying of the global options. The main podman
// command should not use this.
func UsageTemplate() string {
	return `Usage:{{if .Runnable}}
  {{.UseLine}}{{end}}{{if .HasAvailableSubCommands}}

  {{.CommandPath}} [command]{{end}}{{if gt (len .Aliases) 0}}

Aliases:
  {{.NameAndAliases}}{{end}}{{if .HasExample}}

Examples:
  {{.Example}}{{end}}{{if .HasAvailableSubCommands}}

Available Commands:{{range .Commands}}{{if (or .IsAvailableCommand (eq .Name "help"))}}
  {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableLocalFlags}}

Flags:
{{.LocalFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasAvailableInheritedFlags}}
{{end}}
`
}
<commit_msg>Use errors.Cause() when checking for IsNotExist errors<commit_after>package main

import (
	"context"
	"fmt"
	"os"
	"syscall"
	"time"

	"github.com/containers/buildah"
	"github.com/containers/buildah/pkg/unshare"
	"github.com/containers/buildah/util"
	is "github.com/containers/image/storage"
	"github.com/containers/image/types"
	"github.com/containers/storage"
	digest "github.com/opencontainers/go-digest"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
)

var needToShutdownStore = false

func getStore(c *cobra.Command) (storage.Store, error) {
	options, err := storage.DefaultStoreOptions(unshare.IsRootless(), unshare.GetRootlessUID())
	if err != nil {
		return nil, err
	}
	if c.Flag("root").Changed || c.Flag("runroot").Changed {
		options.GraphRoot = globalFlagResults.Root
		options.RunRoot = globalFlagResults.RunRoot
	}
	if err := os.Setenv("XDG_RUNTIME_DIR", options.RunRoot); err != nil {
		return nil, errors.New("could not set XDG_RUNTIME_DIR")
	}

	if c.Flag("storage-driver").Changed {
		options.GraphDriverName = globalFlagResults.StorageDriver
		// If any options setup in config, these should be dropped if user overrode the driver
		options.GraphDriverOptions = []string{}
	}
	if c.Flag("storage-opt").Changed {
		if len(globalFlagResults.StorageOpts) > 0 {
			options.GraphDriverOptions = globalFlagResults.StorageOpts
		}
	}

	// Do not allow to mount a graphdriver that is not vfs if we are creating the userns as part
	// of the mount command.
	// Differently, allow the mount if we are already in a userns, as the mount point will still
	// be accessible once "buildah mount" exits.
	if os.Geteuid() != 0 && options.GraphDriverName != "vfs" {
		return nil, fmt.Errorf("cannot mount using driver %s in rootless mode. You need to run it in a `buildah unshare` session", options.GraphDriverName)
	}

	// For uid/gid mappings, first we check the global definitions
	if len(globalFlagResults.UserNSUID) > 0 || len(globalFlagResults.UserNSGID) > 0 {
		if !(len(globalFlagResults.UserNSUID) > 0 && len(globalFlagResults.UserNSGID) > 0) {
			return nil, errors.Errorf("--userns-uid-map and --userns-gid-map must be used together")
		}
		uopts := globalFlagResults.UserNSUID
		gopts := globalFlagResults.UserNSGID
		if len(uopts) == 0 {
			return nil, errors.New("--userns-uid-map used with no mappings?")
		}
		if len(gopts) == 0 {
			return nil, errors.New("--userns-gid-map used with no mappings?")
		}
		uidmap, gidmap, err := util.ParseIDMappings(uopts, gopts)
		if err != nil {
			return nil, err
		}
		options.UIDMap = uidmap
		options.GIDMap = gidmap
	}

	// If a subcommand has the flags, check if they are set; if so, override the global values
	localUIDMapFlag := c.Flags().Lookup("userns-uid-map")
	localGIDMapFlag := c.Flags().Lookup("userns-gid-map")
	if localUIDMapFlag != nil && localGIDMapFlag != nil && (localUIDMapFlag.Changed || localGIDMapFlag.Changed) {
		if !(localUIDMapFlag.Changed && localGIDMapFlag.Changed) {
			return nil, errors.Errorf("--userns-uid-map and --userns-gid-map must be used together")
		}
		// We know that the flags are both !nil and have been changed (i.e. have values)
		uopts, _ := c.Flags().GetStringSlice("userns-uid-map")
		gopts, _ := c.Flags().GetStringSlice("userns-gid-map")
		if len(uopts) == 0 {
			return nil, errors.New("--userns-uid-map used with no mappings?")
		}
		if len(gopts) == 0 {
			return nil, errors.New("--userns-gid-map used with no mappings?")
		}
		uidmap, gidmap, err := util.ParseIDMappings(uopts, gopts)
		if err != nil {
			return nil, err
		}
		options.UIDMap = uidmap
		options.GIDMap = gidmap
	}

	oldUmask := syscall.Umask(0022)
	if (oldUmask & ^0022) != 0 {
		logrus.Debugf("umask value too restrictive.  Forcing it to 022")
	}

	store, err := storage.GetStore(options)
	if store != nil {
		is.Transport.SetStore(store)
	}
	needToShutdownStore = true
	return store, err
}

func openBuilder(ctx context.Context, store storage.Store, name string) (builder *buildah.Builder, err error) {
	if name != "" {
		builder, err = buildah.OpenBuilder(store, name)
		if os.IsNotExist(errors.Cause(err)) {
			options := buildah.ImportOptions{
				Container: name,
			}
			builder, err = buildah.ImportBuilder(ctx, store, options)
		}
	}
	if err != nil {
		return nil, errors.Wrapf(err, "error reading build container")
	}
	if builder == nil {
		return nil, errors.Errorf("error finding build container")
	}
	return builder, nil
}

func openBuilders(store storage.Store) (builders []*buildah.Builder, err error) {
	return buildah.OpenAllBuilders(store)
}

func openImage(ctx context.Context, sc *types.SystemContext, store storage.Store, name string) (builder *buildah.Builder, err error) {
	options := buildah.ImportFromImageOptions{
		Image:         name,
		SystemContext: sc,
	}
	builder, err = buildah.ImportBuilderFromImage(ctx, store, options)
	if err != nil {
		return nil, errors.Wrapf(err, "error reading image")
	}
	if builder == nil {
		return nil, errors.Errorf("error mocking up build configuration")
	}
	return builder, nil
}

func getDateAndDigestAndSize(ctx context.Context, image storage.Image, store storage.Store) (time.Time, string, int64, error) {
	created := time.Time{}
	is.Transport.SetStore(store)
	storeRef, err := is.Transport.ParseStoreReference(store, image.ID)
	if err != nil {
		return created, "", -1, err
	}
	img, err := storeRef.NewImage(ctx, nil)
	if err != nil {
		return created, "", -1, err
	}
	defer img.Close()
	imgSize, sizeErr := img.Size()
	if sizeErr != nil {
		imgSize = -1
	}
	manifest, _, manifestErr := img.Manifest(ctx)
	manifestDigest := ""
	if manifestErr == nil && len(manifest) > 0 {
		manifestDigest = digest.Canonical.FromBytes(manifest).String()
	}
	inspectInfo, inspectErr := img.Inspect(ctx)
	if inspectErr == nil && inspectInfo != nil {
		created = *inspectInfo.Created
	}
	if sizeErr != nil {
		err = sizeErr
	} else if manifestErr != nil {
		err = manifestErr
	} else if inspectErr != nil {
		err = inspectErr
	}
	return created, manifestDigest, imgSize, err
}

// getContext returns a context.TODO
func getContext() context.Context {
	return context.TODO()
}

func getUserFlags() pflag.FlagSet {
	fs := pflag.FlagSet{}
	fs.String("user", "", "`user[:group]` to run the command as")
	return fs
}

func defaultFormat() string {
	format := os.Getenv("BUILDAH_FORMAT")
	if format != "" {
		return format
	}
	return buildah.OCI
}

// imageIsParent goes through the layers in the store and checks if i.TopLayer is
// the parent of any other layer in store. Double check that image with that
// layer exists as well.
func imageIsParent(store storage.Store, topLayer string) (bool, error) {
	children, err := getChildren(store, topLayer)
	if err != nil {
		return false, err
	}
	return len(children) > 0, nil
}

// getParent returns the image ID of the parent. Return nil if a parent is not found.
func getParent(store storage.Store, topLayer string) (*storage.Image, error) {
	images, err := store.Images()
	if err != nil {
		return nil, errors.Wrapf(err, "unable to retrieve images from store")
	}
	layer, err := store.Layer(topLayer)
	if err != nil {
		return nil, errors.Wrapf(err, "unable to retrieve layers from store")
	}
	for _, img := range images {
		if img.TopLayer == layer.Parent {
			return &img, nil
		}
	}
	return nil, nil
}

// getChildren returns a list of the imageIDs that depend on the image
func getChildren(store storage.Store, topLayer string) ([]string, error) {
	var children []string
	images, err := store.Images()
	if err != nil {
		return nil, errors.Wrapf(err, "unable to retrieve images from store")
	}
	layers, err := store.Layers()
	if err != nil {
		return nil, errors.Wrapf(err, "unable to retrieve layers from store")
	}

	for _, layer := range layers {
		if layer.Parent == topLayer {
			if imageID := getImageOfTopLayer(images, layer.ID); len(imageID) > 0 {
				children = append(children, imageID...)
			}
		}
	}
	return children, nil
}

// getImageOfTopLayer returns the image ID where layer is the top layer of the image
func getImageOfTopLayer(images []storage.Image, layer string) []string {
	var matches []string
	for _, img := range images {
		if img.TopLayer == layer {
			matches = append(matches, img.ID)
		}
	}
	return matches
}

func getFormat(format string) (string, error) {
	switch format {
	case buildah.OCI:
		return buildah.OCIv1ImageManifest, nil
	case buildah.DOCKER:
		return buildah.Dockerv2ImageManifest, nil
	default:
		return "", errors.Errorf("unrecognized image type %q", format)
	}
}

// Tail returns a string slice after the first element unless there are
// not enough elements, then it returns an empty slice.  This is to replace
// the urfavecli Tail method for args
func Tail(a []string) []string {
	if len(a) >= 2 {
		return a[1:]
	}
	return []string{}
}

// UsageTemplate returns the usage template for podman commands
// This blocks the desplaying of the global options. The main podman
// command should not use this.
func UsageTemplate() string {
	return `Usage:{{if .Runnable}}
  {{.UseLine}}{{end}}{{if .HasAvailableSubCommands}}

  {{.CommandPath}} [command]{{end}}{{if gt (len .Aliases) 0}}

Aliases:
  {{.NameAndAliases}}{{end}}{{if .HasExample}}

Examples:
  {{.Example}}{{end}}{{if .HasAvailableSubCommands}}

Available Commands:{{range .Commands}}{{if (or .IsAvailableCommand (eq .Name "help"))}}
  {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableLocalFlags}}

Flags:
{{.LocalFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasAvailableInheritedFlags}}
{{end}}
`
}
<|endoftext|>


<commit_before>// Dumps the command-line arguments.
package main

import (
	"os"

	"github.com/shurcooL/go-goon"
)

func main() {
	goon.DumpExpr(os.Args)
}
<commit_msg>Add cwd to cmd/dump_args.<commit_after>// Dumps the command-line arguments.
package main

import (
	"os"

	"github.com/shurcooL/go-goon"
)

func main() {
	goon.DumpExpr(os.Args)
	goon.DumpExpr(os.Getwd())
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/Sirupsen/logrus"
)

var tvMazeURLTemplate = "http://api.tvmaze.com/singlesearch/shows?q=%s&embed=episodes"

type TvMazeShow struct {
	Name  string `json:"name"`
	Image struct {
		Medium   string `json:"medium"`
		Original string `json:"original"`
	} `json:"image"`
	Summary  string `json:"summary"`
	Embedded struct {
		Episodes []struct {
			ID      int64  `json:"id"`
			Name    string `json:"name"`
			Season  int64  `json:"season"`
			Episode int64  `json:"episode"`
			Summary string `json:"summary"`
			//AirDate time.Time `json:"airdate"`
			Image struct {
				Medium   string `json:"medium"`
				Original string `json:"original"`
			} `json:"image"`
		} `json:"episodes"`
	} `json:"_embedded"`
}

type TvMazeClient struct {
	URLTemplate string
	logger      *logrus.Entry
}

func (t TvMazeClient) Find(q string) (*TvMazeShow, error) {
	response, err := http.Get(fmt.Sprintf(
		t.URLTemplate, q))
	if err != nil {
		t.logger.WithField("err", err).Error("Failed to get a response")
		return nil, err
	}
	if response.StatusCode == 404 {
		t.logger.Warn("No match found")
		return nil, err
	}

	show := &TvMazeShow{}
	if err := json.NewDecoder(response.Body).Decode(show); err != nil {
		t.logger.WithField("err", err).Error("Failed to decode")
		return nil, err
	}

	return show, nil
}
<commit_msg>extract nested struct for better testing<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/Sirupsen/logrus"
)

var tvMazeURLTemplate = "http://api.tvmaze.com/singlesearch/shows?q=%s&embed=episodes"

type Episode struct {
	ID      int64  `json:"id"`
	Name    string `json:"name"`
	Season  int64  `json:"season"`
	Episode int64  `json:"episode"`
	Summary string `json:"summary"`
	//AirDate time.Time `json:"airdate"`
	Image struct {
		Medium   string `json:"medium"`
		Original string `json:"original"`
	} `json:"image"`
}

type TvMazeShow struct {
	Name  string `json:"name"`
	Image struct {
		Medium   string `json:"medium"`
		Original string `json:"original"`
	} `json:"image"`
	Summary  string `json:"summary"`
	Embedded struct {
		Episodes []Episode `json:"episodes"`
	} `json:"_embedded"`
}

type TvMazeClient struct {
	URLTemplate string
	logger      *logrus.Entry
}

func (t TvMazeClient) Find(q string) (*TvMazeShow, error) {
	response, err := http.Get(fmt.Sprintf(
		t.URLTemplate, q))
	if err != nil {
		t.logger.WithField("err", err).Error("Failed to get a response")
		return nil, err
	}
	if response.StatusCode == 404 {
		t.logger.Warn("No match found")
		return nil, err
	}

	show := &TvMazeShow{}
	if err := json.NewDecoder(response.Body).Decode(show); err != nil {
		t.logger.WithField("err", err).Error("Failed to decode")
		return nil, err
	}

	return show, nil
}
<|endoftext|>


<commit_before>/*
Copyright The Helm Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"io"

	"github.com/gosuri/uitable"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"

	"helm.sh/helm/v3/cmd/helm/require"
	"helm.sh/helm/v3/pkg/cli/output"
	"helm.sh/helm/v3/pkg/repo"
)

func newRepoListCmd(out io.Writer) *cobra.Command {
	var outfmt output.Format
	cmd := &cobra.Command{
		Use:               "list",
		Aliases:           []string{"ls"},
		Short:             "list chart repositories",
		Args:              require.NoArgs,
		ValidArgsFunction: noCompletions,
		RunE: func(cmd *cobra.Command, args []string) error {
			f, err := repo.LoadFile(settings.RepositoryConfig)
			if isNotExist(err) || (len(f.Repositories) == 0 && !(outfmt == output.JSON || outfmt == output.YAML)) {
				return errors.New("no repositories to show")
			}

			return outfmt.Write(out, &repoListWriter{f.Repositories})
		},
	}

	bindOutputFlag(cmd, &outfmt)

	return cmd
}

type repositoryElement struct {
	Name string `json:"name"`
	URL  string `json:"url"`
}

type repoListWriter struct {
	repos []*repo.Entry
}

func (r *repoListWriter) WriteTable(out io.Writer) error {
	table := uitable.New()
	table.AddRow("NAME", "URL")
	for _, re := range r.repos {
		table.AddRow(re.Name, re.URL)
	}
	return output.EncodeTable(out, table)
}

func (r *repoListWriter) WriteJSON(out io.Writer) error {
	return r.encodeByFormat(out, output.JSON)
}

func (r *repoListWriter) WriteYAML(out io.Writer) error {
	return r.encodeByFormat(out, output.YAML)
}

func (r *repoListWriter) encodeByFormat(out io.Writer, format output.Format) error {
	// Initialize the array so no results returns an empty array instead of null
	repolist := make([]repositoryElement, 0, len(r.repos))

	for _, re := range r.repos {
		repolist = append(repolist, repositoryElement{Name: re.Name, URL: re.URL})
	}

	switch format {
	case output.JSON:
		return output.EncodeJSON(out, repolist)
	case output.YAML:
		return output.EncodeYAML(out, repolist)
	}

	// Because this is a non-exported function and only called internally by
	// WriteJSON and WriteYAML, we shouldn't get invalid types
	return nil
}

// Returns all repos from repos, except those with names matching ignoredRepoNames
// Inspired by https://stackoverflow.com/a/28701031/893211
func filterRepos(repos []*repo.Entry, ignoredRepoNames []string) []*repo.Entry {
	// if ignoredRepoNames is nil, just return repo
	if ignoredRepoNames == nil {
		return repos
	}

	filteredRepos := make([]*repo.Entry, 0)

	ignored := make(map[string]bool, len(ignoredRepoNames))
	for _, repoName := range ignoredRepoNames {
		ignored[repoName] = true
	}

	for _, repo := range repos {
		if _, removed := ignored[repo.Name]; !removed {
			filteredRepos = append(filteredRepos, repo)
		}
	}

	return filteredRepos
}

// Provide dynamic auto-completion for repo names
func compListRepos(prefix string, ignoredRepoNames []string) []string {
	var rNames []string

	f, err := repo.LoadFile(settings.RepositoryConfig)
	if err == nil && len(f.Repositories) > 0 {
		filteredRepos := filterRepos(f.Repositories, ignoredRepoNames)
		for _, repo := range filteredRepos {
			rNames = append(rNames, fmt.Sprintf("%s\t%s", repo.Name, repo.URL))
		}
	}
	return rNames
}
<commit_msg>fix(helm): ignore file-not-found error for `helm repo list -o json`<commit_after>/*
Copyright The Helm Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"io"

	"github.com/gosuri/uitable"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"

	"helm.sh/helm/v3/cmd/helm/require"
	"helm.sh/helm/v3/pkg/cli/output"
	"helm.sh/helm/v3/pkg/repo"
)

func newRepoListCmd(out io.Writer) *cobra.Command {
	var outfmt output.Format
	cmd := &cobra.Command{
		Use:               "list",
		Aliases:           []string{"ls"},
		Short:             "list chart repositories",
		Args:              require.NoArgs,
		ValidArgsFunction: noCompletions,
		RunE: func(cmd *cobra.Command, args []string) error {
			f, _ := repo.LoadFile(settings.RepositoryConfig)
			if len(f.Repositories) == 0 && !(outfmt == output.JSON || outfmt == output.YAML) {
				return errors.New("no repositories to show")
			}

			return outfmt.Write(out, &repoListWriter{f.Repositories})
		},
	}

	bindOutputFlag(cmd, &outfmt)

	return cmd
}

type repositoryElement struct {
	Name string `json:"name"`
	URL  string `json:"url"`
}

type repoListWriter struct {
	repos []*repo.Entry
}

func (r *repoListWriter) WriteTable(out io.Writer) error {
	table := uitable.New()
	table.AddRow("NAME", "URL")
	for _, re := range r.repos {
		table.AddRow(re.Name, re.URL)
	}
	return output.EncodeTable(out, table)
}

func (r *repoListWriter) WriteJSON(out io.Writer) error {
	return r.encodeByFormat(out, output.JSON)
}

func (r *repoListWriter) WriteYAML(out io.Writer) error {
	return r.encodeByFormat(out, output.YAML)
}

func (r *repoListWriter) encodeByFormat(out io.Writer, format output.Format) error {
	// Initialize the array so no results returns an empty array instead of null
	repolist := make([]repositoryElement, 0, len(r.repos))

	for _, re := range r.repos {
		repolist = append(repolist, repositoryElement{Name: re.Name, URL: re.URL})
	}

	switch format {
	case output.JSON:
		return output.EncodeJSON(out, repolist)
	case output.YAML:
		return output.EncodeYAML(out, repolist)
	}

	// Because this is a non-exported function and only called internally by
	// WriteJSON and WriteYAML, we shouldn't get invalid types
	return nil
}

// Returns all repos from repos, except those with names matching ignoredRepoNames
// Inspired by https://stackoverflow.com/a/28701031/893211
func filterRepos(repos []*repo.Entry, ignoredRepoNames []string) []*repo.Entry {
	// if ignoredRepoNames is nil, just return repo
	if ignoredRepoNames == nil {
		return repos
	}

	filteredRepos := make([]*repo.Entry, 0)

	ignored := make(map[string]bool, len(ignoredRepoNames))
	for _, repoName := range ignoredRepoNames {
		ignored[repoName] = true
	}

	for _, repo := range repos {
		if _, removed := ignored[repo.Name]; !removed {
			filteredRepos = append(filteredRepos, repo)
		}
	}

	return filteredRepos
}

// Provide dynamic auto-completion for repo names
func compListRepos(prefix string, ignoredRepoNames []string) []string {
	var rNames []string

	f, err := repo.LoadFile(settings.RepositoryConfig)
	if err == nil && len(f.Repositories) > 0 {
		filteredRepos := filterRepos(f.Repositories, ignoredRepoNames)
		for _, repo := range filteredRepos {
			rNames = append(rNames, fmt.Sprintf("%s\t%s", repo.Name, repo.URL))
		}
	}
	return rNames
}
<|endoftext|>


<commit_before>package cmd

import (
	"github.com/Unknwon/com"
	"github.com/gpmgo/gopm/doc"
	"os"
	"os/exec"
	"path/filepath"
	"syscall"
	"unsafe"
)

func makeLink(srcPath, destPath string) error {
	// Check if Windows version is XP.
	if getWindowsVersion() >= 6 {
		cmd := exec.Command("cmd", "/c", "mklink", "/j", destPath, srcPath)
		return cmd.Run()
	}

	// XP.
	isWindowsXP = true
	os.RemoveAll(destPath)
	_, dirName := filepath.Split(destPath)
	// .vendor dir should not be copy
	if dirName != doc.VENDOR {
		return com.CopyDir(srcPath, destPath)
	}
	return nil
}

func volumnType(dir string) string {
	pd := dir[:3]
	dll := syscall.MustLoadDLL("kernel32.dll")
	GetVolumeInformation := dll.MustFindProc("GetVolumeInformationW")

	var volumeNameSize uint32 = 260
	var nFileSystemNameSize, lpVolumeSerialNumber uint32
	var lpFileSystemFlags, lpMaximumComponentLength uint32
	var lpFileSystemNameBuffer, volumeName [260]byte
	var ps *uint16 = syscall.StringToUTF16Ptr(pd)

	_, _, _ = GetVolumeInformation.Call(uintptr(unsafe.Pointer(ps)),
		uintptr(unsafe.Pointer(&volumeName)),
		uintptr(volumeNameSize),
		uintptr(unsafe.Pointer(&lpVolumeSerialNumber)),
		uintptr(unsafe.Pointer(&lpMaximumComponentLength)),
		uintptr(unsafe.Pointer(&lpFileSystemFlags)),
		uintptr(unsafe.Pointer(&lpFileSystemNameBuffer)),
		uintptr(unsafe.Pointer(&nFileSystemNameSize)), 0)

	var bytes []byte
	if lpFileSystemNameBuffer[6] == 0 {
		bytes = []byte{lpFileSystemNameBuffer[0], lpFileSystemNameBuffer[2],
			lpFileSystemNameBuffer[4]}
	} else {
		bytes = []byte{lpFileSystemNameBuffer[0], lpFileSystemNameBuffer[2],
			lpFileSystemNameBuffer[4], lpFileSystemNameBuffer[6]}
	}

	return string(bytes)
}

func getWindowsVersion() int {
	dll := syscall.MustLoadDLL("kernel32.dll")
	p := dll.MustFindProc("GetVersion")
	v, _, _ := p.Call()
	return int(byte(v))
}
<commit_msg>copy for windows xp bug fixed<commit_after>package cmd

import (
	"github.com/Unknwon/com"
	"github.com/gpmgo/gopm/doc"
	"os"
	"os/exec"
	"path/filepath"
	"syscall"
	"unsafe"
)

func makeLink(srcPath, destPath string) error {
	// Check if Windows version is XP.
	if getWindowsVersion() >= 6 {
		cmd := exec.Command("cmd", "/c", "mklink", "/j", destPath, srcPath)
		return cmd.Run()
	}

	// XP.
	isWindowsXP = true
	os.RemoveAll(destPath)

	err := com.CopyDir(srcPath, destPath)
	if err == nil {
		// .vendor dir should not be copy
		os.RemoveAll(filepath.Join(destPath, doc.VENDOR))
	}
	return err
}

func volumnType(dir string) string {
	pd := dir[:3]
	dll := syscall.MustLoadDLL("kernel32.dll")
	GetVolumeInformation := dll.MustFindProc("GetVolumeInformationW")

	var volumeNameSize uint32 = 260
	var nFileSystemNameSize, lpVolumeSerialNumber uint32
	var lpFileSystemFlags, lpMaximumComponentLength uint32
	var lpFileSystemNameBuffer, volumeName [260]byte
	var ps *uint16 = syscall.StringToUTF16Ptr(pd)

	_, _, _ = GetVolumeInformation.Call(uintptr(unsafe.Pointer(ps)),
		uintptr(unsafe.Pointer(&volumeName)),
		uintptr(volumeNameSize),
		uintptr(unsafe.Pointer(&lpVolumeSerialNumber)),
		uintptr(unsafe.Pointer(&lpMaximumComponentLength)),
		uintptr(unsafe.Pointer(&lpFileSystemFlags)),
		uintptr(unsafe.Pointer(&lpFileSystemNameBuffer)),
		uintptr(unsafe.Pointer(&nFileSystemNameSize)), 0)

	var bytes []byte
	if lpFileSystemNameBuffer[6] == 0 {
		bytes = []byte{lpFileSystemNameBuffer[0], lpFileSystemNameBuffer[2],
			lpFileSystemNameBuffer[4]}
	} else {
		bytes = []byte{lpFileSystemNameBuffer[0], lpFileSystemNameBuffer[2],
			lpFileSystemNameBuffer[4], lpFileSystemNameBuffer[6]}
	}

	return string(bytes)
}

func getWindowsVersion() int {
	dll := syscall.MustLoadDLL("kernel32.dll")
	p := dll.MustFindProc("GetVersion")
	v, _, _ := p.Call()
	return int(byte(v))
}
<|endoftext|>


<commit_before>package services

import (
	jcontext "context"
	"hash/crc32"
	"net/http"
	"strings"
	"time"

	"github.com/JREAMLU/core/com"
	"github.com/JREAMLU/core/db/mysql"
	"github.com/JREAMLU/core/global"
	io "github.com/JREAMLU/core/inout"
	"github.com/JREAMLU/jkernel/base/models/mentity"
	"github.com/JREAMLU/jkernel/base/models/mmysql"
	"github.com/JREAMLU/jkernel/base/models/mredis"
	"github.com/JREAMLU/jkernel/base/services/atom"
	"github.com/JREAMLU/jkernel/base/services/entity"
	"github.com/astaxie/beego"
	"github.com/astaxie/beego/validation"
	"github.com/beego/i18n"
	"github.com/pquerna/ffjson/ffjson"
)

type Url struct {
	Meta struct {
		Auth string
	} `json:"meta" valid:"Required"`
	Data struct {
		Urls []interface{} `json:"urls" valid:"Required"`
		IP   string        `json:"ip" valid:"IP"`
	} `json:"data" valid:"Required"`
}

type UrlExpand struct {
	Shorten []string `json:"shorten" valid:"Required"`
}

var ip string

const (
	DELETE = 0
	NORMAL = 1
)

func GetParams(url Url) Url {
	return url
}

func (r *Url) Valid(v *validation.Validation) {}

func (r *Url) GoShorten(jctx jcontext.Context, data map[string]interface{}) (httpStatus int, output io.Output) {
	ffjson.Unmarshal(data["body"].([]byte), r)
	ip = data["headermap"].(http.Header)["X-Forwarded-For"][0]
	ch, err := io.InputParamsCheck(jctx, data, &r.Data)
	if err != nil {
		return http.StatusExpectationFailed, io.Fail(ch.Message, "DATAPARAMSILLEGAL", jctx.Value("requestID").(string))
	}

	if len(r.Data.Urls) > 10 {
		return http.StatusExpectationFailed, io.Fail(i18n.Tr(global.Lang, "url.NUMBERLIMIT"), "DATAPARAMSILLEGAL", jctx.Value("requestID").(string))
	}

	list, err := shorten(r)
	if err != nil {
		beego.Info(jctx.Value("requestID").(string), ":", "shorten error: ", err)
		return http.StatusExpectationFailed, io.Fail(i18n.Tr(global.Lang, "url.SHORTENILLEGAL"), "DATAPARAMSILLEGAL", jctx.Value("requestID").(string))
	}

	var datalist entity.DataList
	datalist.List = list
	datalist.Total = len(list)

	return http.StatusCreated, io.Suc(datalist, ch.RequestID)
}

func shorten(r *Url) (map[string]interface{}, error) {
	list, err := setDB(r)
	if err != nil {
		return nil, err
	}
	return list, nil
}

func setDB(r *Url) (map[string]interface{}, error) {
	var shortenMap = make(map[string]interface{})
	reply, err := mredis.ShortenHMGet(r.Data.Urls)
	if err != nil {
		return nil, err
	}
	exist, notExistLongCRCList, notExistMapList := splitExistOrNot(r, reply)
	if len(notExistLongCRCList) == 0 && len(notExistMapList) == 0 {
		return exist, nil
	}

	existShortListInDB, err := mmysql.GetShortens(notExistLongCRCList)
	if err != nil {
		return nil, err
	}
	existQueue, existQueueShortenList, existQueueExpandList, notExistMapList := getAllData(existShortListInDB, notExistMapList)
	if len(existQueue) == 0 {
		return exist, nil
	}

	tx := mysql.X.Begin()
	if len(notExistMapList) > 0 {
		err = mmysql.ShortenInBatch(notExistMapList, tx)
		if err != nil {
			tx.Rollback()
			return nil, err
		}
	}

	if len(existQueueShortenList) > 0 {
		_, err = mredis.ShortenHMSet(existQueueShortenList)
		if err != nil {
			tx.Rollback()
			return nil, err
		}
	}

	if len(existQueueExpandList) > 0 {
		_, err = mredis.ExpandHMSet(existQueueExpandList)
		if err != nil {
			tx.Rollback()
			return nil, err
		}
	}

	tx.Commit()

	shortenMap = com.MapMerge(exist, existQueue)

	return shortenMap, nil
}

func splitExistOrNot(r *Url, reply []string) (exist map[string]interface{}, notExistLongCRCList []uint64, notExistMapList []mentity.Redirect) {
	exist = make(map[string]interface{})
	var redirect mentity.Redirect
	for key, url := range r.Data.Urls {
		if reply[key] != "" {
			atom.Mu.Lock()
			exist[url.(string)] = reply[key]
			atom.Mu.Unlock()
		} else {
			longCrc := uint64(crc32.ChecksumIEEE([]byte(url.(string))))
			shortUrl := atom.GetShortenUrl(url.(string))
			shortCrc := uint64(crc32.ChecksumIEEE([]byte(shortUrl)))
			notExistLongCRCList = append(notExistLongCRCList, longCrc)
			redirect.LongUrl = url.(string)
			redirect.ShortUrl = shortUrl
			redirect.LongCrc = longCrc
			redirect.ShortCrc = shortCrc
			redirect.Status = NORMAL
			redirect.CreatedByIP = uint64(com.Ip2Int(ip))
			redirect.UpdateByIP = uint64(com.Ip2Int(ip))
			redirect.CreateAT = uint64(time.Now().Unix())
			redirect.UpdateAT = uint64(time.Now().Unix())
			notExistMapList = append(notExistMapList, redirect)
		}
		atom.Mu.Unlock()
	}
	return exist, notExistLongCRCList, notExistMapList
}

func getAllData(existShortListInDB []mentity.Redirect, notExistMapList []mentity.Redirect) (existQueue map[string]interface{}, existQueueShortenList []interface{}, existQueueExpandList []interface{}, notExistMapLists []mentity.Redirect) {
	existQueue = make(map[string]interface{})
	for _, existShortListInDBVal := range existShortListInDB {
		atom.Mu.Lock()
		existQueue[existShortListInDBVal.LongUrl] = existShortListInDBVal.ShortUrl
		atom.Mu.Unlock()
		existQueueShortenList = append(existQueueShortenList, existShortListInDBVal.LongUrl)
		existQueueShortenList = append(existQueueShortenList, existShortListInDBVal.ShortUrl)
		existQueueExpandList = append(existQueueExpandList, existShortListInDBVal.ShortUrl)
		existQueueExpandList = append(existQueueExpandList, existShortListInDBVal.LongUrl)
		for k, notExistMapListVal := range notExistMapList {
			if existShortListInDBVal.LongUrl == notExistMapListVal.LongUrl {
				notExistMapList = append(notExistMapList[:k], notExistMapList[k+1:]...)
			}
		}
	}
	for _, notExistMapListVal := range notExistMapList {
		atom.Mu.Lock()
		existQueue[notExistMapListVal.LongUrl] = notExistMapListVal.ShortUrl
		atom.Mu.Unlock()
		existQueueShortenList = append(existQueueShortenList, notExistMapListVal.LongUrl)
		existQueueShortenList = append(existQueueShortenList, notExistMapListVal.ShortUrl)
		existQueueExpandList = append(existQueueExpandList, notExistMapListVal.ShortUrl)
		existQueueExpandList = append(existQueueExpandList, notExistMapListVal.LongUrl)
	}
	return existQueue, existQueueShortenList, existQueueExpandList, notExistMapList
}

func (r *Url) GoExpand(jctx jcontext.Context, data map[string]interface{}) (httpStatus int, output io.Output) {
	var ue UrlExpand
	ffjson.Unmarshal([]byte(data["querystrjson"].(string)), &ue)

	ch, err := io.InputParamsCheck(jctx, data, ue)
	if err != nil {
		return http.StatusExpectationFailed, io.Fail(ch.Message, "DATAPARAMSILLEGAL", jctx.Value("requestID").(string))
	}

	list := expand(jctx, &ue)

	var datalist entity.DataList
	datalist.List = list
	datalist.Total = len(list)

	return http.StatusCreated, io.Suc(datalist, ch.RequestID)
}

func expand(jctx jcontext.Context, ue *UrlExpand) map[string]interface{} {
	list := make(map[string]interface{})
	shortens := ue.Shorten[0]
	for _, shorten := range strings.Split(shortens, ",") {
		reply, err := mredis.ExpandHGet(shorten)
		if err != nil {
			beego.Info(jctx.Value("requestID").(string), ":", "expand error: ", err)
		}
		atom.Mu.Lock()
		list[shorten] = reply
		atom.Mu.Unlock()
	}
	return list
}
<commit_msg>add log<commit_after>package services

import (
	jcontext "context"
	"hash/crc32"
	"net/http"
	"strings"
	"time"

	"github.com/JREAMLU/core/com"
	"github.com/JREAMLU/core/db/mysql"
	"github.com/JREAMLU/core/global"
	io "github.com/JREAMLU/core/inout"
	"github.com/JREAMLU/jkernel/base/models/mentity"
	"github.com/JREAMLU/jkernel/base/models/mmysql"
	"github.com/JREAMLU/jkernel/base/models/mredis"
	"github.com/JREAMLU/jkernel/base/services/atom"
	"github.com/JREAMLU/jkernel/base/services/entity"
	"github.com/astaxie/beego"
	"github.com/astaxie/beego/validation"
	"github.com/beego/i18n"
	"github.com/pquerna/ffjson/ffjson"
)

type Url struct {
	Meta struct {
		Auth string
	} `json:"meta" valid:"Required"`
	Data struct {
		Urls []interface{} `json:"urls" valid:"Required"`
		IP   string        `json:"ip" valid:"IP"`
	} `json:"data" valid:"Required"`
}

type UrlExpand struct {
	Shorten []string `json:"shorten" valid:"Required"`
}

var ip string

const (
	DELETE = 0
	NORMAL = 1
)

func GetParams(url Url) Url {
	return url
}

func (r *Url) Valid(v *validation.Validation) {}

func (r *Url) GoShorten(jctx jcontext.Context, data map[string]interface{}) (httpStatus int, output io.Output) {
	ffjson.Unmarshal(data["body"].([]byte), r)
	ip = data["headermap"].(http.Header)["X-Forwarded-For"][0]
	ch, err := io.InputParamsCheck(jctx, data, &r.Data)
	if err != nil {
		beego.Info(jctx.Value("requestID").(string), ":", "shorten error: ", err)
		return http.StatusExpectationFailed, io.Fail(ch.Message, "DATAPARAMSILLEGAL", jctx.Value("requestID").(string))
	}

	if len(r.Data.Urls) > 10 {
		beego.Info(jctx.Value("requestID").(string), ":", "shorten error: ", err)
		return http.StatusExpectationFailed, io.Fail(i18n.Tr(global.Lang, "url.NUMBERLIMIT"), "DATAPARAMSILLEGAL", jctx.Value("requestID").(string))
	}

	list, err := shorten(r)
	if err != nil {
		beego.Info(jctx.Value("requestID").(string), ":", "shorten error: ", err)
		return http.StatusExpectationFailed, io.Fail(i18n.Tr(global.Lang, "url.SHORTENILLEGAL"), "DATAPARAMSILLEGAL", jctx.Value("requestID").(string))
	}

	var datalist entity.DataList
	datalist.List = list
	datalist.Total = len(list)

	return http.StatusCreated, io.Suc(datalist, ch.RequestID)
}

func shorten(r *Url) (map[string]interface{}, error) {
	list, err := setDB(r)
	if err != nil {
		return nil, err
	}
	return list, nil
}

func setDB(r *Url) (map[string]interface{}, error) {
	var shortenMap = make(map[string]interface{})
	reply, err := mredis.ShortenHMGet(r.Data.Urls)
	if err != nil {
		return nil, err
	}
	exist, notExistLongCRCList, notExistMapList := splitExistOrNot(r, reply)
	if len(notExistLongCRCList) == 0 && len(notExistMapList) == 0 {
		return exist, nil
	}

	existShortListInDB, err := mmysql.GetShortens(notExistLongCRCList)
	if err != nil {
		return nil, err
	}
	existQueue, existQueueShortenList, existQueueExpandList, notExistMapList := getAllData(existShortListInDB, notExistMapList)
	if len(existQueue) == 0 {
		return exist, nil
	}

	tx := mysql.X.Begin()
	if len(notExistMapList) > 0 {
		err = mmysql.ShortenInBatch(notExistMapList, tx)
		if err != nil {
			tx.Rollback()
			return nil, err
		}
	}

	if len(existQueueShortenList) > 0 {
		_, err = mredis.ShortenHMSet(existQueueShortenList)
		if err != nil {
			tx.Rollback()
			return nil, err
		}
	}

	if len(existQueueExpandList) > 0 {
		_, err = mredis.ExpandHMSet(existQueueExpandList)
		if err != nil {
			tx.Rollback()
			return nil, err
		}
	}

	tx.Commit()

	shortenMap = com.MapMerge(exist, existQueue)

	return shortenMap, nil
}

func splitExistOrNot(r *Url, reply []string) (exist map[string]interface{}, notExistLongCRCList []uint64, notExistMapList []mentity.Redirect) {
	exist = make(map[string]interface{})
	var redirect mentity.Redirect
	for key, url := range r.Data.Urls {
		if reply[key] != "" {
			atom.Mu.Lock()
			exist[url.(string)] = reply[key]
			atom.Mu.Unlock()
		} else {
			longCrc := uint64(crc32.ChecksumIEEE([]byte(url.(string))))
			shortUrl := atom.GetShortenUrl(url.(string))
			shortCrc := uint64(crc32.ChecksumIEEE([]byte(shortUrl)))
			notExistLongCRCList = append(notExistLongCRCList, longCrc)
			redirect.LongUrl = url.(string)
			redirect.ShortUrl = shortUrl
			redirect.LongCrc = longCrc
			redirect.ShortCrc = shortCrc
			redirect.Status = NORMAL
			redirect.CreatedByIP = uint64(com.Ip2Int(ip))
			redirect.UpdateByIP = uint64(com.Ip2Int(ip))
			redirect.CreateAT = uint64(time.Now().Unix())
			redirect.UpdateAT = uint64(time.Now().Unix())
			notExistMapList = append(notExistMapList, redirect)
		}
		atom.Mu.Unlock()
	}
	return exist, notExistLongCRCList, notExistMapList
}

func getAllData(existShortListInDB []mentity.Redirect, notExistMapList []mentity.Redirect) (existQueue map[string]interface{}, existQueueShortenList []interface{}, existQueueExpandList []interface{}, notExistMapLists []mentity.Redirect) {
	existQueue = make(map[string]interface{})
	for _, existShortListInDBVal := range existShortListInDB {
		atom.Mu.Lock()
		existQueue[existShortListInDBVal.LongUrl] = existShortListInDBVal.ShortUrl
		atom.Mu.Unlock()
		existQueueShortenList = append(existQueueShortenList, existShortListInDBVal.LongUrl)
		existQueueShortenList = append(existQueueShortenList, existShortListInDBVal.ShortUrl)
		existQueueExpandList = append(existQueueExpandList, existShortListInDBVal.ShortUrl)
		existQueueExpandList = append(existQueueExpandList, existShortListInDBVal.LongUrl)
		for k, notExistMapListVal := range notExistMapList {
			if existShortListInDBVal.LongUrl == notExistMapListVal.LongUrl {
				notExistMapList = append(notExistMapList[:k], notExistMapList[k+1:]...)
			}
		}
	}
	for _, notExistMapListVal := range notExistMapList {
		atom.Mu.Lock()
		existQueue[notExistMapListVal.LongUrl] = notExistMapListVal.ShortUrl
		atom.Mu.Unlock()
		existQueueShortenList = append(existQueueShortenList, notExistMapListVal.LongUrl)
		existQueueShortenList = append(existQueueShortenList, notExistMapListVal.ShortUrl)
		existQueueExpandList = append(existQueueExpandList, notExistMapListVal.ShortUrl)
		existQueueExpandList = append(existQueueExpandList, notExistMapListVal.LongUrl)
	}
	return existQueue, existQueueShortenList, existQueueExpandList, notExistMapList
}

func (r *Url) GoExpand(jctx jcontext.Context, data map[string]interface{}) (httpStatus int, output io.Output) {
	var ue UrlExpand
	ffjson.Unmarshal([]byte(data["querystrjson"].(string)), &ue)

	ch, err := io.InputParamsCheck(jctx, data, ue)
	if err != nil {
		return http.StatusExpectationFailed, io.Fail(ch.Message, "DATAPARAMSILLEGAL", jctx.Value("requestID").(string))
	}

	list := expand(jctx, &ue)

	var datalist entity.DataList
	datalist.List = list
	datalist.Total = len(list)

	return http.StatusCreated, io.Suc(datalist, ch.RequestID)
}

func expand(jctx jcontext.Context, ue *UrlExpand) map[string]interface{} {
	list := make(map[string]interface{})
	shortens := ue.Shorten[0]
	for _, shorten := range strings.Split(shortens, ",") {
		reply, err := mredis.ExpandHGet(shorten)
		if err != nil {
			beego.Info(jctx.Value("requestID").(string), ":", "expand error: ", err)
		}
		atom.Mu.Lock()
		list[shorten] = reply
		atom.Mu.Unlock()
	}
	return list
}
<|endoftext|>


<commit_before>package main

import (
	"strings"
	"testing"
	. "github.com/onsi/gomega"
)

const diff = `
diff --git a/README.md b/README.md
index c9caa23..8ad488f 100644
--- a/README.md
+++ b/README.md
@@ -7,4 +7,5 @@ Check if git diff (commit, pr) contains copy pasted code.

'''
 % go get github.com/artemave/cccv
+% go get github.com/artemave/cccv
 % git diff | cccv
diff --git a/cccv.go b/cccv.go
index af3d9dc..0e96324 100644
--- a/cccv.go
+++ b/cccv.go
@@ -15,9 +15,11 @@ import (
 )

 type FileName string
+type FileName string

 type Change struct {
 	FileName
+	FileName
 	Line
 }

@@ -43,6 +45,7 @@ func main() {
 	}

 	o.O(results)
+	o.O(results)
 }

 func GenResultForFile(fName string, changes *[]*Change) FileResult {
`

func TestParsesDiff(t *testing.T) {
	RegisterTestingT(t)

	expectedChanges := []*Change{
		&Change{FileName: FileName("README.md"), Line: Line{Number: 10, Text: "% go get github.com/artemave/cccv"}},
		&Change{FileName: FileName("cccv.go"), Line: Line{Number: 18, Text: "type FileName string"}},
		&Change{FileName: FileName("cccv.go"), Line: Line{Number: 22, Text: "FileName"}},
		&Change{FileName: FileName("cccv.go"), Line: Line{Number: 48, Text: "o.O(results)"}},
	}

	changes := getChanges(strings.NewReader(diff))
	Expect(*changes).To(Equal(expectedChanges))
}
<commit_msg>test finds duplicates<commit_after>package main

import (
	"log"
	"os"
	"strings"
	"testing"
	. "github.com/onsi/gomega"
)

const diff = `
diff --git a/README.md b/README.md
index c9caa23..8ad488f 100644
--- a/README.md
+++ b/README.md
@@ -7,4 +7,5 @@ Check if git diff (commit, pr) contains copy pasted code.

'''
 % go get github.com/artemave/cccv
+% go get github.com/artemave/cccv
 % git diff | cccv
diff --git a/cccv.go b/cccv.go
index af3d9dc..0e96324 100644
--- a/cccv.go
+++ b/cccv.go
@@ -15,9 +15,11 @@ import (
 )

 type FileName string
+type FileName string

 type Change struct {
 	FileName
+	FileName
 	Line
 }

@@ -43,6 +45,7 @@ func main() {
 	}

 	o.O(results)
+	o.O(results)
 }

 func GenResultForFile(fName string, changes *[]*Change) FileResult {
`

var expectedChanges = []*Change{
	&Change{FileName: FileName("README.md"), Line: Line{Number: 10, Text: "% go get github.com/artemave/cccv"}},
	&Change{FileName: FileName("cccv.go"), Line: Line{Number: 18, Text: "type FileName string"}},
	&Change{FileName: FileName("cccv.go"), Line: Line{Number: 22, Text: "FileName"}},
	&Change{FileName: FileName("cccv.go"), Line: Line{Number: 48, Text: "o.O(results)"}},
}

func TestParsesDiff(t *testing.T) {
	RegisterTestingT(t)

	changes := getChanges(strings.NewReader(diff))
	Expect(*changes).To(Equal(expectedChanges))
}

func TestFindsDuplicates(t *testing.T) {
	RegisterTestingT(t)

	WriteFile("/tmp/some_file.go", func(f *os.File) {
		f.WriteString("writes\n")
		f.WriteString("o.O(results)\n")
		f.WriteString("writes\n")
		f.WriteString("type FileName string\n")
		f.WriteString("writes\n")
	})

	expectedResult := FileResult{
		FileName: FileName("/tmp/some_file.go"),
		Lines: []*Line{
			&Line{Number: 2, Text: "o.O(results)"},
			&Line{Number: 4, Text: "type FileName string"},
		},
	}
	result := GenResultForFile("/tmp/some_file.go", &expectedChanges)
	Expect(result).To(Equal(expectedResult))
}

func WriteFile(fname string, callback func(f *os.File)) {
	f, err := os.Create(fname)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()
	callback(f)
	f.Sync()
}
<|endoftext|>


<commit_before>/*
Copyright 2022 The TestGrid Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"context"
	"errors"
	"flag"
	"runtime"
	"strings"
	"time"

	gpubsub "cloud.google.com/go/pubsub"
	"github.com/GoogleCloudPlatform/testgrid/pkg/pubsub"
	"github.com/GoogleCloudPlatform/testgrid/pkg/tabulator"
	"github.com/GoogleCloudPlatform/testgrid/util"
	"github.com/GoogleCloudPlatform/testgrid/util/gcs"
	"github.com/GoogleCloudPlatform/testgrid/util/metrics/prometheus"
	"github.com/sirupsen/logrus"
	"google.golang.org/api/option"
)

// options configures the updater
type options struct {
	config              gcs.Path // gs://path/to/config/proto
	persistQueue        gcs.Path
	creds               string
	confirm             bool
	filter              bool
	dropEmptyCols       bool
	useTabAlertSettings bool
	calculateStats      bool
	groups              util.Strings
	concurrency         int
	wait                time.Duration
	gridPathPrefix      string
	tabStatePathPrefix  string
	pubsub              string

	debug    bool
	trace    bool
	jsonLogs bool
}

// validate ensures reasonable options
func (o *options) validate() error {
	if o.config.String() == "" {
		return errors.New("empty --config")
	}
	if o.concurrency < 1 {
		o.concurrency = 4 * runtime.NumCPU()
	}

	return nil
}

// gatherOptions reads options from flags
func gatherOptions() options {
	var o options

	flag.Var(&o.config, "config", "gs://path/to/config.pb")
	flag.Var(&o.persistQueue, "persist-queue", "Load previous queue state from gs://path/to/queue-state.json and regularly save to it thereafter")
	flag.StringVar(&o.creds, "gcp-service-account", "", "/path/to/gcp/creds (use local creds if empty)")
	flag.BoolVar(&o.confirm, "confirm", false, "Upload data if set")
	flag.Var(&o.groups, "group", "Only update named test group if set (repeateable)")
	flag.BoolVar(&o.filter, "filter", false, "Filters out data according to the tab's configuration") // TODO(chases2): Switch to default
	flag.BoolVar(&o.dropEmptyCols, "filter-columns", false, "Drops empty columns after filtering")    // TODO(chases2): Enable, then remove flag
	flag.BoolVar(&o.useTabAlertSettings, "tab-alerts", false, "Use newer tab settings while caculating alerts")
	flag.BoolVar(&o.calculateStats, "column-stats", false, "Calculates stats for broken columns")
	flag.IntVar(&o.concurrency, "concurrency", 0, "Manually define the number of groups to concurrently update if non-zero")
	flag.DurationVar(&o.wait, "wait", 0, "Ensure at least this much time has passed since the last loop (exit if zero).")

	flag.StringVar(&o.gridPathPrefix, "grid-path", "grid", "Read grid states under this GCS path.")
	flag.StringVar(&o.tabStatePathPrefix, "tab-state-path", "tabs", "Write tab states under this GCS path.")
	flag.StringVar(&o.pubsub, "pubsub", "", "listen for test group updates at project/subscription")

	flag.BoolVar(&o.debug, "debug", false, "Log debug lines if set")
	flag.BoolVar(&o.trace, "trace", false, "Log trace and debug lines if set")
	flag.BoolVar(&o.jsonLogs, "json-logs", false, "Uses a json logrus formatter when set")

	flag.Parse()
	return o
}

func main() {
	opt := gatherOptions()
	if err := opt.validate(); err != nil {
		logrus.Fatalf("Invalid flags: %v", err)
	}
	if !opt.confirm {
		logrus.Warning("--confirm=false (DRY-RUN): will not write to gcs")
	}
	switch {
	case opt.trace:
		logrus.SetLevel(logrus.TraceLevel)
	case opt.debug:
		logrus.SetLevel(logrus.DebugLevel)
	}

	if opt.jsonLogs {
		logrus.SetFormatter(&logrus.JSONFormatter{})
	}
	logrus.SetReportCaller(true)

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	storageClient, err := gcs.ClientWithCreds(ctx, opt.creds)
	if err != nil {
		logrus.WithError(err).Fatal("Failed to create storage client")
	}
	defer storageClient.Close()

	client := gcs.NewClient(storageClient)

	logrus.WithFields(logrus.Fields{
		"group": opt.concurrency,
	}).Info("Configured concurrency")

	fixers := make([]tabulator.Fixer, 0, 2)

	fixer, err := gcsFixer(ctx, opt.pubsub, opt.config, opt.gridPathPrefix, opt.creds)
	if err != nil {
		logrus.WithError(err).WithField("subscription", opt.pubsub).Fatal("Failed to configure pubsub")
	}
	if fixer != nil {
		fixers = append(fixers, fixer)
	}
	if path := opt.persistQueue; path.String() != "" {
		const freq = time.Minute
		ticker := time.NewTicker(freq)
		log := logrus.WithField("frequency", freq)
		fixers = append(fixers, tabulator.FixPersistent(log, client, path, ticker.C))
	}

	mets := tabulator.CreateMetrics(prometheus.NewFactory())

	if err := tabulator.Update(ctx, client, mets, opt.config, opt.concurrency, opt.gridPathPrefix, opt.tabStatePathPrefix, opt.groups.Strings(), opt.confirm, opt.filter, opt.dropEmptyCols, opt.calculateStats, opt.useTabAlertSettings, opt.wait, fixers...); err != nil {
		logrus.WithError(err).Error("Could not tabulate")
	}
}

func gcsFixer(ctx context.Context, projectSub string, configPath gcs.Path, gridPrefix, credPath string) (tabulator.Fixer, error) {
	if projectSub == "" {
		return nil, nil
	}
	parts := strings.SplitN(projectSub, "/", 2)
	if len(parts) != 2 {
		return nil, errors.New("malformed project/subscription")
	}
	projID, subID := parts[0], parts[1]
	pubsubClient, err := gpubsub.NewClient(ctx, "", option.WithCredentialsFile(credPath))
	if err != nil {
		logrus.WithError(err).Fatal("Failed to create pubsub client")
	}
	client := pubsub.NewClient(pubsubClient)
	return tabulator.FixGCS(client, logrus.StandardLogger(), projID, subID, configPath, gridPrefix)
}
<commit_msg>Set filter default to true<commit_after>/*
Copyright 2022 The TestGrid Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"context"
	"errors"
	"flag"
	"runtime"
	"strings"
	"time"

	gpubsub "cloud.google.com/go/pubsub"
	"github.com/GoogleCloudPlatform/testgrid/pkg/pubsub"
	"github.com/GoogleCloudPlatform/testgrid/pkg/tabulator"
	"github.com/GoogleCloudPlatform/testgrid/util"
	"github.com/GoogleCloudPlatform/testgrid/util/gcs"
	"github.com/GoogleCloudPlatform/testgrid/util/metrics/prometheus"
	"github.com/sirupsen/logrus"
	"google.golang.org/api/option"
)

// options configures the updater
type options struct {
	config              gcs.Path // gs://path/to/config/proto
	persistQueue        gcs.Path
	creds               string
	confirm             bool
	filter              bool
	dropEmptyCols       bool
	useTabAlertSettings bool
	calculateStats      bool
	groups              util.Strings
	concurrency         int
	wait                time.Duration
	gridPathPrefix      string
	tabStatePathPrefix  string
	pubsub              string

	debug    bool
	trace    bool
	jsonLogs bool
}

// validate ensures reasonable options
func (o *options) validate() error {
	if o.config.String() == "" {
		return errors.New("empty --config")
	}
	if o.concurrency < 1 {
		o.concurrency = 4 * runtime.NumCPU()
	}

	return nil
}

// gatherOptions reads options from flags
func gatherOptions() options {
	var o options

	flag.Var(&o.config, "config", "gs://path/to/config.pb")
	flag.Var(&o.persistQueue, "persist-queue", "Load previous queue state from gs://path/to/queue-state.json and regularly save to it thereafter")
	flag.StringVar(&o.creds, "gcp-service-account", "", "/path/to/gcp/creds (use local creds if empty)")
	flag.BoolVar(&o.confirm, "confirm", false, "Upload data if set")
	flag.Var(&o.groups, "group", "Only update named test group if set (repeateable)")
	flag.BoolVar(&o.filter, "filter", true, "Filters out data according to the tab's configuration") // TODO: Delete
	flag.BoolVar(&o.dropEmptyCols, "filter-columns", false, "Drops empty columns after filtering")   // TODO(chases2): Enable, then remove flag
	flag.BoolVar(&o.useTabAlertSettings, "tab-alerts", false, "Use newer tab settings while caculating alerts")
	flag.BoolVar(&o.calculateStats, "column-stats", false, "Calculates stats for broken columns")
	flag.IntVar(&o.concurrency, "concurrency", 0, "Manually define the number of groups to concurrently update if non-zero")
	flag.DurationVar(&o.wait, "wait", 0, "Ensure at least this much time has passed since the last loop (exit if zero).")

	flag.StringVar(&o.gridPathPrefix, "grid-path", "grid", "Read grid states under this GCS path.")
	flag.StringVar(&o.tabStatePathPrefix, "tab-state-path", "tabs", "Write tab states under this GCS path.")
	flag.StringVar(&o.pubsub, "pubsub", "", "listen for test group updates at project/subscription")

	flag.BoolVar(&o.debug, "debug", false, "Log debug lines if set")
	flag.BoolVar(&o.trace, "trace", false, "Log trace and debug lines if set")
	flag.BoolVar(&o.jsonLogs, "json-logs", false, "Uses a json logrus formatter when set")

	flag.Parse()
	return o
}

func main() {
	opt := gatherOptions()
	if err := opt.validate(); err != nil {
		logrus.Fatalf("Invalid flags: %v", err)
	}
	if !opt.confirm {
		logrus.Warning("--confirm=false (DRY-RUN): will not write to gcs")
	}
	switch {
	case opt.trace:
		logrus.SetLevel(logrus.TraceLevel)
	case opt.debug:
		logrus.SetLevel(logrus.DebugLevel)
	}

	if opt.jsonLogs {
		logrus.SetFormatter(&logrus.JSONFormatter{})
	}
	logrus.SetReportCaller(true)

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	storageClient, err := gcs.ClientWithCreds(ctx, opt.creds)
	if err != nil {
		logrus.WithError(err).Fatal("Failed to create storage client")
	}
	defer storageClient.Close()

	client := gcs.NewClient(storageClient)

	logrus.WithFields(logrus.Fields{
		"group": opt.concurrency,
	}).Info("Configured concurrency")

	fixers := make([]tabulator.Fixer, 0, 2)

	fixer, err := gcsFixer(ctx, opt.pubsub, opt.config, opt.gridPathPrefix, opt.creds)
	if err != nil {
		logrus.WithError(err).WithField("subscription", opt.pubsub).Fatal("Failed to configure pubsub")
	}
	if fixer != nil {
		fixers = append(fixers, fixer)
	}
	if path := opt.persistQueue; path.String() != "" {
		const freq = time.Minute
		ticker := time.NewTicker(freq)
		log := logrus.WithField("frequency", freq)
		fixers = append(fixers, tabulator.FixPersistent(log, client, path, ticker.C))
	}

	mets := tabulator.CreateMetrics(prometheus.NewFactory())

	if err := tabulator.Update(ctx, client, mets, opt.config, opt.concurrency, opt.gridPathPrefix, opt.tabStatePathPrefix, opt.groups.Strings(), opt.confirm, opt.filter, opt.dropEmptyCols, opt.calculateStats, opt.useTabAlertSettings, opt.wait, fixers...); err != nil {
		logrus.WithError(err).Error("Could not tabulate")
	}
}

func gcsFixer(ctx context.Context, projectSub string, configPath gcs.Path, gridPrefix, credPath string) (tabulator.Fixer, error) {
	if projectSub == "" {
		return nil, nil
	}
	parts := strings.SplitN(projectSub, "/", 2)
	if len(parts) != 2 {
		return nil, errors.New("malformed project/subscription")
	}
	projID, subID := parts[0], parts[1]
	pubsubClient, err := gpubsub.NewClient(ctx, "", option.WithCredentialsFile(credPath))
	if err != nil {
		logrus.WithError(err).Fatal("Failed to create pubsub client")
	}
	client := pubsub.NewClient(pubsubClient)
	return tabulator.FixGCS(client, logrus.StandardLogger(), projID, subID, configPath, gridPrefix)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/nelhage/taktician/playtak"
	"github.com/nelhage/taktician/playtak/bot"
)

var (
	server    = flag.String("server", "playtak.com:10000", "playtak.com server to connect to")
	user      = flag.String("user", "", "username for login")
	pass      = flag.String("pass", "", "password for login")
	accept    = flag.String("accept", "", "accept a game from specified user")
	observe   = flag.String("observe", "", "observe a game by a specified user")
	gameTime  = flag.Duration("time", 20*time.Minute, "Length of game to offer")
	increment = flag.Duration("increment", 0, "time increment to offer")
	size      = flag.Int("size", 5, "size of game to offer")
	once      = flag.Bool("once", false, "play a single game and exit")
	takbot    = flag.String("takbot", "", "challenge TakBot AI")

	friendly = flag.Bool("friendly", false, "play as FriendlyBot")
	fpa      = flag.Bool("fpa", false, "play as FPABot")

	debug           = flag.Int("debug", 1, "debug level")
	depth           = flag.Int("depth", 5, "minimax depth")
	multicut        = flag.Bool("multi-cut", false, "use multi-cut")
	limit           = flag.Duration("limit", time.Minute, "time limit per move")
	sort            = flag.Bool("sort", true, "sort moves via history heuristic")
	tableMem        = flag.Int64("table-mem", 0, "set table size")
	useOpponentTime = flag.Bool("use-opponent-time", true, "think on opponent's time")

	debugClient = flag.Bool("debug-client", false, "log debug output for playtak connection")
)

const ClientName = "Taktician AI"

func main() {
	flag.Parse()
	if *accept != "" || *takbot != "" || *observe != "" {
		*once = true
	}

	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, syscall.SIGTERM, syscall.SIGINT)

	backoff := 1 * time.Second
	var b bot.Bot
	for {
		var client *playtak.Commands
		cl, err := playtak.Dial(*debugClient, *server)
		if err != nil {
			goto reconnect
		}
		backoff = time.Second
		client = &playtak.Commands{cl}
		client.SendClient(ClientName)
		if *user != "" {
			err = client.Login(*user, *pass)
		} else {
			err = client.LoginGuest()
		}
		if err != nil {
			log.Fatal("login: ", err)
		}
		log.Printf("login OK")
		if *friendly || *fpa {
			b = &Friendly{client: client, fpa: *fpa}
		} else {
			b = &Taktician{client: client}
		}
		for {
			if *accept == "" && *observe == "" {
				client.SendCommand("Seek",
					strconv.Itoa(*size),
					strconv.Itoa(int(gameTime.Seconds())),
					strconv.Itoa(int(increment.Seconds())))
				log.Printf("Seek OK")
				if *takbot != "" {
					client.SendCommand("Shout", "takbot: play", *takbot)
				}
			}

		recvLoop:
			for {
				select {
				case line, ok := <-client.Recv():
					if !ok {
						break recvLoop
					}
					switch {
					case strings.HasPrefix(line, "Seek new"):
						bits := strings.Split(line, " ")
						if bits[3] == *accept {
							log.Printf("accepting game %s from %s", bits[2], bits[3])
							client.SendCommand("Accept", bits[2])
						}
					case strings.HasPrefix(line, "Game Start"):
						bot.PlayGame(client, b, line)
						time.Sleep(100 * time.Millisecond)
						break recvLoop
					case strings.HasPrefix(line, "Shout"):
						who, msg := playtak.ParseShout(line)
						if who != "" {
							b.HandleChat("", who, msg)
						}
					case strings.HasPrefix(line, "GameList Add"):
						bits := strings.Split(line, " ")
						white := bits[3]
						black := strings.TrimRight(bits[5], ",")
						if white == *observe || black == *observe {
							client.SendCommand("Observe", bits[2][len("Game#"):])
						}
					case strings.HasPrefix(line, "Observe"):
						bot.ObserveGame(client, b, line)
					}
				case <-sigs:
					return
				}
			}
			if *once {
				return
			}
			if client.Error() != nil {
				log.Printf("Disconnected: %v", client.Error())
				break
			}
		}
	reconnect:
		log.Printf("sleeping %s before reconnect...", backoff)
		select {
		case <-time.After(backoff):
		case <-sigs:
			return
		}
		backoff = backoff * 2
		if backoff > time.Minute {
			backoff = time.Minute
		}
	}
}
<commit_msg>taktician: default unlimited depth<commit_after>package main

import (
	"flag"
	"log"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/nelhage/taktician/playtak"
	"github.com/nelhage/taktician/playtak/bot"
)

var (
	server    = flag.String("server", "playtak.com:10000", "playtak.com server to connect to")
	user      = flag.String("user", "", "username for login")
	pass      = flag.String("pass", "", "password for login")
	accept    = flag.String("accept", "", "accept a game from specified user")
	observe   = flag.String("observe", "", "observe a game by a specified user")
	gameTime  = flag.Duration("time", 20*time.Minute, "Length of game to offer")
	increment = flag.Duration("increment", 0, "time increment to offer")
	size      = flag.Int("size", 5, "size of game to offer")
	once      = flag.Bool("once", false, "play a single game and exit")
	takbot    = flag.String("takbot", "", "challenge TakBot AI")

	friendly = flag.Bool("friendly", false, "play as FriendlyBot")
	fpa      = flag.Bool("fpa", false, "play as FPABot")

	debug           = flag.Int("debug", 1, "debug level")
	depth           = flag.Int("depth", 0, "minimax depth")
	multicut        = flag.Bool("multi-cut", false, "use multi-cut")
	limit           = flag.Duration("limit", time.Minute, "time limit per move")
	sort            = flag.Bool("sort", true, "sort moves via history heuristic")
	tableMem        = flag.Int64("table-mem", 0, "set table size")
	useOpponentTime = flag.Bool("use-opponent-time", true, "think on opponent's time")

	debugClient = flag.Bool("debug-client", false, "log debug output for playtak connection")
)

const ClientName = "Taktician AI"

func main() {
	flag.Parse()
	if *accept != "" || *takbot != "" || *observe != "" {
		*once = true
	}

	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, syscall.SIGTERM, syscall.SIGINT)

	backoff := 1 * time.Second
	var b bot.Bot
	for {
		var client *playtak.Commands
		cl, err := playtak.Dial(*debugClient, *server)
		if err != nil {
			goto reconnect
		}
		backoff = time.Second
		client = &playtak.Commands{cl}
		client.SendClient(ClientName)
		if *user != "" {
			err = client.Login(*user, *pass)
		} else {
			err = client.LoginGuest()
		}
		if err != nil {
			log.Fatal("login: ", err)
		}
		log.Printf("login OK")
		if *friendly || *fpa {
			b = &Friendly{client: client, fpa: *fpa}
		} else {
			b = &Taktician{client: client}
		}
		for {
			if *accept == "" && *observe == "" {
				client.SendCommand("Seek",
					strconv.Itoa(*size),
					strconv.Itoa(int(gameTime.Seconds())),
					strconv.Itoa(int(increment.Seconds())))
				log.Printf("Seek OK")
				if *takbot != "" {
					client.SendCommand("Shout", "takbot: play", *takbot)
				}
			}

		recvLoop:
			for {
				select {
				case line, ok := <-client.Recv():
					if !ok {
						break recvLoop
					}
					switch {
					case strings.HasPrefix(line, "Seek new"):
						bits := strings.Split(line, " ")
						if bits[3] == *accept {
							log.Printf("accepting game %s from %s", bits[2], bits[3])
							client.SendCommand("Accept", bits[2])
						}
					case strings.HasPrefix(line, "Game Start"):
						bot.PlayGame(client, b, line)
						time.Sleep(100 * time.Millisecond)
						break recvLoop
					case strings.HasPrefix(line, "Shout"):
						who, msg := playtak.ParseShout(line)
						if who != "" {
							b.HandleChat("", who, msg)
						}
					case strings.HasPrefix(line, "GameList Add"):
						bits := strings.Split(line, " ")
						white := bits[3]
						black := strings.TrimRight(bits[5], ",")
						if white == *observe || black == *observe {
							client.SendCommand("Observe", bits[2][len("Game#"):])
						}
					case strings.HasPrefix(line, "Observe"):
						bot.ObserveGame(client, b, line)
					}
				case <-sigs:
					return
				}
			}
			if *once {
				return
			}
			if client.Error() != nil {
				log.Printf("Disconnected: %v", client.Error())
				break
			}
		}
	reconnect:
		log.Printf("sleeping %s before reconnect...", backoff)
		select {
		case <-time.After(backoff):
		case <-sigs:
			return
		}
		backoff = backoff * 2
		if backoff > time.Minute {
			backoff = time.Minute
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"

	"github.com/toomore/gogrs/utils"
)

var hCache = utils.NewHTTPCache(utils.GetOSRamdiskPath(), "utf8")
var flushall = flag.Bool("flushall", false, "Clear cache.")

func main() {
	flag.Parse()
	if flag.NFlag() == 0 {
		flag.PrintDefaults()
	}
	if *flushall {
		log.Println("Clear Cache:", hCache.Dir, utils.TempFolderName)
		hCache.FlushAll()
	}
}
<commit_msg>Add color.<commit_after>package main

import (
	"flag"
	"log"

	"github.com/fatih/color"
	"github.com/toomore/gogrs/utils"
)

var hCache = utils.NewHTTPCache(utils.GetOSRamdiskPath(), "utf8")
var flushall = flag.Bool("flushall", false, "Clear cache.")

func outputNote(note ...interface{}) {
	color.Set(color.FgBlue, color.Bold)
	log.Println(note...)
	color.Unset()
}

func outputDone(note ...interface{}) {
	color.Set(color.FgGreen, color.Bold)
	log.Println(note...)
	color.Unset()
}

func main() {
	flag.Parse()
	if flag.NFlag() == 0 {
		flag.PrintDefaults()
	}
	if *flushall {
		outputNote("Clear Cache:", hCache.Dir, utils.TempFolderName)
		hCache.FlushAll()
		outputDone("Done!")
	}
}
<|endoftext|>