<commit_before>package command

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/go-metrics"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/color"
	"github.com/funkygao/golib/gofmt"
	"github.com/funkygao/golib/signal"
)

var (
	stats *peekStats
)

type peekStats struct {
	MsgCountPerSecond metrics.Meter
	MsgBytesPerSecond metrics.Meter
}

func newPeekStats() *peekStats {
	this := &peekStats{
		MsgCountPerSecond: metrics.NewMeter(),
		MsgBytesPerSecond: metrics.NewMeter(),
	}

	metrics.Register("msg.count.per.second", this.MsgCountPerSecond)
	metrics.Register("msg.bytes.per.second", this.MsgBytesPerSecond)
	return this
}

func (this *peekStats) start() {
	metrics.Log(metrics.DefaultRegistry, time.Second*10,
		log.New(os.Stdout, "metrics: ", log.Lmicroseconds))
}

type Peek struct {
	Ui  cli.Ui
	Cmd string

	offset   int64
	lastN    int64 // peek the most recent N messages
	colorize bool
	limit    int
	quit     chan struct{}
	once     sync.Once
	column   string
	beep     bool
	pretty   bool
	bodyOnly bool
}

func (this *Peek) Run(args []string) (exitCode int) {
	var (
		cluster      string
		zone         string
		topicPattern string
		partitionId  int
		wait         time.Duration
		silence      bool
	)
	cmdFlags := flag.NewFlagSet("peek", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.StringVar(&cluster, "c", "", "")
	cmdFlags.StringVar(&topicPattern, "t", "", "")
	cmdFlags.IntVar(&partitionId, "p", 0, "")
	cmdFlags.BoolVar(&this.colorize, "color", true, "")
	cmdFlags.Int64Var(&this.lastN, "last", -1, "")
	cmdFlags.BoolVar(&this.pretty, "pretty", false, "")
	cmdFlags.IntVar(&this.limit, "n", -1, "")
	cmdFlags.StringVar(&this.column, "col", "", "") // TODO support multiple columns
	cmdFlags.BoolVar(&this.beep, "beep", false, "")
	cmdFlags.Int64Var(&this.offset, "offset", sarama.OffsetNewest, "")
	cmdFlags.BoolVar(&silence, "s", false, "")
	cmdFlags.DurationVar(&wait, "d", time.Hour, "")
	cmdFlags.BoolVar(&this.bodyOnly, "body", false, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if this.pretty {
		this.bodyOnly = true
	}

	this.quit = make(chan struct{})

	if silence {
		stats := newPeekStats()
		go stats.start()
	}

	zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
	msgChan := make(chan *sarama.ConsumerMessage, 20000) // msg aggerator channel
	if cluster == "" {
		zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
			this.consumeCluster(zkcluster, topicPattern, partitionId, msgChan)
		})
	} else {
		zkcluster := zkzone.NewCluster(cluster)
		this.consumeCluster(zkcluster, topicPattern, partitionId, msgChan)
	}

	signal.RegisterHandler(func(sig os.Signal) {
		log.Printf("received signal: %s", strings.ToUpper(sig.String()))
		log.Println("quiting...")

		this.once.Do(func() {
			close(this.quit)
		})
	}, syscall.SIGINT, syscall.SIGTERM)

	var (
		startAt = time.Now()
		msg     *sarama.ConsumerMessage
		total   int
		bytesN  int64
	)

	var (
		j          map[string]interface{}
		prettyJSON bytes.Buffer
	)

LOOP:
	for {
		if time.Since(startAt) >= wait {
			this.Ui.Output(fmt.Sprintf("Total: %s msgs, %s, elapsed: %s",
				gofmt.Comma(int64(total)), gofmt.ByteSize(bytesN), time.Since(startAt)))
			elapsed := time.Since(startAt).Seconds()
			if elapsed > 1. {
				this.Ui.Output(fmt.Sprintf("Speed: %d/s", total/int(elapsed)))
				if total > 0 {
					this.Ui.Output(fmt.Sprintf("Size : %s/msg", gofmt.ByteSize(bytesN/int64(total))))
				}
			}

			return
		}

		select {
		case <-this.quit:
			this.Ui.Output(fmt.Sprintf("Total: %s msgs, %s, elapsed: %s",
				gofmt.Comma(int64(total)), gofmt.ByteSize(bytesN), time.Since(startAt)))
			elapsed := time.Since(startAt).Seconds()
			if elapsed > 1. {
				this.Ui.Output(fmt.Sprintf("Speed: %d/s", total/int(elapsed)))
				if total > 0 {
					this.Ui.Output(fmt.Sprintf("Size : %s/msg", gofmt.ByteSize(bytesN/int64(total))))
				}
			}

			return

		case <-time.After(time.Second):
			continue

		case msg = <-msgChan:
			if silence {
				stats.MsgCountPerSecond.Mark(1)
				stats.MsgBytesPerSecond.Mark(int64(len(msg.Value)))
			} else {
				var outmsg string
				if this.column != "" {
					if err := json.Unmarshal(msg.Value, &j); err != nil {
						this.Ui.Error(err.Error())
					} else {
						var colVal string
						switch t := j[this.column].(type) {
						case string:
							colVal = t
						case float64:
							colVal = fmt.Sprintf("%.0f", t)
						case int:
							colVal = fmt.Sprintf("%d", t)
						}

						if this.bodyOnly {
							if this.pretty {
								if err = json.Indent(&prettyJSON, []byte(colVal), "", "    "); err != nil {
									fmt.Println(err.Error())
								} else {
									outmsg = string(prettyJSON.Bytes())
								}
							} else {
								outmsg = colVal
							}
						} else if this.colorize {
							outmsg = fmt.Sprintf("%s/%d %s k:%s v:%s",
								color.Green(msg.Topic), msg.Partition,
								gofmt.Comma(msg.Offset), string(msg.Key), colVal)
						} else {
							// colored UI will have invisible chars output
							outmsg = fmt.Sprintf("%s/%d %s k:%s v:%s",
								msg.Topic, msg.Partition,
								gofmt.Comma(msg.Offset), string(msg.Key), colVal)
						}
					}

				} else {
					if this.bodyOnly {
						if this.pretty {
							json.Indent(&prettyJSON, msg.Value, "", "    ")
							outmsg = string(prettyJSON.Bytes())
						} else {
							outmsg = string(msg.Value)
						}
					} else if this.colorize {
						outmsg = fmt.Sprintf("%s/%d %s k:%s, v:%s",
							color.Green(msg.Topic), msg.Partition,
							gofmt.Comma(msg.Offset), string(msg.Key), string(msg.Value))
					} else {
						// colored UI will have invisible chars output
						outmsg = fmt.Sprintf("%s/%d %s k:%s, v:%s",
							msg.Topic, msg.Partition,
							gofmt.Comma(msg.Offset), string(msg.Key), string(msg.Value))
					}
				}

				if outmsg != "" {
					if this.beep {
						outmsg += "\a"
					}

					this.Ui.Output(outmsg)
				}
			}

			total++
			bytesN += int64(len(msg.Value))

			if this.limit > 0 && total >= this.limit {
				break LOOP
			}
			if this.lastN > 0 && total >= int(this.lastN) {
				break LOOP
			}

		}
	}

	return
}

func (this *Peek) consumeCluster(zkcluster *zk.ZkCluster, topicPattern string,
	partitionId int, msgChan chan *sarama.ConsumerMessage) {
	brokerList := zkcluster.BrokerList()
	if len(brokerList) == 0 {
		return
	}
	kfk, err := sarama.NewClient(brokerList, sarama.NewConfig())
	if err != nil {
		this.Ui.Output(err.Error())
		return
	}
	//defer kfk.Close() // FIXME how to close it

	topics, err := kfk.Topics()
	if err != nil {
		this.Ui.Output(err.Error())
		return
	}

	for _, t := range topics {
		if patternMatched(t, topicPattern) {
			go this.simpleConsumeTopic(zkcluster, kfk, t, int32(partitionId), msgChan)
		}
	}

}

func (this *Peek) simpleConsumeTopic(zkcluster *zk.ZkCluster, kfk sarama.Client, topic string, partitionId int32,
	msgCh chan *sarama.ConsumerMessage) {
	consumer, err := sarama.NewConsumerFromClient(kfk)
	if err != nil {
		panic(err)
	}
	defer consumer.Close()

	if partitionId == -1 {
		// all partitions
		partitions, err := kfk.Partitions(topic)
		if err != nil {
			panic(err)
		}

		for _, p := range partitions {
			offset := this.offset
			if this.lastN > 0 {
				latestOffset, err := kfk.GetOffset(topic, p, sarama.OffsetNewest)
				swallow(err)

				oldestOffset, err := kfk.GetOffset(topic, p, sarama.OffsetOldest)
				swallow(err)

				offset = latestOffset - this.lastN
				if offset < oldestOffset {
					offset = oldestOffset
				}

				if offset == 0 {
					// no message in store
					return
				}
			}

			go this.consumePartition(zkcluster, kfk, consumer, topic, p, msgCh, offset)
		}

	} else {
		offset := this.offset
		if this.lastN > 0 {
			latestOffset, err := kfk.GetOffset(topic, partitionId, sarama.OffsetNewest)
			swallow(err)
			offset = latestOffset - this.lastN
			if offset < 0 {
				offset = sarama.OffsetOldest
			}
		}
		this.consumePartition(zkcluster, kfk, consumer, topic, partitionId, msgCh, offset)
	}

}

func (this *Peek) consumePartition(zkcluster *zk.ZkCluster, kfk sarama.Client, consumer sarama.Consumer,
	topic string, partitionId int32, msgCh chan *sarama.ConsumerMessage, offset int64) {
	p, err := consumer.ConsumePartition(topic, partitionId, offset)
	if err != nil {
		this.Ui.Error(fmt.Sprintf("%s %s/%d: offset=%d %v", zkcluster.Name(), topic, partitionId, offset, err))
		return
	}
	defer p.Close()

	n := int64(0)
	for {
		select {
		case <-this.quit:
			return

		case msg := <-p.Messages():
			msgCh <- msg

			n++
			if this.lastN > 0 && n >= this.lastN {
				return
			}
		}
	}
}

func (*Peek) Synopsis() string {
	return "Peek kafka cluster messages ongoing from any offset"
}

func (this *Peek) Help() string {
	help := fmt.Sprintf(`
Usage: %s peek [options]

    %s

Options:

    -z zone
      Default %s

    -c cluster

    -t topic pattern

    -p partition id
      -1 will peek all partitions of a topic

    -beep
      Make a beep sound for each message

    -pretty
      Pretty print the json message body

    -col json column name
      Will json decode message and extract specified column value only

    -last n
      Peek the most recent N messages

    -offset message offset value
      -1 OffsetNewest, -2 OffsetOldest.
      You can specify your own offset.
      Default -1(OffsetNewest)

    -n count
      Limit how many messages to consume

    -d duration
      Limit how long to keep peeking
      e,g. -d 5m

    -body
      Only display message body

    -s
      Silence mode, only display statastics instead of message content

    -color
      Enable colorized output
`, this.Cmd, this.Synopsis(), ctx.ZkDefaultZone())
	return strings.TrimSpace(help)
}
<commit_msg>peek till end of stream<commit_after>package command

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/go-metrics"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/color"
	"github.com/funkygao/golib/gofmt"
	"github.com/funkygao/golib/signal"
)

var (
	stats *peekStats
)

type peekStats struct {
	MsgCountPerSecond metrics.Meter
	MsgBytesPerSecond metrics.Meter
}

func newPeekStats() *peekStats {
	this := &peekStats{
		MsgCountPerSecond: metrics.NewMeter(),
		MsgBytesPerSecond: metrics.NewMeter(),
	}

	metrics.Register("msg.count.per.second", this.MsgCountPerSecond)
	metrics.Register("msg.bytes.per.second", this.MsgBytesPerSecond)
	return this
}

func (this *peekStats) start() {
	metrics.Log(metrics.DefaultRegistry, time.Second*10,
		log.New(os.Stdout, "metrics: ", log.Lmicroseconds))
}

type Peek struct {
	Ui  cli.Ui
	Cmd string

	offset   int64
	lastN    int64 // peek the most recent N messages
	colorize bool
	limit    int
	quit     chan struct{}
	once     sync.Once
	column   string
	beep     bool
	pretty   bool
	bodyOnly bool
}

func (this *Peek) Run(args []string) (exitCode int) {
	var (
		cluster      string
		zone         string
		topicPattern string
		partitionId  int
		wait         time.Duration
		tillNow      bool
		silence      bool
	)
	cmdFlags := flag.NewFlagSet("peek", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.StringVar(&cluster, "c", "", "")
	cmdFlags.StringVar(&topicPattern, "t", "", "")
	cmdFlags.IntVar(&partitionId, "p", 0, "")
	cmdFlags.BoolVar(&this.colorize, "color", true, "")
	cmdFlags.Int64Var(&this.lastN, "last", -1, "")
	cmdFlags.BoolVar(&this.pretty, "pretty", false, "")
	cmdFlags.IntVar(&this.limit, "n", -1, "")
	cmdFlags.StringVar(&this.column, "col", "", "") // TODO support multiple columns
	cmdFlags.BoolVar(&this.beep, "beep", false, "")
	cmdFlags.Int64Var(&this.offset, "offset", sarama.OffsetNewest, "")
	cmdFlags.BoolVar(&silence, "s", false, "")
	cmdFlags.DurationVar(&wait, "d", time.Hour, "")
	cmdFlags.BoolVar(&tillNow, "now", false, "")
	cmdFlags.BoolVar(&this.bodyOnly, "body", false, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if this.pretty {
		this.bodyOnly = true
	}

	this.quit = make(chan struct{})

	if silence {
		stats := newPeekStats()
		go stats.start()
	}

	zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
	msgChan := make(chan *sarama.ConsumerMessage, 20000) // msg aggerator channel
	if cluster == "" {
		zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
			this.consumeCluster(zkcluster, topicPattern, partitionId, msgChan)
		})
	} else {
		zkcluster := zkzone.NewCluster(cluster)
		this.consumeCluster(zkcluster, topicPattern, partitionId, msgChan)
	}

	signal.RegisterHandler(func(sig os.Signal) {
		log.Printf("received signal: %s", strings.ToUpper(sig.String()))
		log.Println("quiting...")

		this.once.Do(func() {
			close(this.quit)
		})
	}, syscall.SIGINT, syscall.SIGTERM)

	var (
		startAt = time.Now()
		msg     *sarama.ConsumerMessage
		total   int
		bytesN  int64
	)

	var (
		j          map[string]interface{}
		prettyJSON bytes.Buffer
	)

LOOP:
	for {
		if time.Since(startAt) >= wait {
			this.Ui.Output(fmt.Sprintf("Total: %s msgs, %s, elapsed: %s",
				gofmt.Comma(int64(total)), gofmt.ByteSize(bytesN), time.Since(startAt)))
			elapsed := time.Since(startAt).Seconds()
			if elapsed > 1. {
				this.Ui.Output(fmt.Sprintf("Speed: %d/s", total/int(elapsed)))
				if total > 0 {
					this.Ui.Output(fmt.Sprintf("Size : %s/msg", gofmt.ByteSize(bytesN/int64(total))))
				}
			}

			return
		}

		select {
		case <-this.quit:
			this.Ui.Output(fmt.Sprintf("Total: %s msgs, %s, elapsed: %s",
				gofmt.Comma(int64(total)), gofmt.ByteSize(bytesN), time.Since(startAt)))
			elapsed := time.Since(startAt).Seconds()
			if elapsed > 1. {
				this.Ui.Output(fmt.Sprintf("Speed: %d/s", total/int(elapsed)))
				if total > 0 {
					this.Ui.Output(fmt.Sprintf("Size : %s/msg", gofmt.ByteSize(bytesN/int64(total))))
				}
			}

			return

		case <-time.After(time.Second):
			if tillNow {
				return
			} else {
				continue
			}

		case msg = <-msgChan:
			if silence {
				stats.MsgCountPerSecond.Mark(1)
				stats.MsgBytesPerSecond.Mark(int64(len(msg.Value)))
			} else {
				var outmsg string
				if this.column != "" {
					if err := json.Unmarshal(msg.Value, &j); err != nil {
						this.Ui.Error(err.Error())
					} else {
						var colVal string
						switch t := j[this.column].(type) {
						case string:
							colVal = t
						case float64:
							colVal = fmt.Sprintf("%.0f", t)
						case int:
							colVal = fmt.Sprintf("%d", t)
						}

						if this.bodyOnly {
							if this.pretty {
								if err = json.Indent(&prettyJSON, []byte(colVal), "", "    "); err != nil {
									fmt.Println(err.Error())
								} else {
									outmsg = string(prettyJSON.Bytes())
								}
							} else {
								outmsg = colVal
							}
						} else if this.colorize {
							outmsg = fmt.Sprintf("%s/%d %s k:%s v:%s",
								color.Green(msg.Topic), msg.Partition,
								gofmt.Comma(msg.Offset), string(msg.Key), colVal)
						} else {
							// colored UI will have invisible chars output
							outmsg = fmt.Sprintf("%s/%d %s k:%s v:%s",
								msg.Topic, msg.Partition,
								gofmt.Comma(msg.Offset), string(msg.Key), colVal)
						}
					}

				} else {
					if this.bodyOnly {
						if this.pretty {
							json.Indent(&prettyJSON, msg.Value, "", "    ")
							outmsg = string(prettyJSON.Bytes())
						} else {
							outmsg = string(msg.Value)
						}
					} else if this.colorize {
						outmsg = fmt.Sprintf("%s/%d %s k:%s, v:%s",
							color.Green(msg.Topic), msg.Partition,
							gofmt.Comma(msg.Offset), string(msg.Key), string(msg.Value))
					} else {
						// colored UI will have invisible chars output
						outmsg = fmt.Sprintf("%s/%d %s k:%s, v:%s",
							msg.Topic, msg.Partition,
							gofmt.Comma(msg.Offset), string(msg.Key), string(msg.Value))
					}
				}

				if outmsg != "" {
					if this.beep {
						outmsg += "\a"
					}

					this.Ui.Output(outmsg)
				}
			}

			total++
			bytesN += int64(len(msg.Value))

			if this.limit > 0 && total >= this.limit {
				break LOOP
			}
			if this.lastN > 0 && total >= int(this.lastN) {
				break LOOP
			}

		}
	}

	return
}

func (this *Peek) consumeCluster(zkcluster *zk.ZkCluster, topicPattern string,
	partitionId int, msgChan chan *sarama.ConsumerMessage) {
	brokerList := zkcluster.BrokerList()
	if len(brokerList) == 0 {
		return
	}
	kfk, err := sarama.NewClient(brokerList, sarama.NewConfig())
	if err != nil {
		this.Ui.Output(err.Error())
		return
	}
	//defer kfk.Close() // FIXME how to close it

	topics, err := kfk.Topics()
	if err != nil {
		this.Ui.Output(err.Error())
		return
	}

	for _, t := range topics {
		if patternMatched(t, topicPattern) {
			go this.simpleConsumeTopic(zkcluster, kfk, t, int32(partitionId), msgChan)
		}
	}

}

func (this *Peek) simpleConsumeTopic(zkcluster *zk.ZkCluster, kfk sarama.Client, topic string, partitionId int32,
	msgCh chan *sarama.ConsumerMessage) {
	consumer, err := sarama.NewConsumerFromClient(kfk)
	if err != nil {
		panic(err)
	}
	defer consumer.Close()

	if partitionId == -1 {
		// all partitions
		partitions, err := kfk.Partitions(topic)
		if err != nil {
			panic(err)
		}

		for _, p := range partitions {
			offset := this.offset
			if this.lastN > 0 {
				latestOffset, err := kfk.GetOffset(topic, p, sarama.OffsetNewest)
				swallow(err)

				oldestOffset, err := kfk.GetOffset(topic, p, sarama.OffsetOldest)
				swallow(err)

				offset = latestOffset - this.lastN
				if offset < oldestOffset {
					offset = oldestOffset
				}

				if offset == 0 {
					// no message in store
					return
				}
			}

			go this.consumePartition(zkcluster, kfk, consumer, topic, p, msgCh, offset)
		}

	} else {
		offset := this.offset
		if this.lastN > 0 {
			latestOffset, err := kfk.GetOffset(topic, partitionId, sarama.OffsetNewest)
			swallow(err)
			offset = latestOffset - this.lastN
			if offset < 0 {
				offset = sarama.OffsetOldest
			}
		}
		this.consumePartition(zkcluster, kfk, consumer, topic, partitionId, msgCh, offset)
	}

}

func (this *Peek) consumePartition(zkcluster *zk.ZkCluster, kfk sarama.Client, consumer sarama.Consumer,
	topic string, partitionId int32, msgCh chan *sarama.ConsumerMessage, offset int64) {
	p, err := consumer.ConsumePartition(topic, partitionId, offset)
	if err != nil {
		this.Ui.Error(fmt.Sprintf("%s %s/%d: offset=%d %v", zkcluster.Name(), topic, partitionId, offset, err))
		return
	}
	defer p.Close()

	n := int64(0)
	for {
		select {
		case <-this.quit:
			return

		case msg := <-p.Messages():
			msgCh <- msg

			n++
			if this.lastN > 0 && n >= this.lastN {
				return
			}
		}
	}
}

func (*Peek) Synopsis() string {
	return "Peek kafka cluster messages ongoing from any offset"
}

func (this *Peek) Help() string {
	help := fmt.Sprintf(`
Usage: %s peek [options]

    %s

Options:

    -z zone
      Default %s

    -c cluster

    -t topic pattern

    -p partition id
      -1 will peek all partitions of a topic

    -beep
      Make a beep sound for each message

    -pretty
      Pretty print the json message body

    -col json column name
      Will json decode message and extract specified column value only

    -last n
      Peek the most recent N messages

    -offset message offset value
      -1 OffsetNewest, -2 OffsetOldest.
      You can specify your own offset.
      Default -1(OffsetNewest)

    -n count
      Limit how many messages to consume

    -d duration
      Limit how long to keep peeking
      e,g. -d 5m

    -body
      Only display message body

    -now
      Iterate the stream till now

    -s
      Silence mode, only display statastics instead of message content

    -color
      Enable colorized output
`, this.Cmd, this.Synopsis(), ctx.ZkDefaultZone())
	return strings.TrimSpace(help)
}
<|endoftext|>


<commit_before>package command

import (
	"flag"
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/funkygao/gocli"
)

type Time struct {
	Ui  cli.Ui
	Cmd string
}

func (this *Time) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("time", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	if err := cmdFlags.Parse(args); err != nil {
		return 2
	}

	if len(args) == 0 {
		this.Ui.Error("missing <timestamp>")
		return 2
	}

	timestamp := args[len(args)-1]
	this.Ui.Output(fmt.Sprintf("%s %s", timestamp, this.timestampToTime(timestamp)))

	return
}

func (this *Time) timestampToTime(t string) time.Time {
	i, err := strconv.ParseInt(t, 10, 64)
	swallow(err)

	if i > 1469686979 {
		// in ms
		i = i / 1000
	}
	return time.Unix(i, 0)
}

func (*Time) Synopsis() string {
	return "Parse Unix timestamp to human readable time"
}

func (this *Time) Help() string {
	help := fmt.Sprintf(`
Usage: %s time <timestamp>

    %s

`, this.Cmd, this.Synopsis())
	return strings.TrimSpace(help)
}
<commit_msg>fix 'gk time' bug<commit_after>package command

import (
	"flag"
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/funkygao/gocli"
)

type Time struct {
	Ui  cli.Ui
	Cmd string
}

func (this *Time) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("time", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	if err := cmdFlags.Parse(args); err != nil {
		return 2
	}

	if len(args) == 0 {
		this.Ui.Error("missing <timestamp>")
		return 2
	}

	timestamp := args[len(args)-1]
	this.Ui.Output(fmt.Sprintf("%s %s", timestamp, this.timestampToTime(timestamp)))

	return
}

func (this *Time) timestampToTime(t string) time.Time {
	i, err := strconv.ParseInt(t, 10, 64)
	swallow(err)

	if i > 133761237100 {
		// in ms
		i = i / 1000
	}
	return time.Unix(i, 0)
}

func (*Time) Synopsis() string {
	return "Parse Unix timestamp to human readable time"
}

func (this *Time) Help() string {
	help := fmt.Sprintf(`
Usage: %s time <timestamp>

    %s

`, this.Cmd, this.Synopsis())
	return strings.TrimSpace(help)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
	"os"

	"github.com/ernesto-jimenez/gogen/automock"
)

var (
	out      = flag.String("o", "", "what file to write")
	mockName = flag.String("mock-name", "", "name for the mock")
	mockPkg  = flag.String("mock-pkg", "", "package name for the mock")
	pkg      = flag.String("pkg", ".", "what package to get the interface from")
	inPkg    = flag.Bool("in-pkg", false, "whether the mock is internal to the package")
)

func main() {
	flag.Parse()
	log.SetFlags(0)

	iface := flag.Arg(0)

	if iface == "" {
		log.Fatal("need to specify an interface name")
	}

	gen, err := automock.NewGenerator(*pkg, iface)
	if err != nil {
		log.Fatal(err)
	}

	if *mockName != "" {
		gen.SetName(*mockName)
	}
	if *mockPkg != "" {
		gen.SetPackage(*mockPkg)
	}
	if *pkg == "." {
		*inPkg = true
	}
	gen.SetInternal(*inPkg)

	w := os.Stdout
	if *out != "" {
		w, err = os.OpenFile(*out, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0666)
		if err != nil {
			log.Fatal(err)
		}
	}

	err = gen.Write(w)
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>goautomock prints info when writing mock to a file<commit_after>package main

import (
	"flag"
	"log"
	"os"

	"github.com/ernesto-jimenez/gogen/automock"
)

var (
	out      = flag.String("o", "", "what file to write")
	mockName = flag.String("mock-name", "", "name for the mock")
	mockPkg  = flag.String("mock-pkg", "", "package name for the mock")
	pkg      = flag.String("pkg", ".", "what package to get the interface from")
	inPkg    = flag.Bool("in-pkg", false, "whether the mock is internal to the package")
)

func main() {
	flag.Parse()
	log.SetFlags(0)

	iface := flag.Arg(0)

	if iface == "" {
		log.Fatal("need to specify an interface name")
	}

	gen, err := automock.NewGenerator(*pkg, iface)
	if err != nil {
		log.Fatal(err)
	}

	if *mockName != "" {
		gen.SetName(*mockName)
	}
	if *mockPkg != "" {
		gen.SetPackage(*mockPkg)
	}
	if *pkg == "." {
		*inPkg = true
	}
	gen.SetInternal(*inPkg)

	w := os.Stdout
	if *out != "" {
		log.Printf("Generating mock for %s in %s", iface, *out)
		w, err = os.OpenFile(*out, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0666)
		if err != nil {
			log.Fatal(err)
		}
	}

	err = gen.Write(w)
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"html/template"
	"io"
	"log"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"os/exec"
	"os/signal"
	"path"
	"strings"

	"github.com/steder/gophernaut"
)

// Event is basically just an enum
type Event int

// Events that can be generated by our child processes
const (
	Start Event = iota
	Shutdown
	PiningForTheFjords
)

// TODO look into "go generate stringer -type Event"
func (e Event) String() string {
	return fmt.Sprintf("Event(%d)", e)
}

var hostnames = []string{
	fmt.Sprintf("http://127.0.0.1:%d", 8080),
	fmt.Sprintf("http://127.0.0.1:%d", 8081),
}

var executables = []string{
	fmt.Sprintf("python -m SimpleHTTPServer %d", 8080),
	fmt.Sprintf("python -m SimpleHTTPServer %d", 8081),
}

func copyToLog(dst *log.Logger, src io.Reader) {
	scanner := bufio.NewScanner(src)
	for scanner.Scan() {
		dst.Print(scanner.Text())
	}
}

func startProcess(control <-chan Event, events chan<- Event, executable string) {
	procLog := log.New(os.Stdout, fmt.Sprintf("gopher-worker(%s) ", executable), log.Ldate|log.Ltime)

	commandParts := strings.Split(executable, " ")
	command := exec.Command(commandParts[0], commandParts[1:]...)
	log.Printf("Command: %v\n", command)

	stdout, err := command.StdoutPipe()
	if err != nil {
		procLog.Fatalln("Unable to connect to stdout from command...")
	}
	stderr, err := command.StderrPipe()
	if err != nil {
		procLog.Fatalln("Unable to connect to stderr from command...")
	}

	//go io.Copy(os.Stdout, stdout)
	//go io.Copy(os.Stderr, stderr)
	go copyToLog(procLog, stdout)
	go copyToLog(procLog, stderr)
	command.Start()

	events <- Start
	for {
		_, ok := <-control
		if !ok {
			fmt.Println("Killing worker process after receiving close event.")
			command.Process.Kill()
			events <- Shutdown
			break
		}
	}
}

var requestCount = 0

func myHandler(w http.ResponseWriter, myReq *http.Request) {
	requestPath := myReq.URL.Path
	// TODO: multiprocess, pick one of n hostnames based on pool status
	hostname := hostnames[requestCount%len(hostnames)]
	requestCount++
	targetURL, _ := url.Parse(hostname)
	director := func(req *http.Request) {
		targetQuery := targetURL.RawQuery
		req.URL.Scheme = targetURL.Scheme
		// TODO: adjust request host to assign the request to the appropriate child process
		req.URL.Host = targetURL.Host

		// clean up but preserve trailing slash:
		trailing := strings.HasSuffix(req.URL.Path, "/")
		req.URL.Path = path.Join(targetURL.Path, req.URL.Path)
		if trailing && !strings.HasSuffix(req.URL.Path, "/") {
			req.URL.Path += "/"
		}

		// preserve query string:
		if targetQuery == "" || req.URL.RawQuery == "" {
			req.URL.RawQuery = targetQuery + req.URL.RawQuery
		} else {
			req.URL.RawQuery = targetQuery + "&" + req.URL.RawQuery
		}
	}

	proxy := &httputil.ReverseProxy{Director: director}

	staticHandler := http.StripPrefix("/static", http.FileServer(http.Dir("static")))
	adminTemplate := template.Must(template.ParseFiles("templates/admin.html"))
	adminHandler := func(w http.ResponseWriter, req *http.Request) {
		adminTemplate.Execute(w, nil)
	}

	switch {
	case requestPath == "/admin":
		adminHandler(w, myReq)
		return
	case strings.HasPrefix(requestPath, "/static"):
		staticHandler.ServeHTTP(w, myReq)
		return
	}
	proxy.ServeHTTP(w, myReq)
}

func main() {
	log.SetPrefix("gophernaut ")
	log.SetFlags(log.Ldate | log.Ltime)
	c := gophernaut.ReadConfig()
	log.Printf("Host %s and Port %d\n", c.Host, c.Port)

	controlChannel := make(chan Event)
	eventsChannel := make(chan Event)

	// Handle signals to try to do a graceful shutdown:
	receivedSignals := make(chan os.Signal, 1)
	signal.Notify(receivedSignals, os.Interrupt) // , syscall.SIGINT, syscall.SIGTERM)
	go func() {
		for sig := range receivedSignals {
			fmt.Printf("Received signal, %s, shutting down workers...\n", sig)
			break
		}
		close(controlChannel)
		signal.Stop(receivedSignals)
	}()

	// Actually start some processes
	for _, executable := range executables {
		go startProcess(controlChannel, eventsChannel, executable)
	}

	// wait for child processes to exit before shutting down:

	processCount := 0
	stoppedCount := 0
	go func() {
		// TODO: turn this into a ProcessPool?
		for event := range eventsChannel {
			switch event {
			case Shutdown:
				stoppedCount++
			case Start:
				processCount++
			}
			if processCount == stoppedCount {
				fmt.Printf("%d workers stopped, shutting down.\n", processCount)
				os.Exit(1)
			}
		}
	}()

	log.Printf("Gophernaut is gopher launch!\n")
	// TODO: our own ReverseProxy implementation of at least, ServeHTTP so that we can
	// monitor the response codes to track successes and failures
	log.Fatal(http.ListenAndServe(":8483", http.HandlerFunc(myHandler)))
}
<commit_msg>Everybody git in the pool<commit_after>package main

import (
	"bufio"
	"fmt"
	"html/template"
	"io"
	"log"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"os/exec"
	"os/signal"
	"path"
	"strings"

	"github.com/steder/gophernaut"
)

// Event is basically just an enum
type Event int

// Events that can be generated by our child processes
const (
	Start Event = iota
	Shutdown
	PiningForTheFjords
)

// TODO look into "go generate stringer -type Event"
func (e Event) String() string {
	return fmt.Sprintf("Event(%d)", e)
}

var hostnames = []string{
	fmt.Sprintf("http://127.0.0.1:%d", 8080),
	fmt.Sprintf("http://127.0.0.1:%d", 8081),
}

var executables = []string{
	fmt.Sprintf("python -m SimpleHTTPServer %d", 8080),
	fmt.Sprintf("python -m SimpleHTTPServer %d", 8081),
}

func copyToLog(dst *log.Logger, src io.Reader) {
	scanner := bufio.NewScanner(src)
	for scanner.Scan() {
		dst.Print(scanner.Text())
	}
}

func startProcess(control <-chan Event, events chan<- Event, executable string) {
	procLog := log.New(os.Stdout, fmt.Sprintf("gopher-worker(%s) ", executable), log.Ldate|log.Ltime)

	commandParts := strings.Split(executable, " ")
	command := exec.Command(commandParts[0], commandParts[1:]...)
	log.Printf("Command: %v\n", command)

	stdout, err := command.StdoutPipe()
	if err != nil {
		procLog.Fatalln("Unable to connect to stdout from command...")
	}
	stderr, err := command.StderrPipe()
	if err != nil {
		procLog.Fatalln("Unable to connect to stderr from command...")
	}

	//go io.Copy(os.Stdout, stdout)
	//go io.Copy(os.Stderr, stderr)
	go copyToLog(procLog, stdout)
	go copyToLog(procLog, stderr)
	command.Start()

	events <- Start
	for {
		_, ok := <-control
		if !ok {
			fmt.Println("Killing worker process after receiving close event.")
			command.Process.Kill()
			events <- Shutdown
			break
		}
	}
}

// Pool manages the pool of processes to which gophernaut dispatches
// requests.
type Pool struct {
	stoppedCount   int
	processCount   int
	controlChannel chan Event
	eventsChannel  chan Event
}

// Start up the pool
func (p *Pool) Start() {
	p.controlChannel = make(chan Event)
	p.eventsChannel = make(chan Event)

	// Handle signals to try to do a graceful shutdown:
	receivedSignals := make(chan os.Signal, 1)
	signal.Notify(receivedSignals, os.Interrupt) // , syscall.SIGINT, syscall.SIGTERM)
	go func() {
		for sig := range receivedSignals {
			fmt.Printf("Received signal, %s, shutting down workers...\n", sig)
			break
		}
		close(p.controlChannel)
		signal.Stop(receivedSignals)
	}()

	// Actually start some processes
	for _, executable := range executables {
		go startProcess(p.controlChannel, p.eventsChannel, executable)
	}
}

// ManageProcesses manages soem processes
func (p *Pool) ManageProcesses() {
	for event := range p.eventsChannel {
		switch event {
		case Shutdown:
			p.stoppedCount++
		case Start:
			p.processCount++
		}
		if p.processCount == p.stoppedCount {
			log.Printf("%d workers stopped, shutting down.\n", p.processCount)
			os.Exit(1)
		}
	}
}

var requestCount = 0

func myHandler(w http.ResponseWriter, myReq *http.Request) {
	requestPath := myReq.URL.Path
	// TODO: multiprocess, pick one of n hostnames based on pool status
	hostname := hostnames[requestCount%len(hostnames)]
	requestCount++
	targetURL, _ := url.Parse(hostname)
	director := func(req *http.Request) {
		targetQuery := targetURL.RawQuery
		req.URL.Scheme = targetURL.Scheme
		// TODO: adjust request host to assign the request to the appropriate child process
		req.URL.Host = targetURL.Host

		// clean up but preserve trailing slash:
		trailing := strings.HasSuffix(req.URL.Path, "/")
		req.URL.Path = path.Join(targetURL.Path, req.URL.Path)
		if trailing && !strings.HasSuffix(req.URL.Path, "/") {
			req.URL.Path += "/"
		}

		// preserve query string:
		if targetQuery == "" || req.URL.RawQuery == "" {
			req.URL.RawQuery = targetQuery + req.URL.RawQuery
		} else {
			req.URL.RawQuery = targetQuery + "&" + req.URL.RawQuery
		}
	}

	proxy := &httputil.ReverseProxy{Director: director}

	staticHandler := http.StripPrefix("/static", http.FileServer(http.Dir("static")))
	adminTemplate := template.Must(template.ParseFiles("templates/admin.html"))
	adminHandler := func(w http.ResponseWriter, req *http.Request) {
		adminTemplate.Execute(w, nil)
	}

	switch {
	case requestPath == "/admin":
		adminHandler(w, myReq)
		return
	case strings.HasPrefix(requestPath, "/static"):
		staticHandler.ServeHTTP(w, myReq)
		return
	}
	proxy.ServeHTTP(w, myReq)
}

func main() {
	log.SetPrefix("gophernaut ")
	log.SetFlags(log.Ldate | log.Ltime)
	c := gophernaut.ReadConfig()
	log.Printf("Host %s and Port %d\n", c.Host, c.Port)

	pool := new(Pool)
	pool.Start()
	go pool.ManageProcesses()

	log.Printf("Gophernaut is gopher launch!\n")
	// TODO: our own ReverseProxy implementation of at least, ServeHTTP so that we can
	// monitor the response codes to track successes and failures
	log.Fatal(http.ListenAndServe(":8483", http.HandlerFunc(myHandler)))
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"encoding/json"
	"errors"
	"github.com/codegangsta/cli"
	"github.com/kubernetes/helm/pkg/format"
	"github.com/kubernetes/helm/pkg/repo"
)

func init() {
	addCommands(repoCommands())
}

const chartRepoPath = "chart_repositories"

func repoCommands() cli.Command {
	return cli.Command{
		Name:    "repository",
		Aliases: []string{"repo"},
		Usage:   "Perform chart repository operations.",
		Subcommands: []cli.Command{
			{
				Name:      "add",
				Usage:     "Add a chart repository to the remote manager.",
				ArgsUsage: "REPOSITORY_URL",
				Action:    func(c *cli.Context) { run(c, addRepo) },
			},
			{
				Name:      "list",
				Usage:     "List the chart repositories on the remote manager.",
				ArgsUsage: "",
				Action:    func(c *cli.Context) { run(c, listRepos) },
			},
			{
				Name:      "remove",
				Aliases:   []string{"rm"},
				Usage:     "Remove a chart repository from the remote manager.",
				ArgsUsage: "REPOSITORY_URL",
				Action:    func(c *cli.Context) { run(c, removeRepo) },
			},
		},
	}
}

func addRepo(c *cli.Context) error {
	args := c.Args()
	if len(args) < 1 {
		return errors.New("'helm repo add' requires a repository url as an argument")
	}
	repoURL := args[0]
	payload, _ := json.Marshal(repo.Repo{URL: repoURL})
	msg := ""
	if _, err := NewClient(c).Post(chartRepoPath, payload, &msg); err != nil {
		return err
	}
	format.Msg(repoURL + " has been added to your list of chart repositories")
	return nil
}

func listRepos(c *cli.Context) error {
	dest := []string{}
	if _, err := NewClient(c).Get(chartRepoPath, &dest); err != nil {
		return err
	}
	if len(dest) < 1 {
		format.Info("Looks like you don't have any chart repositories.")
		format.Info("Add a chart repository using the `helm repo add [REPOSITORY_URL]` command.")
	} else {
		format.Msg("Chart Repositories:\n")
		format.List(dest)
	}
	return nil
}

func removeRepo(c *cli.Context) error {
	args := c.Args()
	if len(args) < 1 {
		return errors.New("'helm repo remove' requires a repository url as an argument")
	}
	repoURL := args[0]
	if _, err := NewClient(c).Delete(chartRepoPath, repoURL); err != nil {
		return err
	}
	format.Msg(repoURL + "has been removed.\n")
	return nil
}
<commit_msg>feat(repo): update repo path<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"encoding/json"
	"errors"
	"github.com/codegangsta/cli"
	"github.com/kubernetes/helm/pkg/format"
	"github.com/kubernetes/helm/pkg/repo"
)

func init() {
	addCommands(repoCommands())
}

const chartRepoPath = "repositories"

func repoCommands() cli.Command {
	return cli.Command{
		Name:    "repository",
		Aliases: []string{"repo"},
		Usage:   "Perform chart repository operations.",
		Subcommands: []cli.Command{
			{
				Name:      "add",
				Usage:     "Add a chart repository to the remote manager.",
				ArgsUsage: "REPOSITORY_URL",
				Action:    func(c *cli.Context) { run(c, addRepo) },
			},
			{
				Name:      "list",
				Usage:     "List the chart repositories on the remote manager.",
				ArgsUsage: "",
				Action:    func(c *cli.Context) { run(c, listRepos) },
			},
			{
				Name:      "remove",
				Aliases:   []string{"rm"},
				Usage:     "Remove a chart repository from the remote manager.",
				ArgsUsage: "REPOSITORY_URL",
				Action:    func(c *cli.Context) { run(c, removeRepo) },
			},
		},
	}
}

func addRepo(c *cli.Context) error {
	args := c.Args()
	if len(args) < 1 {
		return errors.New("'helm repo add' requires a repository url as an argument")
	}
	repoURL := args[0]
	payload, _ := json.Marshal(repo.Repo{URL: repoURL})
	msg := ""
	if _, err := NewClient(c).Post(chartRepoPath, payload, &msg); err != nil {
		return err
	}
	format.Msg(repoURL + " has been added to your list of chart repositories")
	return nil
}

func listRepos(c *cli.Context) error {
	dest := []string{}
	if _, err := NewClient(c).Get(chartRepoPath, &dest); err != nil {
		return err
	}
	if len(dest) < 1 {
		format.Info("Looks like you don't have any chart repositories.")
		format.Info("Add a chart repository using the `helm repo add [REPOSITORY_URL]` command.")
	} else {
		format.Msg("Chart Repositories:\n")
		format.List(dest)
	}
	return nil
}

func removeRepo(c *cli.Context) error {
	args := c.Args()
	if len(args) < 1 {
		return errors.New("'helm repo remove' requires a repository url as an argument")
	}
	repoURL := args[0]
	if _, err := NewClient(c).Delete(chartRepoPath, repoURL); err != nil {
		return err
	}
	format.Msg(repoURL + "has been removed.\n")
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/pierrre/mangadownloader"
)

func main() {
	var service mangadownloader.Service = &mangadownloader.MangaReaderService{}
	mangas, err := service.Mangas()
	if err != nil {
		panic(err)
	}
	for _, manga := range mangas {
		fmt.Println(manga)
		chapters, err := manga.Chapters()
		if err != nil {
			panic(err)
		}
		for _, chapter := range chapters {
			fmt.Println(chapter)
		}
	}
}
<commit_msg>Add formatting in test command<commit_after>package main

import (
	"fmt"
	"github.com/pierrre/mangadownloader"
)

func main() {
	var service mangadownloader.Service = &mangadownloader.MangaReaderService{}
	mangas, err := service.Mangas()
	if err != nil {
		panic(err)
	}
	for _, manga := range mangas {
		fmt.Println(manga)
		chapters, err := manga.Chapters()
		if err != nil {
			panic(err)
		}
		for _, chapter := range chapters {
			fmt.Println("	" + fmt.Sprint(chapter))
		}
	}
}
<|endoftext|>


<commit_before>package main

import "github.com/urfave/cli"

var Flags = []cli.Flag{
	cli.StringFlag{
		Name:   "room_id",
		EnvVar: "ROOM_ID",
		Value:  "chatter",
		Usage:  "Game On registration room id",
	},

	cli.StringFlag{
		Name:   "room_service_url",
		EnvVar: "ROOM_SERVICE_URL",
		Value:  "http://localhost:6379/room",
		Usage:  "Room service URL",
	},

	cli.IntFlag{
		Name:   "websocket_port",
		EnvVar: "WEBSOCKET_PORT",
		Value:  3000,
		Usage:  "Port to listen for incoming websocket connections",
	},
}

type Config struct {
	RoomID         string
	RoomServiceURL string
	WebsocketPort  int
}

func newConfig(context *cli.Context) *Config {
	return &Config{
		RoomID:         context.String("room_id"),
		RoomServiceURL: context.String("room_service_url"),
		WebsocketPort:  context.Int("websocket_port"),
	}
}
<commit_msg>remove default value for room ID configuration<commit_after>package main

import "github.com/urfave/cli"

var Flags = []cli.Flag{
	cli.StringFlag{
		Name:   "room_id",
		EnvVar: "ROOM_ID",
		Value:  "",
		Usage:  "Game On registration room id",
	},

	cli.StringFlag{
		Name:   "room_service_url",
		EnvVar: "ROOM_SERVICE_URL",
		Value:  "http://localhost:6379/room",
		Usage:  "Room service URL",
	},

	cli.IntFlag{
		Name:   "websocket_port",
		EnvVar: "WEBSOCKET_PORT",
		Value:  3000,
		Usage:  "Port to listen for incoming websocket connections",
	},
}

type Config struct {
	RoomID         string
	RoomServiceURL string
	WebsocketPort  int
}

func newConfig(context *cli.Context) *Config {
	return &Config{
		RoomID:         context.String("room_id"),
		RoomServiceURL: context.String("room_service_url"),
		WebsocketPort:  context.Int("websocket_port"),
	}
}
<|endoftext|>


<commit_before>package collector

import (
	"fmt"
	"os"
	"strings"
)

import "github.com/gwenn/yacr"

type ReaderCSVConfig struct {
	File   string
	Fields string
}

type ReaderCSV struct {
	file    string
	header  []string
	counter int32
}

func NewReaderCSV(config ReaderCSVConfig) *ReaderCSV {
	reader := new(ReaderCSV)
	reader.SetConfig(config)

	return reader
}

func (self *ReaderCSV) SetConfig(config ReaderCSVConfig) {
	for _, field := range strings.Split(config.Fields, ",") {
		self.header = append(self.header, field)
	}

	self.file = config.File
}

func (self *ReaderCSV) ReadIntoChannel(channel chan map[string]string) {
	file, err := os.Open(self.file)
	if err != nil {
		panic(fmt.Sprintf("open %s: %v", self.file, err))
	}

	defer file.Close()

	reader := yacr.DefaultReader(file)

	row := make(map[string]string)
	for reader.Scan() {
		if reader.EmptyLine() { // skip empty line (or line comment)
			continue
		}

		row[self.header[len(row)]] = reader.Text()

		if reader.EndOfRecord() {
			self.emitRecord(channel, row)
			row = make(map[string]string)
		}
	}

	self.emitRecord(channel, row)
}

func (self *ReaderCSV) emitRecord(channel chan map[string]string, row map[string]string) {
	if len(row) > 0 {
		channel <- row
		self.counter++
	}
}
<commit_msg>pattern exploration<commit_after>package collector

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
)

import "github.com/gwenn/yacr"

type ReaderCSVConfig struct {
	File    string
	Fields  string
	Pattern string
}

type ReaderCSV struct {
	file    string
	pattern string
	header  []string
	counter int32
}

func NewReaderCSV(config ReaderCSVConfig) *ReaderCSV {
	reader := new(ReaderCSV)
	reader.SetConfig(config)

	return reader
}

func (self *ReaderCSV) SetConfig(config ReaderCSVConfig) {
	for _, field := range strings.Split(config.Fields, ",") {
		self.header = append(self.header, field)
	}

	self.file = config.File
	self.pattern = config.Pattern
}

func (self *ReaderCSV) ReadIntoChannel(channel chan map[string]string) {
	//if ok := self.file; ok {
	//	self.readFileInChannel(self.file, channel)
	//} else {
	files, err := filepath.Glob(self.pattern)
	if err != nil {
		panic(fmt.Sprintf("open %s: %v", self.pattern, err))
	}

	for _, file := range files {
		self.readFileInChannel(file, channel)
	}

	fmt.Println(files)
	//}
}

func (self *ReaderCSV) readFileInChannel(filename string, channel chan map[string]string) {
	GetLogger().Info("Processing '%s'", filename)

	file, err := os.Open(filename)
	if err != nil {
		GetLogger().Error("open %s: %v", self.file, err)
	}

	defer file.Close()

	reader := yacr.DefaultReader(file)

	row := make(map[string]string)
	for reader.Scan() {
		if reader.EmptyLine() { // skip empty line (or line comment)
			continue
		}

		row[self.header[len(row)]] = reader.Text()

		if reader.EndOfRecord() {
			self.emitRecord(channel, row)
			row = make(map[string]string)
		}
	}

	self.emitRecord(channel, row)
}

func (self *ReaderCSV) emitRecord(channel chan map[string]string, row map[string]string) {
	if len(row) > 0 {
		channel <- row
		self.counter++
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// The main package for the Prometheus server executeable.
package main

import (
	"bytes"
	"flag"
	"fmt"
	_ "net/http/pprof" // Comment this line to disable pprof endpoint.
	"os"
	"os/signal"
	"strings"
	"syscall"
	"text/template"
	"time"

	"github.com/prometheus/log"

	"github.com/prometheus/client_golang/prometheus"

	"github.com/prometheus/prometheus/config"
	"github.com/prometheus/prometheus/notification"
	"github.com/prometheus/prometheus/promql"
	"github.com/prometheus/prometheus/retrieval"
	"github.com/prometheus/prometheus/rules"
	"github.com/prometheus/prometheus/storage"
	"github.com/prometheus/prometheus/storage/local"
	"github.com/prometheus/prometheus/storage/remote"
	"github.com/prometheus/prometheus/version"
	"github.com/prometheus/prometheus/web"
)

func main() {
	os.Exit(Main())
}

var (
	configSuccess = prometheus.NewGauge(prometheus.GaugeOpts{
		Namespace: "prometheus",
		Name:      "config_last_reload_successful",
		Help:      "Whether the last configuration reload attempt was successful.",
	})
	configSuccessTime = prometheus.NewGauge(prometheus.GaugeOpts{
		Namespace: "prometheus",
		Name:      "config_last_reload_success_timestamp_seconds",
		Help:      "Timestamp of the last successful configuration reload.",
	})
)

// Main manages the startup and shutdown lifecycle of the entire Prometheus server.
func Main() int {
	if err := parse(os.Args[1:]); err != nil {
		return 2
	}

	printVersion()
	if cfg.printVersion {
		return 0
	}

	var reloadables []Reloadable

	var (
		memStorage     = local.NewMemorySeriesStorage(&cfg.storage)
		remoteStorage  = remote.New(&cfg.remote)
		sampleAppender = storage.Fanout{memStorage}
	)
	if remoteStorage != nil {
		sampleAppender = append(sampleAppender, remoteStorage)
		reloadables = append(reloadables, remoteStorage)
	}

	var (
		notificationHandler = notification.NewNotificationHandler(&cfg.notification)
		targetManager       = retrieval.NewTargetManager(sampleAppender)
		queryEngine         = promql.NewEngine(memStorage, &cfg.queryEngine)
	)

	ruleManager := rules.NewManager(&rules.ManagerOptions{
		SampleAppender:      sampleAppender,
		NotificationHandler: notificationHandler,
		QueryEngine:         queryEngine,
		ExternalURL:         cfg.web.ExternalURL,
	})

	flags := map[string]string{}
	cfg.fs.VisitAll(func(f *flag.Flag) {
		flags[f.Name] = f.Value.String()
	})

	status := &web.PrometheusStatus{
		TargetPools: targetManager.Pools,
		Rules:       ruleManager.Rules,
		Flags:       flags,
		Birth:       time.Now(),
	}

	webHandler := web.New(memStorage, queryEngine, ruleManager, status, &cfg.web)

	reloadables = append(reloadables, status, targetManager, ruleManager, webHandler, notificationHandler)

	if !reloadConfig(cfg.configFile, reloadables...) {
		return 1
	}

	// Wait for reload or termination signals. Start the handler for SIGHUP as
	// early as possible, but ignore it until we are ready to handle reloading
	// our config.
	hup := make(chan os.Signal)
	hupReady := make(chan bool)
	signal.Notify(hup, syscall.SIGHUP)
	go func() {
		<-hupReady
		for {
			select {
			case <-hup:
			case <-webHandler.Reload():
			}
			reloadConfig(cfg.configFile, reloadables...)
		}
	}()

	// Start all components.
	if err := memStorage.Start(); err != nil {
		log.Errorln("Error opening memory series storage:", err)
		return 1
	}
	defer func() {
		if err := memStorage.Stop(); err != nil {
			log.Errorln("Error stopping storage:", err)
		}
	}()

	if remoteStorage != nil {
		prometheus.MustRegister(remoteStorage)

		go remoteStorage.Run()
		defer remoteStorage.Stop()
	}
	// The storage has to be fully initialized before registering.
	prometheus.MustRegister(memStorage)
	prometheus.MustRegister(notificationHandler)
	prometheus.MustRegister(configSuccess)
	prometheus.MustRegister(configSuccessTime)

	go ruleManager.Run()
	defer ruleManager.Stop()

	go notificationHandler.Run()
	defer notificationHandler.Stop()

	go targetManager.Run()
	defer targetManager.Stop()

	defer queryEngine.Stop()

	go webHandler.Run()

	// Wait for reload or termination signals.
	close(hupReady) // Unblock SIGHUP handler.

	term := make(chan os.Signal)
	signal.Notify(term, os.Interrupt, syscall.SIGTERM)
	select {
	case <-term:
		log.Warn("Received SIGTERM, exiting gracefully...")
	case <-webHandler.Quit():
		log.Warn("Received termination request via web service, exiting gracefully...")
	case err := <-webHandler.ListenError():
		log.Errorln("Error starting web server, exiting gracefully:", err)
	}

	log.Info("See you next time!")
	return 0
}

// Reloadable things can change their internal state to match a new config
// and handle failure gracefully.
type Reloadable interface {
	ApplyConfig(*config.Config) bool
}

func reloadConfig(filename string, rls ...Reloadable) (success bool) {
	log.Infof("Loading configuration file %s", filename)
	defer func() {
		if success {
			configSuccess.Set(1)
			configSuccessTime.Set(float64(time.Now().Unix()))
		} else {
			configSuccess.Set(0)
		}
	}()

	conf, err := config.LoadFile(filename)
	if err != nil {
		log.Errorf("Couldn't load configuration (-config.file=%s): %v", filename, err)
		log.Errorf("Note: The configuration format has changed with version 0.14. Please see the documentation (http://prometheus.io/docs/operating/configuration/) and the provided configuration migration tool (https://github.com/prometheus/migrate).")
		return false
	}
	success = true

	for _, rl := range rls {
		success = success && rl.ApplyConfig(conf)
	}
	return success
}

var versionInfoTmpl = `
prometheus, version {{.version}} (branch: {{.branch}}, revision: {{.revision}})
  build user:       {{.buildUser}}
  build date:       {{.buildDate}}
  go version:       {{.goVersion}}
`

func printVersion() {
	t := template.Must(template.New("version").Parse(versionInfoTmpl))

	var buf bytes.Buffer
	if err := t.ExecuteTemplate(&buf, "version", version.Map); err != nil {
		panic(err)
	}
	fmt.Fprintln(os.Stdout, strings.TrimSpace(buf.String()))
}
<commit_msg>Remove notice about 0.14.x config file format change.<commit_after>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// The main package for the Prometheus server executeable.
package main

import (
	"bytes"
	"flag"
	"fmt"
	_ "net/http/pprof" // Comment this line to disable pprof endpoint.
	"os"
	"os/signal"
	"strings"
	"syscall"
	"text/template"
	"time"

	"github.com/prometheus/log"

	"github.com/prometheus/client_golang/prometheus"

	"github.com/prometheus/prometheus/config"
	"github.com/prometheus/prometheus/notification"
	"github.com/prometheus/prometheus/promql"
	"github.com/prometheus/prometheus/retrieval"
	"github.com/prometheus/prometheus/rules"
	"github.com/prometheus/prometheus/storage"
	"github.com/prometheus/prometheus/storage/local"
	"github.com/prometheus/prometheus/storage/remote"
	"github.com/prometheus/prometheus/version"
	"github.com/prometheus/prometheus/web"
)

func main() {
	os.Exit(Main())
}

var (
	configSuccess = prometheus.NewGauge(prometheus.GaugeOpts{
		Namespace: "prometheus",
		Name:      "config_last_reload_successful",
		Help:      "Whether the last configuration reload attempt was successful.",
	})
	configSuccessTime = prometheus.NewGauge(prometheus.GaugeOpts{
		Namespace: "prometheus",
		Name:      "config_last_reload_success_timestamp_seconds",
		Help:      "Timestamp of the last successful configuration reload.",
	})
)

// Main manages the startup and shutdown lifecycle of the entire Prometheus server.
func Main() int {
	if err := parse(os.Args[1:]); err != nil {
		return 2
	}

	printVersion()
	if cfg.printVersion {
		return 0
	}

	var reloadables []Reloadable

	var (
		memStorage     = local.NewMemorySeriesStorage(&cfg.storage)
		remoteStorage  = remote.New(&cfg.remote)
		sampleAppender = storage.Fanout{memStorage}
	)
	if remoteStorage != nil {
		sampleAppender = append(sampleAppender, remoteStorage)
		reloadables = append(reloadables, remoteStorage)
	}

	var (
		notificationHandler = notification.NewNotificationHandler(&cfg.notification)
		targetManager       = retrieval.NewTargetManager(sampleAppender)
		queryEngine         = promql.NewEngine(memStorage, &cfg.queryEngine)
	)

	ruleManager := rules.NewManager(&rules.ManagerOptions{
		SampleAppender:      sampleAppender,
		NotificationHandler: notificationHandler,
		QueryEngine:         queryEngine,
		ExternalURL:         cfg.web.ExternalURL,
	})

	flags := map[string]string{}
	cfg.fs.VisitAll(func(f *flag.Flag) {
		flags[f.Name] = f.Value.String()
	})

	status := &web.PrometheusStatus{
		TargetPools: targetManager.Pools,
		Rules:       ruleManager.Rules,
		Flags:       flags,
		Birth:       time.Now(),
	}

	webHandler := web.New(memStorage, queryEngine, ruleManager, status, &cfg.web)

	reloadables = append(reloadables, status, targetManager, ruleManager, webHandler, notificationHandler)

	if !reloadConfig(cfg.configFile, reloadables...) {
		return 1
	}

	// Wait for reload or termination signals. Start the handler for SIGHUP as
	// early as possible, but ignore it until we are ready to handle reloading
	// our config.
	hup := make(chan os.Signal)
	hupReady := make(chan bool)
	signal.Notify(hup, syscall.SIGHUP)
	go func() {
		<-hupReady
		for {
			select {
			case <-hup:
			case <-webHandler.Reload():
			}
			reloadConfig(cfg.configFile, reloadables...)
		}
	}()

	// Start all components.
	if err := memStorage.Start(); err != nil {
		log.Errorln("Error opening memory series storage:", err)
		return 1
	}
	defer func() {
		if err := memStorage.Stop(); err != nil {
			log.Errorln("Error stopping storage:", err)
		}
	}()

	if remoteStorage != nil {
		prometheus.MustRegister(remoteStorage)

		go remoteStorage.Run()
		defer remoteStorage.Stop()
	}
	// The storage has to be fully initialized before registering.
	prometheus.MustRegister(memStorage)
	prometheus.MustRegister(notificationHandler)
	prometheus.MustRegister(configSuccess)
	prometheus.MustRegister(configSuccessTime)

	go ruleManager.Run()
	defer ruleManager.Stop()

	go notificationHandler.Run()
	defer notificationHandler.Stop()

	go targetManager.Run()
	defer targetManager.Stop()

	defer queryEngine.Stop()

	go webHandler.Run()

	// Wait for reload or termination signals.
	close(hupReady) // Unblock SIGHUP handler.

	term := make(chan os.Signal)
	signal.Notify(term, os.Interrupt, syscall.SIGTERM)
	select {
	case <-term:
		log.Warn("Received SIGTERM, exiting gracefully...")
	case <-webHandler.Quit():
		log.Warn("Received termination request via web service, exiting gracefully...")
	case err := <-webHandler.ListenError():
		log.Errorln("Error starting web server, exiting gracefully:", err)
	}

	log.Info("See you next time!")
	return 0
}

// Reloadable things can change their internal state to match a new config
// and handle failure gracefully.
type Reloadable interface {
	ApplyConfig(*config.Config) bool
}

func reloadConfig(filename string, rls ...Reloadable) (success bool) {
	log.Infof("Loading configuration file %s", filename)
	defer func() {
		if success {
			configSuccess.Set(1)
			configSuccessTime.Set(float64(time.Now().Unix()))
		} else {
			configSuccess.Set(0)
		}
	}()

	conf, err := config.LoadFile(filename)
	if err != nil {
		log.Errorf("Couldn't load configuration (-config.file=%s): %v", filename, err)
		return false
	}
	success = true

	for _, rl := range rls {
		success = success && rl.ApplyConfig(conf)
	}
	return success
}

var versionInfoTmpl = `
prometheus, version {{.version}} (branch: {{.branch}}, revision: {{.revision}})
  build user:       {{.buildUser}}
  build date:       {{.buildDate}}
  go version:       {{.goVersion}}
`

func printVersion() {
	t := template.Must(template.New("version").Parse(versionInfoTmpl))

	var buf bytes.Buffer
	if err := t.ExecuteTemplate(&buf, "version", version.Map); err != nil {
		panic(err)
	}
	fmt.Fprintln(os.Stdout, strings.TrimSpace(buf.String()))
}
<|endoftext|>


<commit_before>// Copyright (c) 2016, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package main

import (
	"bufio"
	"bytes"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"testing"

	"mvdan.cc/sh/v3/syntax"
)

func init() {
	parser = syntax.NewParser(syntax.KeepComments)
	printer = syntax.NewPrinter()
}

func TestStdin(t *testing.T) {
	var buf bytes.Buffer
	out = &buf
	t.Run("Regular", func(t *testing.T) {
		in = strings.NewReader(" foo")
		buf.Reset()
		if err := formatStdin(); err != nil {
			t.Fatal(err)
		}
		if got, want := buf.String(), "foo\n"; got != want {
			t.Fatalf("got=%q want=%q", got, want)
		}
	})

	t.Run("List", func(t *testing.T) {
		*list = true
		defer func() { *list = false }()
		in = strings.NewReader(" foo")
		buf.Reset()
		if err := formatStdin(); err != nil {
			t.Fatal(err)
		}
		if got, want := buf.String(), "<standard input>\n"; got != want {
			t.Fatalf("got=%q want=%q", got, want)
		}
	})

	t.Run("Diff", func(t *testing.T) {
		*diff = true
		defer func() { *diff = false }()
		in = strings.NewReader(" foo\nbar\n\n")
		buf.Reset()
		if err := formatStdin(); err != errChangedWithDiff {
			t.Fatalf("got=%q want=%q", err, errChangedWithDiff)
		}
		want := `diff -u <standard input>.orig <standard input>
@@ -1,3 +1,2 @@
- foo
+foo
 bar
-
`
		if got := buf.String(); got != want {
			t.Fatalf("got:\n%swant:\n%s", got, want)
		}
	})
}

type action uint

const (
	None action = iota
	Modify
	Error
)

var walkTests = []struct {
	want       action
	symlink    bool
	path, body string
}{
	{Modify, false, "shebang-1", "#!/bin/sh\n foo"},
	{Modify, false, "shebang-2", "#!/bin/bash\n foo"},
	{Modify, false, "shebang-3", "#!/usr/bin/sh\n foo"},
	{Modify, false, "shebang-4", "#!/usr/bin/env bash\n foo"},
	{Modify, false, "shebang-5", "#!/bin/env sh\n foo"},
	{Modify, false, "shebang-space", "#! /bin/sh\n foo"},
	{Modify, false, "shebang-tabs", "#!\t/bin/env\tsh\n foo"},
	{Modify, false, "shebang-args", "#!/bin/bash -e -x\nfoo"},
	{Modify, false, "ext.sh", " foo"},
	{Modify, false, "ext.bash", " foo"},
	{Modify, false, "ext-shebang.sh", "#!/bin/sh\n foo"},
	{Modify, false, filepath.Join("dir", "ext.sh"), " foo"},
	{None, false, ".hidden", " foo long enough"},
	{None, false, ".hidden-shebang", "#!/bin/sh\n foo"},
	{None, false, "..hidden-shebang", "#!/bin/sh\n foo"},
	{None, false, "noext-empty", " foo"},
	{None, false, "noext-noshebang", " foo long enough"},
	{None, false, "shebang-nonewline", "#!/bin/shfoo"},
	{None, false, "ext.other", " foo"},
	{None, false, "ext-shebang.other", "#!/bin/sh\n foo"},
	{None, false, "shebang-nospace", "#!/bin/envsh\n foo"},
	{None, false, filepath.Join(".git", "ext.sh"), " foo"},
	{None, false, filepath.Join(".svn", "ext.sh"), " foo"},
	{None, false, filepath.Join(".hg", "ext.sh"), " foo"},
	{Error, false, "parse-error.sh", " foo("},
	{None, true, "reallylongdir/symlink-file", "ext-shebang.sh"},
	{None, true, "symlink-dir", "reallylongdir"},
	{None, true, "symlink-none", "reallylongdir/nonexistent"},
}

var errPathMentioned = regexp.MustCompile(`([^ :]+):`)

func TestWalk(t *testing.T) {
	t.Parallel()
	tdir, err := ioutil.TempDir("", "shfmt-walk")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(tdir)
	for _, wt := range walkTests {
		path := filepath.Join(tdir, wt.path)
		dir, _ := filepath.Split(path)
		os.MkdirAll(dir, 0777)
		if wt.symlink {
			if err := os.Symlink(wt.body, path); err != nil {
				t.Fatal(err)
			}
			continue
		}
		err := ioutil.WriteFile(path, []byte(wt.body), 0666)
		if err != nil {
			t.Fatal(err)
		}
	}
	var outBuf bytes.Buffer
	out = &outBuf
	*list, *write = true, true
	*simple = true
	gotError := false
	errored := map[string]bool{}
	onError := func(err error) {
		gotError = true
		line := err.Error()
		if sub := errPathMentioned.FindStringSubmatch(line); sub != nil {
			errored[sub[1]] = true
		}
	}
	doWalk := func(path string) {
		gotError = false
		outBuf.Reset()
		walk(path, onError)
	}
	doWalk(tdir)
	modified := map[string]bool{}
	outScan := bufio.NewScanner(&outBuf)
	for outScan.Scan() {
		path := outScan.Text()
		modified[path] = true
	}
	for _, wt := range walkTests {
		t.Run(wt.path, func(t *testing.T) {
			mod := modified[filepath.Join(tdir, wt.path)]
			if mod && wt.want != Modify {
				t.Fatalf("walk had to not run on %s but did", wt.path)
			} else if !mod && wt.want == Modify {
				t.Fatalf("walk had to run on %s but didn't", wt.path)
			}
			err := errored[filepath.Join(tdir, wt.path)]
			if err && wt.want != Error {
				t.Fatalf("walk had to not err on %s but did", wt.path)
			} else if !err && wt.want == Error {
				t.Fatalf("walk had to err on %s but didn't", wt.path)
			}
		})
	}
	if doWalk(tdir); outBuf.Len() > 0 {
		t.Fatal("shfmt -l -w printed paths on a duplicate run")
	}
	*list, *write = false, false
	if doWalk(tdir); outBuf.Len() == 0 {
		t.Fatal("shfmt without -l nor -w did not print anything")
	}
	if doWalk(filepath.Join(tdir, ".hidden")); outBuf.Len() == 0 {
		t.Fatal("`shfmt .hidden` did not print anything")
	}
	if doWalk(filepath.Join(tdir, "nonexistent")); !gotError {
		t.Fatal("`shfmt nonexistent` did not error")
	}
	*find = true
	doWalk(tdir)
	numFound := strings.Count(outBuf.String(), "\n")
	if want := 13; numFound != want {
		t.Fatalf("shfmt -f printed %d paths, but wanted %d", numFound, want)
	}
	*find = false
}
<commit_msg>cmd/shfmt: fix walk tests on Windows<commit_after>// Copyright (c) 2016, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package main

import (
	"bufio"
	"bytes"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"testing"

	"mvdan.cc/sh/v3/syntax"
)

func init() {
	parser = syntax.NewParser(syntax.KeepComments)
	printer = syntax.NewPrinter()
}

func TestStdin(t *testing.T) {
	var buf bytes.Buffer
	out = &buf
	t.Run("Regular", func(t *testing.T) {
		in = strings.NewReader(" foo")
		buf.Reset()
		if err := formatStdin(); err != nil {
			t.Fatal(err)
		}
		if got, want := buf.String(), "foo\n"; got != want {
			t.Fatalf("got=%q want=%q", got, want)
		}
	})

	t.Run("List", func(t *testing.T) {
		*list = true
		defer func() { *list = false }()
		in = strings.NewReader(" foo")
		buf.Reset()
		if err := formatStdin(); err != nil {
			t.Fatal(err)
		}
		if got, want := buf.String(), "<standard input>\n"; got != want {
			t.Fatalf("got=%q want=%q", got, want)
		}
	})

	t.Run("Diff", func(t *testing.T) {
		*diff = true
		defer func() { *diff = false }()
		in = strings.NewReader(" foo\nbar\n\n")
		buf.Reset()
		if err := formatStdin(); err != errChangedWithDiff {
			t.Fatalf("got=%q want=%q", err, errChangedWithDiff)
		}
		want := `diff -u <standard input>.orig <standard input>
@@ -1,3 +1,2 @@
- foo
+foo
 bar
-
`
		if got := buf.String(); got != want {
			t.Fatalf("got:\n%swant:\n%s", got, want)
		}
	})
}

type action uint

const (
	None action = iota
	Modify
	Error
)

var walkTests = []struct {
	want       action
	symlink    bool
	path, body string
}{
	{Modify, false, "shebang-1", "#!/bin/sh\n foo"},
	{Modify, false, "shebang-2", "#!/bin/bash\n foo"},
	{Modify, false, "shebang-3", "#!/usr/bin/sh\n foo"},
	{Modify, false, "shebang-4", "#!/usr/bin/env bash\n foo"},
	{Modify, false, "shebang-5", "#!/bin/env sh\n foo"},
	{Modify, false, "shebang-space", "#! /bin/sh\n foo"},
	{Modify, false, "shebang-tabs", "#!\t/bin/env\tsh\n foo"},
	{Modify, false, "shebang-args", "#!/bin/bash -e -x\nfoo"},
	{Modify, false, "ext.sh", " foo"},
	{Modify, false, "ext.bash", " foo"},
	{Modify, false, "ext-shebang.sh", "#!/bin/sh\n foo"},
	{Modify, false, filepath.Join("dir", "ext.sh"), " foo"},
	{None, false, ".hidden", " foo long enough"},
	{None, false, ".hidden-shebang", "#!/bin/sh\n foo"},
	{None, false, "..hidden-shebang", "#!/bin/sh\n foo"},
	{None, false, "noext-empty", " foo"},
	{None, false, "noext-noshebang", " foo long enough"},
	{None, false, "shebang-nonewline", "#!/bin/shfoo"},
	{None, false, "ext.other", " foo"},
	{None, false, "ext-shebang.other", "#!/bin/sh\n foo"},
	{None, false, "shebang-nospace", "#!/bin/envsh\n foo"},
	{None, false, filepath.Join(".git", "ext.sh"), " foo"},
	{None, false, filepath.Join(".svn", "ext.sh"), " foo"},
	{None, false, filepath.Join(".hg", "ext.sh"), " foo"},
	{Error, false, "parse-error.sh", " foo("},
	{None, true, "reallylongdir/symlink-file", "ext-shebang.sh"},
	{None, true, "symlink-dir", "reallylongdir"},
	{None, true, "symlink-none", "reallylongdir/nonexistent"},
}

// errPathMentioned extracts filenames from error lines. We can't rely on
// Windows paths not containing colon characters, so we must find the end of the
// path based on the ":line:col: " suffix.
var errPathMentioned = regexp.MustCompile(`^(.+):\d+:\d+: `)

func TestWalk(t *testing.T) {
	t.Parallel()
	tdir, err := ioutil.TempDir("", "shfmt-walk")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(tdir)
	for _, wt := range walkTests {
		path := filepath.Join(tdir, wt.path)
		dir, _ := filepath.Split(path)
		os.MkdirAll(dir, 0777)
		if wt.symlink {
			if err := os.Symlink(wt.body, path); err != nil {
				t.Fatal(err)
			}
			continue
		}
		err := ioutil.WriteFile(path, []byte(wt.body), 0666)
		if err != nil {
			t.Fatal(err)
		}
	}
	var outBuf bytes.Buffer
	out = &outBuf
	*list, *write = true, true
	*simple = true
	gotError := false
	errored := map[string]bool{}
	onError := func(err error) {
		gotError = true
		line := err.Error()
		if sub := errPathMentioned.FindStringSubmatch(line); sub != nil {
			errored[sub[1]] = true
		}
	}
	doWalk := func(path string) {
		gotError = false
		outBuf.Reset()
		walk(path, onError)
	}
	doWalk(tdir)
	modified := map[string]bool{}
	outScan := bufio.NewScanner(&outBuf)
	for outScan.Scan() {
		path := outScan.Text()
		modified[path] = true
	}
	for _, wt := range walkTests {
		t.Run(wt.path, func(t *testing.T) {
			mod := modified[filepath.Join(tdir, wt.path)]
			if mod && wt.want != Modify {
				t.Fatalf("walk had to not run on %s but did", wt.path)
			} else if !mod && wt.want == Modify {
				t.Fatalf("walk had to run on %s but didn't", wt.path)
			}
			err := errored[filepath.Join(tdir, wt.path)]
			if err && wt.want != Error {
				t.Fatalf("walk had to not error on %s but did", wt.path)
			} else if !err && wt.want == Error {
				t.Fatalf("walk had to error on %s but didn't", wt.path)
			}
		})
	}
	if doWalk(tdir); outBuf.Len() > 0 {
		t.Fatal("shfmt -l -w printed paths on a duplicate run")
	}
	*list, *write = false, false
	if doWalk(tdir); outBuf.Len() == 0 {
		t.Fatal("shfmt without -l nor -w did not print anything")
	}
	if doWalk(filepath.Join(tdir, ".hidden")); outBuf.Len() == 0 {
		t.Fatal("`shfmt .hidden` did not print anything")
	}
	if doWalk(filepath.Join(tdir, "nonexistent")); !gotError {
		t.Fatal("`shfmt nonexistent` did not error")
	}
	*find = true
	doWalk(tdir)
	numFound := strings.Count(outBuf.String(), "\n")
	if want := 13; numFound != want {
		t.Fatalf("shfmt -f printed %d paths, but wanted %d", numFound, want)
	}
	*find = false
}
<|endoftext|>


<commit_before>package main

// App version variables

// Version set in source code
const Version = "0.33"

// Build time filled by make on program build
var Build string

// Commit tag from git, filled in by the compiler.
var Commit string
<commit_msg>v0.34<commit_after>package main

// App version variables

// Version set in source code
const Version = "0.34"

// Build time filled by make on program build
var Build string

// Commit tag from git, filled in by the compiler.
var Commit string
<|endoftext|>


<commit_before>package base

import "fmt"

// gotype is used in the profile field lookup table to represent the data type
// (or type category) for a field when decoded into a Go message struct.
type GoType uint8

const (
	Fit       GoType = 0 // Standard 	-> Fit base type/alias
	TimeUTC          = 1 // Time UTC 	-> time.Time
	TimeLocal        = 2 // Time Local 	-> time.Time with Location
	Lat              = 3 // Latitude 	-> fit.Latitude
	Lng              = 4 // Longitude 	-> fit.Longitude
)

func (g GoType) String() string {
	if int(g) >= len(gstring) {
		return fmt.Sprintf("<unknown base.GoType(%d)>", g)
	}
	return gstring[g]
}

var gstring = [...]string{
	"fit",
	"timeutc",
	"timelocal",
	"lat",
	"lng",
}

func (g GoType) InvalidValueAsString() string {
	if g == 0 {
		panic("no invalid string value for gotype of type fit")
	}
	if int(g) > len(tinvalid) {
		return fmt.Sprintf("<unknown base.GoType(%d)>", g)
	}
	return ginvalid[g]
}

var ginvalid = [...]string{
	"",
	"timeBase",
	"timeBase",
	"NewLatitudeInvalid()",
	"NewLongitudeInvalid()",
}
<commit_msg>internal/base: remove unused gotypes.go<commit_after><|endoftext|>


<commit_before>package parser

// type EnvoyNode struct {
// 	ID string `yaml:"id"`
// }
type ConnectTimeout struct {
	Seconds int64 `yaml:"seconds"`
}

type Endpoint struct {
	Name    string `yaml:"name"`
	Address string `yaml:"address"`
	Port    uint32 `yaml:"port"`
}

type Cluster struct {
	Name           string         `yaml:"name"`
	Endpoints      []Endpoint     `yaml:"endpoints"`
	ConnectTimeout ConnectTimeout `yaml:"connect_timeout"`
}

type Route struct {
	Name         string   `yaml:"name"`
	Prefix       string   `yaml:"prefix"`
	ClusterNames []string `yaml:"clusters"`
}

type Listener struct {
	Name    string  `yaml:"name"`
	Address string  `yaml:"address"`
	Port    uint32  `yaml:"port"`
	Routes  []Route `yaml:"routes"`
}

type Runtime struct {
	Name string `yaml:"name"`
}

type Secret struct {
	Name string `yaml:"name"`
}

type Resources struct {
	Endpoints []Endpoint `yaml:"endpoints"`
	Clusters  []Cluster  `yaml:"clusters"`
	Routes    []Route    `yaml:"routes"`
	Listeners []Listener `yaml:"listeners"`
	Runtimes  []Runtime  `yaml:"runtimes"`
	Secrets   []Secret   `yaml:"secret"`
}

type Snapshot struct {
	Node      string    `yaml:"node"`
	Version   string    `yaml:"version"`
	Resources Resources `yaml:"resources"`
}

type DiscoveryResponse struct {
	VersionInfo string    `default:"" yaml:"version_info"`
	TypeURL     string    `yaml:"type_url"`
	Resources   []Cluster `yaml:"resources"` //hack for now, should be any type of resource
	Nonce       string    `default:"" yaml:"nonce"`
}

type SimpleResponse struct {
	// Only the info used for validating our tests
	Version string
	Resources []string
	Nonce string
}
<commit_msg>Remove unused envoyNode type<commit_after>package parser

type ConnectTimeout struct {
	Seconds int64 `yaml:"seconds"`
}

type Endpoint struct {
	Name    string `yaml:"name"`
	Address string `yaml:"address"`
	Port    uint32 `yaml:"port"`
}

type Cluster struct {
	Name           string         `yaml:"name"`
	Endpoints      []Endpoint     `yaml:"endpoints"`
	ConnectTimeout ConnectTimeout `yaml:"connect_timeout"`
}

type Route struct {
	Name         string   `yaml:"name"`
	Prefix       string   `yaml:"prefix"`
	ClusterNames []string `yaml:"clusters"`
}

type Listener struct {
	Name    string  `yaml:"name"`
	Address string  `yaml:"address"`
	Port    uint32  `yaml:"port"`
	Routes  []Route `yaml:"routes"`
}

type Runtime struct {
	Name string `yaml:"name"`
}

type Secret struct {
	Name string `yaml:"name"`
}

type Resources struct {
	Endpoints []Endpoint `yaml:"endpoints"`
	Clusters  []Cluster  `yaml:"clusters"`
	Routes    []Route    `yaml:"routes"`
	Listeners []Listener `yaml:"listeners"`
	Runtimes  []Runtime  `yaml:"runtimes"`
	Secrets   []Secret   `yaml:"secret"`
}

type Snapshot struct {
	Node      string    `yaml:"node"`
	Version   string    `yaml:"version"`
	Resources Resources `yaml:"resources"`
}

type DiscoveryResponse struct {
	VersionInfo string    `default:"" yaml:"version_info"`
	TypeURL     string    `yaml:"type_url"`
	Resources   []Cluster `yaml:"resources"` //hack for now, should be any type of resource
	Nonce       string    `default:"" yaml:"nonce"`
}

type SimpleResponse struct {
	// Only the info used for validating our tests
	Version string
	Resources []string
	Nonce string
}
<|endoftext|>


<commit_before>package bot

import (
	"fmt"
	"log"
	"strconv"
	"strings"
	"time"

	"github.com/belak/irc"
	"github.com/khades/servbot/commandHandlers"
	"github.com/khades/servbot/ircClient"
	"github.com/khades/servbot/models"
	"github.com/khades/servbot/repos"
)

var chatHandler irc.HandlerFunc = func(client *irc.Client, message *irc.Message) {
	//	log.Println(message.String())
	msgID, found := message.Tags.GetTag("msg-id")
	if found {
		switch msgID {
		case "room_mods":
			{
				commaIndex := strings.Index(message.Params[1], ":")
				if commaIndex != -1 {
					//				log.Printf("Channel %v: got mods list", message.Params[0])
					mods := strings.Split(message.Params[1][commaIndex+2:], ", ")
					repos.PushMods(message.Params[0][1:], mods)
				}
			}
		case "resub":
			{
				msgParamMonths, msgParamMonthsFound := message.Tags.GetTag("msg-param-months")
				user, userFound := message.Tags.GetTag("display-name")
				channel := message.Params[0]
				if msgParamMonthsFound && userFound && channel != "" {
					resubCount, resubCountError := strconv.Atoi(msgParamMonths)
					if resubCountError == nil {
						formedMessage := models.ChatMessage{
							Channel:          channel,
							User:             user,
							IsMod:            false,
							IsSub:            true,
							Date:             time.Now(),
							SubscriptionInfo: &models.SubscriptionInfo{Count: resubCount}}
						repos.LogMessage(formedMessage)
						channelInfo, error := repos.GetChannelInfo(channel)
						if error == nil && channelInfo.SubAlert.Enabled == true {
							messageBody := strings.TrimSpace(fmt.Sprintf("%s %s%s",
								channelInfo.SubAlert.RepeatPrefix,
								strings.Repeat(channelInfo.SubAlert.RepeatBody+" ", formedMessage.SubscriptionInfo.Count),
								channelInfo.SubAlert.RepeatPostfix))
							if messageBody != "" {
								IrcClientInstance.SendPublic(models.OutgoingMessage{
									Body:    messageBody,
									Channel: channel,
									User:    user})
							}
						}
						log.Printf("Channel %v: %v resubbed for %v months\n", formedMessage.Channel, formedMessage.User, formedMessage.SubscriptionInfo.Count)
					}
				}
			}
		}
	}

	if message.User == "twitchnotify" {
		log.Println("Got first sub")
		user := strings.Split(message.Params[1], " ")[0]
		channel := message.Params[0]
		log.Println(user)
		log.Println(channel)

		if user != "" && channel != "" {
			formedMessage := models.ChatMessage{
				Channel:          channel,
				User:             user,
				IsMod:            false,
				IsSub:            true,
				IsPrime:          strings.Contains(message.String(), "Twitch Prime"),
				Date:             time.Now(),
				SubscriptionInfo: &models.SubscriptionInfo{Count: 1}}
			repos.LogMessage(formedMessage)
			channelInfo, error := repos.GetChannelInfo(channel)
			if error == nil && channelInfo.SubAlert.Enabled == true && channelInfo.SubAlert.FirstMessage != "" {
				IrcClientInstance.SendPublic(models.OutgoingMessage{
					Body:    channelInfo.SubAlert.FirstMessage,
					Channel: channel,
					User:    user})
			}
			log.Printf("Channel %v: %v subbed\n", formedMessage.Channel, formedMessage.User)
		}
	}
	if message.Command == "CLEARCHAT" {
		banDuration, banDurationFound := message.Tags.GetTag("ban-duration")
		intBanDuration := 0
		if banDurationFound {
			parsedValue, parseError := strconv.Atoi(banDuration)
			if parseError == nil {
				intBanDuration = parsedValue
			}
		}
		banReason, _ := message.Tags.GetTag("ban-reason")
		user := message.Params[1]
		channel := message.Params[0]
		formedMessage := models.ChatMessage{
			Channel: channel,
			User:    user,
			IsMod:   false,
			IsSub:   true,
			Date:    time.Now(),
			BanInfo: &models.BanInfo{Duration: intBanDuration, Reason: banReason}}
		repos.LogMessage(formedMessage)
		//	log.Printf("Channel %v: %v is banned for %v \n", channel, user, intBanDuration)
	}
	if message.Command == "PRIVMSG" {
		formedMessage := models.ChatMessage{
			Channel:     message.Params[0][1:],
			User:        message.User,
			MessageBody: message.Params[1],
			IsMod:       message.Tags["mod"] == "1" || message.User == "khadesru",
			IsSub:       message.Tags["subscriber"] == "1",
			IsPrime:     strings.Contains(message.Tags["badges"].Encode(), "premium/1"),
			Date:        time.Now()}
		repos.LogMessage(formedMessage)
		isCommand, commandBody := formedMessage.IsCommand()
		if isCommand {
			handlerFunction := commandHandlers.Router.Go(commandBody.Command)
			handlerFunction(true, &formedMessage, commandBody, &IrcClientInstance)
		}
	}

	if message.Command == "001" {
		client.Write("CAP REQ twitch.tv/tags")
		client.Write("CAP REQ twitch.tv/membership")
		client.Write("CAP REQ twitch.tv/commands")
		for _, value := range repos.Config.Channels {
			client.Write("JOIN #" + value)
		}
		IrcClientInstance = ircClient.IrcClient{Client: client, Bounces: make(map[string]time.Time), Ready: true}
		IrcClientInstance.SendModsCommand()
		log.Println("Bot is started")
	}
}
<commit_msg>Initial subscription alert is fixed<commit_after>package bot

import (
	"fmt"
	"log"
	"strconv"
	"strings"
	"time"

	"github.com/belak/irc"
	"github.com/khades/servbot/commandHandlers"
	"github.com/khades/servbot/ircClient"
	"github.com/khades/servbot/models"
	"github.com/khades/servbot/repos"
)

var chatHandler irc.HandlerFunc = func(client *irc.Client, message *irc.Message) {
	//	log.Println(message.String())
	msgID, found := message.Tags.GetTag("msg-id")
	if found {
		switch msgID {
		case "room_mods":
			{
				commaIndex := strings.Index(message.Params[1], ":")
				if commaIndex != -1 {
					//				log.Printf("Channel %v: got mods list", message.Params[0])
					mods := strings.Split(message.Params[1][commaIndex+2:], ", ")
					repos.PushMods(message.Params[0][1:], mods)
				}
			}
		case "resub":
			{
				msgParamMonths, msgParamMonthsFound := message.Tags.GetTag("msg-param-months")
				user, userFound := message.Tags.GetTag("display-name")
				channel := message.Params[0]
				if msgParamMonthsFound && userFound && channel != "" {
					resubCount, resubCountError := strconv.Atoi(msgParamMonths)
					if resubCountError == nil {
						formedMessage := models.ChatMessage{
							Channel:          channel,
							User:             user,
							IsMod:            false,
							IsSub:            true,
							Date:             time.Now(),
							SubscriptionInfo: &models.SubscriptionInfo{Count: resubCount}}
						repos.LogMessage(formedMessage)
						channelInfo, error := repos.GetChannelInfo(channel)
						if error == nil && channelInfo.SubAlert.Enabled == true {
							messageBody := strings.TrimSpace(fmt.Sprintf("%s %s%s",
								channelInfo.SubAlert.RepeatPrefix,
								strings.Repeat(channelInfo.SubAlert.RepeatBody+" ", formedMessage.SubscriptionInfo.Count),
								channelInfo.SubAlert.RepeatPostfix))
							if messageBody != "" {
								IrcClientInstance.SendPublic(models.OutgoingMessage{
									Body:    messageBody,
									Channel: channel,
									User:    user})
							}
						}
						log.Printf("Channel %v: %v resubbed for %v months\n", formedMessage.Channel, formedMessage.User, formedMessage.SubscriptionInfo.Count)
					}
				}
			}
		}
	}

	if message.User == "twitchnotify" {
		log.Println("Got first sub")
		user := strings.Split(message.Params[1], " ")[0]
		channel := message.Params[0]
		if user != "" && channel != "" {
			formedMessage := models.ChatMessage{
				Channel:          channel,
				User:             user,
				IsMod:            false,
				IsSub:            true,
				IsPrime:          strings.Contains(message.String(), "Twitch Prime"),
				Date:             time.Now(),
				SubscriptionInfo: &models.SubscriptionInfo{Count: 1}}
			repos.LogMessage(formedMessage)
			channelInfo, error := repos.GetChannelInfo(channel)
			if error == nil && channelInfo.SubAlert.Enabled == true && channelInfo.SubAlert.FirstMessage != "" {
				IrcClientInstance.SendPublic(models.OutgoingMessage{
					Body:    channelInfo.SubAlert.FirstMessage,
					Channel: channel,
					User:    user})
			}
			log.Printf("Channel %v: %v subbed\n", formedMessage.Channel, formedMessage.User)
		}
	}
	if message.Command == "CLEARCHAT" {
		banDuration, banDurationFound := message.Tags.GetTag("ban-duration")
		intBanDuration := 0
		if banDurationFound {
			parsedValue, parseError := strconv.Atoi(banDuration)
			if parseError == nil {
				intBanDuration = parsedValue
			}
		}
		banReason, _ := message.Tags.GetTag("ban-reason")
		user := message.Params[1]
		channel := message.Params[0]
		formedMessage := models.ChatMessage{
			Channel: channel,
			User:    user,
			IsMod:   false,
			IsSub:   true,
			Date:    time.Now(),
			BanInfo: &models.BanInfo{Duration: intBanDuration, Reason: banReason}}
		repos.LogMessage(formedMessage)
		//	log.Printf("Channel %v: %v is banned for %v \n", channel, user, intBanDuration)
	}
	if message.Command == "PRIVMSG" {
		formedMessage := models.ChatMessage{
			Channel:     message.Params[0][1:],
			User:        message.User,
			MessageBody: message.Params[1],
			IsMod:       message.Tags["mod"] == "1" || message.User == "khadesru",
			IsSub:       message.Tags["subscriber"] == "1",
			IsPrime:     strings.Contains(message.Tags["badges"].Encode(), "premium/1"),
			Date:        time.Now()}
		repos.LogMessage(formedMessage)
		isCommand, commandBody := formedMessage.IsCommand()
		if isCommand {
			handlerFunction := commandHandlers.Router.Go(commandBody.Command)
			handlerFunction(true, &formedMessage, commandBody, &IrcClientInstance)
		}
	}

	if message.Command == "001" {
		client.Write("CAP REQ twitch.tv/tags")
		client.Write("CAP REQ twitch.tv/membership")
		client.Write("CAP REQ twitch.tv/commands")
		for _, value := range repos.Config.Channels {
			client.Write("JOIN #" + value)
		}
		IrcClientInstance = ircClient.IrcClient{Client: client, Bounces: make(map[string]time.Time), Ready: true}
		IrcClientInstance.SendModsCommand()
		log.Println("Bot is started")
	}
}
<|endoftext|>


<commit_before>package bouncer

import (
	"log"
	"os"
	"testing"

	"github.com/stretchr/testify/assert"
)

var testDB *DB

func TestMain(m *testing.M) {
	var err error
	testDB, err = NewDB("root@tcp(127.0.0.1:3306)/bouncer_test")
	if err != nil {
		log.Fatal(err)
	}
	defer testDB.Close()
	os.Exit(m.Run())
}

func TestAliasFor(t *testing.T) {
	res, err := testDB.AliasFor("firefox-latest")
	assert.NoError(t, err)
	assert.Equal(t, "Firefox", res)
}

func TestOSID(t *testing.T) {
	res, err := testDB.OSID("win64")
	assert.NoError(t, err)
	assert.Equal(t, "1", res)
}

func TestProductForLanguage(t *testing.T) {
	res, sslOnly, err := testDB.ProductForLanguage("Firefox", "en-US")
	assert.NoError(t, err)
	assert.False(t, sslOnly)
	assert.Equal(t, "1", res)

	res, sslOnly, err = testDB.ProductForLanguage("Firefox-SSL", "en-US")
	assert.NoError(t, err)
	assert.True(t, sslOnly)
	assert.Equal(t, "2", res)
}

func TestMirrors(t *testing.T) {
	mirrors, err := testDB.Mirrors(false, "en-US", "1", true)
	assert.NoError(t, err)
	assert.Equal(t, 1, len(mirrors))

	mirrors, err = testDB.Mirrors(true, "en-US", "1", true)
	assert.NoError(t, err)
	assert.Equal(t, 1, len(mirrors))
	assert.Equal(t, "2", mirrors[0].ID)
}
<commit_msg>test sentry queries<commit_after>package bouncer

import (
	"log"
	"os"
	"testing"

	"github.com/stretchr/testify/assert"
)

var testDB *DB

func TestMain(m *testing.M) {
	var err error
	testDB, err = NewDB("root@tcp(127.0.0.1:3306)/bouncer_test")
	if err != nil {
		log.Fatal(err)
	}
	defer testDB.Close()
	os.Exit(m.Run())
}

func TestAliasFor(t *testing.T) {
	res, err := testDB.AliasFor("firefox-latest")
	assert.NoError(t, err)
	assert.Equal(t, "Firefox", res)
}

func TestOSID(t *testing.T) {
	res, err := testDB.OSID("win64")
	assert.NoError(t, err)
	assert.Equal(t, "1", res)
}

func TestProductForLanguage(t *testing.T) {
	res, sslOnly, err := testDB.ProductForLanguage("Firefox", "en-US")
	assert.NoError(t, err)
	assert.False(t, sslOnly)
	assert.Equal(t, "1", res)

	res, sslOnly, err = testDB.ProductForLanguage("Firefox-SSL", "en-US")
	assert.NoError(t, err)
	assert.True(t, sslOnly)
	assert.Equal(t, "2", res)
}

func TestMirrors(t *testing.T) {
	mirrors, err := testDB.Mirrors(false, "en-US", "1", true)
	assert.NoError(t, err)
	assert.Len(t, mirrors, 1)

	mirrors, err = testDB.Mirrors(true, "en-US", "1", true)
	assert.NoError(t, err)
	assert.Len(t, mirrors, 1)
	assert.Equal(t, "2", mirrors[0].ID)
}

func LocationsActive(t *testing.T) {
	locations, err := testDB.LocationsActive(false)
	assert.NoError(t, err)
	assert.Len(t, locations, 3)
}

func MirrorsActive(t *testing.T) {
	mirrors, err := testDB.MirrorsActive("")
	assert.NoError(t, err)
	assert.Len(t, mirrors, 2)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/inominate/eve-api-proxy/apicache"
	"log"
	"net"
	"net/http"
	"path"
	"runtime"
	"strings"
	"time"
)

type APIMux struct{}

func makeParams(req *http.Request) map[string]string {
	params := make(map[string]string)
	for key, val := range req.Form {
		params[key] = val[0]
	}

	// force is for internal use only!
	// other internal use flags should be deleted here
	delete(params, "force")

	return params
}

func logRequest(req *http.Request, url string, params map[string]string, resp *apicache.Response, startTime time.Time) {
	remoteAddr, _, err := net.SplitHostPort(req.RemoteAddr)
	if err != nil {
		remoteAddr = req.RemoteAddr
	}
	// Should we use a different header for our real address?
	if conf.RealRemoteAddrHeader != "" && req.Header.Get(conf.RealRemoteAddrHeader) != "" {
		if conf.ProxyAddr == "" || remoteAddr == conf.ProxyAddr {
			remoteAddr = req.Header.Get(conf.RealRemoteAddrHeader)
		}
	}

	if resp == nil {
		if conf.Logging.LogRequests && !debug {
			log.Printf("%s - Invalid Request for %s", remoteAddr, url)
		}
		debugLog.Printf("%s - Invalid Request for %s - %+v", remoteAddr, url, req)
		return
	}

	var errorStr string
	if resp.Error.ErrorCode != 0 {
		errorStr = fmt.Sprintf("Error %d: %s", resp.Error.ErrorCode, resp.Error.ErrorText)
	}

	logParams := ""
	var paramVal string
	for k, _ := range params {
		// vCode censorship
		if conf.Logging.CensorLog && strings.ToLower(k) == "vcode" {
			paramVal = params[k][0:8] + "..."
		} else {
			paramVal = params[k]
		}
		logParams = fmt.Sprintf("%s&%s=%s", logParams, k, paramVal)
	}

	usingParams := ""
	if logParams != "" {
		usingParams = "?"
	}
	log.Printf("%s - %s%s%s - http: %d - expires: %s - %.2f seconds - %s",
		remoteAddr, url, usingParams, logParams, resp.HTTPCode,
		resp.Expires.Format("2006-01-02 15:04:05"), time.Since(startTime).Seconds(),
		errorStr)
}

// The muxer for the whole operation.  Everything starts here.
func (a APIMux) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	var resp *apicache.Response
	startTime := time.Now()

	req.ParseForm()

	url := path.Clean(req.URL.Path)
	params := makeParams(req)

	debugLog.Printf("Starting request for %s...", url)

	w.Header().Add("Content-Type", "text/xml")
	if handler, valid := validPages[strings.ToLower(url)]; valid {
		if handler == nil {
			handler = defaultHandler
		}

		resp = handler(url, params)

		w.WriteHeader(resp.HTTPCode)
		w.Write(resp.Data)
	} else {
		w.WriteHeader(404)
		w.Write(apicache.SynthesizeAPIError(404, "Invalid API page.", 24*time.Hour))
	}

	if conf.Logging.LogRequests || resp.HTTPCode != 200 {
		logRequest(req, url, params, resp, startTime)
	}

	if debug && time.Since(startTime).Seconds() > 10 {
		debugLog.Printf("Slow Request took %.2f seconds:", time.Since(startTime).Seconds())
		debugLog.Printf("%+v", req)
	}
}

func LogStats() {
	PrintWorkerStats()
	dc.LogStats()
	LogMemStats()
}

func LogMemStats() {
	var m runtime.MemStats
	runtime.ReadMemStats(&m)
	log.Printf("Alloc: %dkb Sys: %dkb", m.Alloc/1024, m.Sys/1024)
	log.Printf("HeapAlloc: %dkb HeapSys: %dkb", m.HeapAlloc/1024, m.HeapSys/1024)
}
<commit_msg>nil pointer bugfix<commit_after>package main

import (
	"fmt"
	"github.com/inominate/eve-api-proxy/apicache"
	"log"
	"net"
	"net/http"
	"path"
	"runtime"
	"strings"
	"time"
)

type APIMux struct{}

func makeParams(req *http.Request) map[string]string {
	params := make(map[string]string)
	for key, val := range req.Form {
		params[key] = val[0]
	}

	// force is for internal use only!
	// other internal use flags should be deleted here
	delete(params, "force")

	return params
}

func logRequest(req *http.Request, url string, params map[string]string, resp *apicache.Response, startTime time.Time) {
	remoteAddr, _, err := net.SplitHostPort(req.RemoteAddr)
	if err != nil {
		remoteAddr = req.RemoteAddr
	}
	// Should we use a different header for our real address?
	if conf.RealRemoteAddrHeader != "" && req.Header.Get(conf.RealRemoteAddrHeader) != "" {
		if conf.ProxyAddr == "" || remoteAddr == conf.ProxyAddr {
			remoteAddr = req.Header.Get(conf.RealRemoteAddrHeader)
		}
	}

	if resp == nil {
		if conf.Logging.LogRequests && !debug {
			log.Printf("%s - Invalid Request for %s", remoteAddr, url)
		}
		debugLog.Printf("%s - Invalid Request for %s - %+v", remoteAddr, url, req)
		return
	}

	var errorStr string
	if resp.Error.ErrorCode != 0 {
		errorStr = fmt.Sprintf("Error %d: %s", resp.Error.ErrorCode, resp.Error.ErrorText)
	}

	logParams := ""
	var paramVal string
	for k, _ := range params {
		// vCode censorship
		if conf.Logging.CensorLog && strings.ToLower(k) == "vcode" {
			paramVal = params[k][0:8] + "..."
		} else {
			paramVal = params[k]
		}
		logParams = fmt.Sprintf("%s&%s=%s", logParams, k, paramVal)
	}

	usingParams := ""
	if logParams != "" {
		usingParams = "?"
	}
	log.Printf("%s - %s%s%s - http: %d - expires: %s - %.2f seconds - %s",
		remoteAddr, url, usingParams, logParams, resp.HTTPCode,
		resp.Expires.Format("2006-01-02 15:04:05"), time.Since(startTime).Seconds(),
		errorStr)
}

// The muxer for the whole operation.  Everything starts here.
func (a APIMux) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	var resp *apicache.Response
	startTime := time.Now()

	req.ParseForm()

	url := path.Clean(req.URL.Path)
	params := makeParams(req)

	debugLog.Printf("Starting request for %s...", url)

	w.Header().Add("Content-Type", "text/xml")
	if handler, valid := validPages[strings.ToLower(url)]; valid {
		if handler == nil {
			handler = defaultHandler
		}

		resp = handler(url, params)

		w.WriteHeader(resp.HTTPCode)
		w.Write(resp.Data)
	} else {
		w.WriteHeader(404)
		w.Write(apicache.SynthesizeAPIError(404, "Invalid API page.", 24*time.Hour))
	}

	if conf.Logging.LogRequests || (resp != nil && resp.HTTPCode != 200) {
		logRequest(req, url, params, resp, startTime)
	}

	if debug && time.Since(startTime).Seconds() > 10 {
		debugLog.Printf("Slow Request took %.2f seconds:", time.Since(startTime).Seconds())
		debugLog.Printf("%+v", req)
	}
}

func LogStats() {
	PrintWorkerStats()
	dc.LogStats()
	LogMemStats()
}

func LogMemStats() {
	var m runtime.MemStats
	runtime.ReadMemStats(&m)
	log.Printf("Alloc: %dkb Sys: %dkb", m.Alloc/1024, m.Sys/1024)
	log.Printf("HeapAlloc: %dkb HeapSys: %dkb", m.HeapAlloc/1024, m.HeapSys/1024)
}
<|endoftext|>