<commit_before>// This is a client that writes out to a file, and optionally rolls the file

package main

import (
	"compress/gzip"
	"errors"
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"path"
	"path/filepath"
	"strings"
	"syscall"
	"time"

	"github.com/bitly/go-nsq"
	"github.com/bitly/nsq/util"
)

var (
	showVersion = flag.Bool("version", false, "print version string")

	topic       = flag.String("topic", "", "nsq topic")
	channel     = flag.String("channel", "nsq_to_file", "nsq channel")
	maxInFlight = flag.Int("max-in-flight", 200, "max number of messages to allow in flight")

	outputDir      = flag.String("output-dir", "/tmp", "directory to write output files to")
	datetimeFormat = flag.String("datetime-format", "%Y-%m-%d_%H", "strftime compatible format for <DATETIME> in filename format")
	filenameFormat = flag.String("filename-format", "<TOPIC>.<HOST><GZIPREV>.<DATETIME>.log", "output filename format (<TOPIC>, <HOST>, <DATETIME>, <GZIPREV> are replaced. <GZIPREV> is a suffix when an existing gzip file already exists)")
	hostIdentifier = flag.String("host-identifier", "", "value to output in log filename in place of hostname. <SHORT_HOST> and <HOSTNAME> are valid replacement tokens")
	gzipLevel      = flag.Int("gzip-level", 6, "gzip compression level (1-9, 1=BestSpeed, 9=BestCompression)")
	gzipEnabled    = flag.Bool("gzip", false, "gzip output files.")
	skipEmptyFiles = flag.Bool("skip-empty-files", false, "Skip writting empty files")

	readerOpts       = util.StringArray{}
	nsqdTCPAddrs     = util.StringArray{}
	lookupdHTTPAddrs = util.StringArray{}

	// TODO: remove, deprecated
	gzipCompression = flag.Int("gzip-compression", 3, "(deprecated) use --gzip-level, gzip compression level (1 = BestSpeed, 2 = BestCompression, 3 = DefaultCompression)")
	verbose         = flag.Bool("verbose", false, "(depgrecated) use --reader-opt=verbose")
)

func init() {
	flag.Var(&readerOpts, "reader-opt", "option to passthrough to nsq.Reader (may be given multiple times)")
	flag.Var(&nsqdTCPAddrs, "nsqd-tcp-address", "nsqd TCP address (may be given multiple times)")
	flag.Var(&lookupdHTTPAddrs, "lookupd-http-address", "lookupd HTTP address (may be given multiple times)")
}

type FileLogger struct {
	out              *os.File
	gzipWriter       *gzip.Writer
	lastFilename     string
	logChan          chan *Message
	compressionLevel int
	gzipEnabled      bool
	filenameFormat   string

	ExitChan chan int
}

type Message struct {
	*nsq.Message
	returnChannel chan *nsq.FinishedMessage
}

type SyncMsg struct {
	m             *nsq.FinishedMessage
	returnChannel chan *nsq.FinishedMessage
}

func (l *FileLogger) HandleMessage(m *nsq.Message, responseChannel chan *nsq.FinishedMessage) {
	l.logChan <- &Message{m, responseChannel}
}

func (f *FileLogger) router(r *nsq.Reader, termChan chan os.Signal, hupChan chan os.Signal) {
	pos := 0
	output := make([]*Message, r.MaxInFlight())
	sync := false
	ticker := time.NewTicker(time.Duration(30) * time.Second)
	closing := false
	closeFile := false
	exit := false

	for {
		select {
		case <-r.ExitChan:
			sync = true
			closeFile = true
			exit = true
		case <-termChan:
			ticker.Stop()
			r.Stop()
			sync = true
			closing = true
		case <-hupChan:
			sync = true
			closeFile = true
		case <-ticker.C:
			if f.needsFileRotate() {
				if *skipEmptyFiles {
					closeFile = true
				} else {
					f.updateFile()
				}
			}
			sync = true
		case m := <-f.logChan:
			if f.updateFile() {
				sync = true
			}
			_, err := f.Write(m.Body)
			if err != nil {
				log.Fatalf("ERROR: writing message to disk - %s", err.Error())
			}
			_, err = f.Write([]byte("\n"))
			if err != nil {
				log.Fatalf("ERROR: writing newline to disk - %s", err.Error())
			}
			output[pos] = m
			pos++
			if pos == r.MaxInFlight() {
				sync = true
			}
		}

		if closing || sync || r.IsStarved() {
			if pos > 0 {
				log.Printf("syncing %d records to disk", pos)
				err := f.Sync()
				if err != nil {
					log.Fatalf("ERROR: failed syncing messages - %s", err.Error())
				}
				for pos > 0 {
					pos--
					m := output[pos]
					m.returnChannel <- &nsq.FinishedMessage{m.Id, 0, true}
					output[pos] = nil
				}
			}
			sync = false
		}

		if closeFile {
			f.Close()
			closeFile = false
		}
		if exit {
			close(f.ExitChan)
			break
		}
	}
}

func (f *FileLogger) Close() {
	if f.out != nil {
		if f.gzipWriter != nil {
			f.gzipWriter.Close()
		}
		f.out.Close()
		f.out = nil
	}
}

func (f *FileLogger) Write(p []byte) (n int, err error) {
	if f.gzipWriter != nil {
		return f.gzipWriter.Write(p)
	}
	return f.out.Write(p)
}

func (f *FileLogger) Sync() error {
	var err error
	if f.gzipWriter != nil {
		f.gzipWriter.Close()
		err = f.out.Sync()
		f.gzipWriter, _ = gzip.NewWriterLevel(f.out, f.compressionLevel)
	} else {
		err = f.out.Sync()
	}
	return err
}

func (f *FileLogger) calculateCurrentFilename() string {
	t := time.Now()

	datetime := strftime(*datetimeFormat, t)
	filename := strings.Replace(f.filenameFormat, "<DATETIME>", datetime, -1)
	if !f.gzipEnabled {
		filename = strings.Replace(filename, "<GZIPREV>", "", -1)
	}
	return filename

}

func (f *FileLogger) needsFileRotate() bool {
	filename := f.calculateCurrentFilename()
	return filename != f.lastFilename
}

func (f *FileLogger) updateFile() bool {
	filename := f.calculateCurrentFilename()
	maxGzipRevisions := 1000
	if filename != f.lastFilename || f.out == nil {
		f.Close()
		os.MkdirAll(*outputDir, 777)
		var newFile *os.File
		var err error
		if f.gzipEnabled {
			// for gzip files, we never append to an existing file
			// we try to create different revisions, replacing <GZIPREV> in the filename
			for gzipRevision := 0; gzipRevision < maxGzipRevisions; gzipRevision += 1 {
				var revisionSuffix string
				if gzipRevision > 0 {
					revisionSuffix = fmt.Sprintf("-%d", gzipRevision)
				}
				tempFilename := strings.Replace(filename, "<GZIPREV>", revisionSuffix, -1)
				fullPath := path.Join(*outputDir, tempFilename)
				dir, _ := filepath.Split(fullPath)
				if dir != "" {
					err = os.MkdirAll(dir, 777)
					if err != nil {
						log.Fatalf("ERROR: %s Unable to create %s", err, dir)
					}
				}
				newFile, err = os.OpenFile(fullPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666)
				if err != nil && os.IsExist(err) {
					log.Printf("INFO: file already exists: %s", fullPath)
					continue
				}
				if err != nil {
					log.Fatalf("ERROR: %s Unable to open %s", err, fullPath)
				}
				log.Printf("opening %s", fullPath)
				break
			}
			if newFile == nil {
				log.Fatalf("ERROR: Unable to open a new gzip file after %d tries", maxGzipRevisions)
			}
		} else {
			log.Printf("opening %s/%s", *outputDir, filename)
			fullPath := path.Join(*outputDir, filename)
			dir, _ := filepath.Split(fullPath)
			if dir != "" {
				err = os.MkdirAll(dir, 777)
				if err != nil {
					log.Fatalf("ERROR: %s Unable to create %s", err, dir)
				}
			}
			newFile, err = os.OpenFile(fullPath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
			if err != nil {
				log.Fatal(err)
			}
		}

		f.out = newFile
		f.lastFilename = filename
		if f.gzipEnabled {
			f.gzipWriter, _ = gzip.NewWriterLevel(newFile, f.compressionLevel)
		}
		return true
	}

	return false
}

func NewFileLogger(gzipEnabled bool, compressionLevel int, filenameFormat string) (*FileLogger, error) {
	if gzipEnabled && strings.Index(filenameFormat, "<GZIPREV>") == -1 {
		return nil, errors.New("missing <GZIPREV> in filenameFormat")
	}

	hostname, err := os.Hostname()
	if err != nil {
		return nil, err
	}
	shortHostname := strings.Split(hostname, ".")[0]
	identifier := shortHostname
	if len(*hostIdentifier) != 0 {
		identifier = strings.Replace(*hostIdentifier, "<SHORT_HOST>", shortHostname, -1)
		identifier = strings.Replace(identifier, "<HOSTNAME>", hostname, -1)
	}
	filenameFormat = strings.Replace(filenameFormat, "<TOPIC>", *topic, -1)
	filenameFormat = strings.Replace(filenameFormat, "<HOST>", identifier, -1)
	if gzipEnabled && !strings.HasSuffix(filenameFormat, ".gz") {
		filenameFormat = filenameFormat + ".gz"
	}

	f := &FileLogger{
		logChan:          make(chan *Message, 1),
		compressionLevel: compressionLevel,
		filenameFormat:   filenameFormat,
		gzipEnabled:      gzipEnabled,
		ExitChan:         make(chan int),
	}
	return f, nil
}

func hasArg(s string) bool {
	for _, arg := range os.Args {
		if strings.Contains(arg, s) {
			return true
		}
	}
	return false
}

func main() {
	flag.Parse()

	if *showVersion {
		fmt.Printf("nsq_to_file v%s\n", util.BINARY_VERSION)
		return
	}

	if *topic == "" || *channel == "" {
		log.Fatalf("--topic and --channel are required")
	}

	if len(nsqdTCPAddrs) == 0 && len(lookupdHTTPAddrs) == 0 {
		log.Fatalf("--nsqd-tcp-address or --lookupd-http-address required.")
	}
	if len(nsqdTCPAddrs) != 0 && len(lookupdHTTPAddrs) != 0 {
		log.Fatalf("use --nsqd-tcp-address or --lookupd-http-address not both")
	}

	if *gzipLevel < 1 || *gzipLevel > 9 {
		log.Fatalf("invalid --gzip-level value (%d), should be 1-9", *gzipLevel)
	}

	// TODO: remove, deprecated
	if hasArg("gzip-compression") {
		log.Printf("WARNING: --gzip-compression is deprecated in favor of --gzip-level")
		switch *gzipCompression {
		case 1:
			*gzipLevel = gzip.BestSpeed
		case 2:
			*gzipLevel = gzip.BestCompression
		case 3:
			*gzipLevel = gzip.DefaultCompression
		default:
			log.Fatalf("invalid --gzip-compression value (%d), should be 1,2,3", *gzipCompression)
		}
	}

	hupChan := make(chan os.Signal, 1)
	termChan := make(chan os.Signal, 1)
	signal.Notify(hupChan, syscall.SIGHUP)
	signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM)

	f, err := NewFileLogger(*gzipEnabled, *gzipLevel, *filenameFormat)
	if err != nil {
		log.Fatal(err.Error())
	}

	r, err := nsq.NewReader(*topic, *channel)
	if err != nil {
		log.Fatalf(err.Error())
	}
	err = util.ParseReaderOpts(r, readerOpts)
	if err != nil {
		log.Fatalf(err.Error())
	}
	r.SetMaxInFlight(*maxInFlight)
	r.AddAsyncHandler(f)

	// TODO: remove, deprecated
	if hasArg("verbose") {
		log.Printf("WARNING: --verbose is deprecated in favor of --reader-opt=verbose")
		r.Configure("verbose", true)
	}

	go f.router(r, termChan, hupChan)

	for _, addrString := range nsqdTCPAddrs {
		err := r.ConnectToNSQ(addrString)
		if err != nil {
			log.Fatalf(err.Error())
		}
	}

	for _, addrString := range lookupdHTTPAddrs {
		log.Printf("lookupd addr %s", addrString)
		err := r.ConnectToLookupd(addrString)
		if err != nil {
			log.Fatalf(err.Error())
		}
	}

	<-f.ExitChan
}
<commit_msg>Allow nsq_to_file to listen to many topics.<commit_after>// This is a client that writes out to a file, and optionally rolls the file

package main

import (
	"compress/gzip"
	"errors"
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/bitly/go-nsq"
	"github.com/bitly/nsq/util"
	"github.com/bitly/nsq/util/lookupd"
)

var (
	showVersion = flag.Bool("version", false, "print version string")

	channel     = flag.String("channel", "nsq_to_file", "nsq channel")
	maxInFlight = flag.Int("max-in-flight", 200, "max number of messages to allow in flight")

	outputDir      = flag.String("output-dir", "/tmp", "directory to write output files to")
	datetimeFormat = flag.String("datetime-format", "%Y-%m-%d_%H", "strftime compatible format for <DATETIME> in filename format")
	filenameFormat = flag.String("filename-format", "<TOPIC>.<HOST><GZIPREV>.<DATETIME>.log", "output filename format (<TOPIC>, <HOST>, <DATETIME>, <GZIPREV> are replaced. <GZIPREV> is a suffix when an existing gzip file already exists)")
	hostIdentifier = flag.String("host-identifier", "", "value to output in log filename in place of hostname. <SHORT_HOST> and <HOSTNAME> are valid replacement tokens")
	gzipLevel      = flag.Int("gzip-level", 6, "gzip compression level (1-9, 1=BestSpeed, 9=BestCompression)")
	gzipEnabled    = flag.Bool("gzip", false, "gzip output files.")
	skipEmptyFiles = flag.Bool("skip-empty-files", false, "Skip writting empty files")
	topicPollRate  = flag.Duration("topic-refresh", time.Minute, "how frequently the topic list should be refreshed")

	readerOpts       = util.StringArray{}
	nsqdTCPAddrs     = util.StringArray{}
	lookupdHTTPAddrs = util.StringArray{}
	topics           = util.StringArray{}

	// TODO: remove, deprecated
	gzipCompression = flag.Int("gzip-compression", 3, "(deprecated) use --gzip-level, gzip compression level (1 = BestSpeed, 2 = BestCompression, 3 = DefaultCompression)")
	verbose         = flag.Bool("verbose", false, "(depgrecated) use --reader-opt=verbose")
)

func init() {
	flag.Var(&readerOpts, "reader-opt", "option to passthrough to nsq.Reader (may be given multiple times)")
	flag.Var(&nsqdTCPAddrs, "nsqd-tcp-address", "nsqd TCP address (may be given multiple times)")
	flag.Var(&lookupdHTTPAddrs, "lookupd-http-address", "lookupd HTTP address (may be given multiple times)")
	flag.Var(&topics, "topic", "nsq topic (may be given multiple times)")
}

type FileLogger struct {
	out              *os.File
	gzipWriter       *gzip.Writer
	lastFilename     string
	logChan          chan *Message
	compressionLevel int
	gzipEnabled      bool
	filenameFormat   string

	ExitChan chan int
	termChan chan bool
	hupChan  chan bool
}

type Message struct {
	*nsq.Message
	returnChannel chan *nsq.FinishedMessage
}

type SyncMsg struct {
	m             *nsq.FinishedMessage
	returnChannel chan *nsq.FinishedMessage
}

type ReaderFileLogger struct {
	F *FileLogger
	R *nsq.Reader
}

type TopicDiscoverer struct {
	topics   map[string]*ReaderFileLogger
	termChan chan os.Signal
	hupChan  chan os.Signal
	wg       sync.WaitGroup
}

func newTopicDiscoverer() *TopicDiscoverer {
	return &TopicDiscoverer{
		topics:   make(map[string]*ReaderFileLogger),
		termChan: make(chan os.Signal),
		hupChan:  make(chan os.Signal),
	}
}

func (l *FileLogger) HandleMessage(m *nsq.Message, responseChannel chan *nsq.FinishedMessage) {
	l.logChan <- &Message{m, responseChannel}
}

func (f *FileLogger) router(r *nsq.Reader) {
	pos := 0
	output := make([]*Message, r.MaxInFlight())
	sync := false
	ticker := time.NewTicker(time.Duration(30) * time.Second)
	closing := false
	closeFile := false
	exit := false

	for {
		select {
		case <-r.ExitChan:
			sync = true
			closeFile = true
			exit = true
		case <-f.termChan:
			ticker.Stop()
			r.Stop()
			sync = true
			closing = true
		case <-f.hupChan:
			sync = true
			closeFile = true
		case <-ticker.C:
			if f.needsFileRotate() {
				if *skipEmptyFiles {
					closeFile = true
				} else {
					f.updateFile()
				}
			}
			sync = true
		case m := <-f.logChan:
			if f.updateFile() {
				sync = true
			}
			_, err := f.Write(m.Body)
			if err != nil {
				log.Fatalf("ERROR: writing message to disk - %s", err.Error())
			}
			_, err = f.Write([]byte("\n"))
			if err != nil {
				log.Fatalf("ERROR: writing newline to disk - %s", err.Error())
			}
			output[pos] = m
			pos++
			if pos == r.MaxInFlight() {
				sync = true
			}
		}

		if closing || sync || r.IsStarved() {
			if pos > 0 {
				log.Printf("syncing %d records to disk", pos)
				err := f.Sync()
				if err != nil {
					log.Fatalf("ERROR: failed syncing messages - %s", err.Error())
				}
				for pos > 0 {
					pos--
					m := output[pos]
					m.returnChannel <- &nsq.FinishedMessage{m.Id, 0, true}
					output[pos] = nil
				}
			}
			sync = false
		}

		if closeFile {
			f.Close()
			closeFile = false
		}
		if exit {
			close(f.ExitChan)
			break
		}
	}
}

func (f *FileLogger) Close() {
	if f.out != nil {
		if f.gzipWriter != nil {
			f.gzipWriter.Close()
		}
		f.out.Close()
		f.out = nil
	}
}

func (f *FileLogger) Write(p []byte) (n int, err error) {
	if f.gzipWriter != nil {
		return f.gzipWriter.Write(p)
	}
	return f.out.Write(p)
}

func (f *FileLogger) Sync() error {
	var err error
	if f.gzipWriter != nil {
		f.gzipWriter.Close()
		err = f.out.Sync()
		f.gzipWriter, _ = gzip.NewWriterLevel(f.out, f.compressionLevel)
	} else {
		err = f.out.Sync()
	}
	return err
}

func (f *FileLogger) calculateCurrentFilename() string {
	t := time.Now()

	datetime := strftime(*datetimeFormat, t)
	filename := strings.Replace(f.filenameFormat, "<DATETIME>", datetime, -1)
	if !f.gzipEnabled {
		filename = strings.Replace(filename, "<GZIPREV>", "", -1)
	}
	return filename

}

func (f *FileLogger) needsFileRotate() bool {
	filename := f.calculateCurrentFilename()
	return filename != f.lastFilename
}

func (f *FileLogger) updateFile() bool {
	filename := f.calculateCurrentFilename()
	maxGzipRevisions := 1000
	if filename != f.lastFilename || f.out == nil {
		f.Close()
		os.MkdirAll(*outputDir, 777)
		var newFile *os.File
		var err error
		if f.gzipEnabled {
			// for gzip files, we never append to an existing file
			// we try to create different revisions, replacing <GZIPREV> in the filename
			for gzipRevision := 0; gzipRevision < maxGzipRevisions; gzipRevision += 1 {
				var revisionSuffix string
				if gzipRevision > 0 {
					revisionSuffix = fmt.Sprintf("-%d", gzipRevision)
				}
				tempFilename := strings.Replace(filename, "<GZIPREV>", revisionSuffix, -1)
				fullPath := path.Join(*outputDir, tempFilename)
				dir, _ := filepath.Split(fullPath)
				if dir != "" {
					err = os.MkdirAll(dir, 777)
					if err != nil {
						log.Fatalf("ERROR: %s Unable to create %s", err, dir)
					}
				}
				newFile, err = os.OpenFile(fullPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666)
				if err != nil && os.IsExist(err) {
					log.Printf("INFO: file already exists: %s", fullPath)
					continue
				}
				if err != nil {
					log.Fatalf("ERROR: %s Unable to open %s", err, fullPath)
				}
				log.Printf("opening %s", fullPath)
				break
			}
			if newFile == nil {
				log.Fatalf("ERROR: Unable to open a new gzip file after %d tries", maxGzipRevisions)
			}
		} else {
			log.Printf("opening %s/%s", *outputDir, filename)
			fullPath := path.Join(*outputDir, filename)
			dir, _ := filepath.Split(fullPath)
			if dir != "" {
				err = os.MkdirAll(dir, 777)
				if err != nil {
					log.Fatalf("ERROR: %s Unable to create %s", err, dir)
				}
			}
			newFile, err = os.OpenFile(fullPath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
			if err != nil {
				log.Fatal(err)
			}
		}

		f.out = newFile
		f.lastFilename = filename
		if f.gzipEnabled {
			f.gzipWriter, _ = gzip.NewWriterLevel(newFile, f.compressionLevel)
		}
		return true
	}

	return false
}

func NewFileLogger(gzipEnabled bool, compressionLevel int, filenameFormat, topic string) (*FileLogger, error) {
	if gzipEnabled && strings.Index(filenameFormat, "<GZIPREV>") == -1 {
		return nil, errors.New("missing <GZIPREV> in filenameFormat")
	}

	hostname, err := os.Hostname()
	if err != nil {
		return nil, err
	}
	shortHostname := strings.Split(hostname, ".")[0]
	identifier := shortHostname
	if len(*hostIdentifier) != 0 {
		identifier = strings.Replace(*hostIdentifier, "<SHORT_HOST>", shortHostname, -1)
		identifier = strings.Replace(identifier, "<HOSTNAME>", hostname, -1)
	}
	filenameFormat = strings.Replace(filenameFormat, "<TOPIC>", topic, -1)
	filenameFormat = strings.Replace(filenameFormat, "<HOST>", identifier, -1)
	if gzipEnabled && !strings.HasSuffix(filenameFormat, ".gz") {
		filenameFormat = filenameFormat + ".gz"
	}

	f := &FileLogger{
		logChan:          make(chan *Message, 1),
		compressionLevel: compressionLevel,
		filenameFormat:   filenameFormat,
		gzipEnabled:      gzipEnabled,
		ExitChan:         make(chan int),
		termChan:         make(chan bool),
		hupChan:          make(chan bool),
	}
	return f, nil
}

func hasArg(s string) bool {
	for _, arg := range os.Args {
		if strings.Contains(arg, s) {
			return true
		}
	}
	return false
}

func newReaderFileLogger(topic string) (*ReaderFileLogger, error) {
	f, err := NewFileLogger(*gzipEnabled, *gzipLevel, *filenameFormat, topic)
	if err != nil {
		return nil, err
	}

	r, err := nsq.NewReader(topic, *channel)
	if err != nil {
		return nil, err
	}
	err = util.ParseReaderOpts(r, readerOpts)
	if err != nil {
		return nil, err
	}
	r.SetMaxInFlight(*maxInFlight)
	r.AddAsyncHandler(f)

	// TODO: remove, deprecated
	if hasArg("verbose") {
		log.Printf("WARNING: --verbose is deprecated in favor of --reader-opt=verbose")
		r.Configure("verbose", true)
	}

	for _, addrString := range nsqdTCPAddrs {
		err := r.ConnectToNSQ(addrString)
		if err != nil {
			log.Fatalf(err.Error())
		}
	}

	for _, addrString := range lookupdHTTPAddrs {
		log.Printf("lookupd addr %s", addrString)
		err := r.ConnectToLookupd(addrString)
		if err != nil {
			log.Fatalf(err.Error())
		}
	}

	return &ReaderFileLogger{
		R: r,
		F: f,
	}, nil
}

func (t *TopicDiscoverer) startTopicRouter(logger *ReaderFileLogger) {
	t.wg.Add(1)
	defer t.wg.Done()
	go logger.F.router(logger.R)
	<-logger.F.ExitChan
}

func (t *TopicDiscoverer) syncTopics(addrs []string) {
	newTopics, err := lookupd.GetLookupdTopics(addrs)
	if err != nil {
		log.Print("ERROR: could not retrieve topic list: %s", err)
	}
	for _, topic := range newTopics {
		if _, ok := t.topics[topic]; !ok {
			logger, err := newReaderFileLogger(topic)
			if err != nil {
				log.Printf("ERROR: couldn't create logger for new topic %s: %s", topic, err.Error())
				continue
			}
			t.topics[topic] = logger
			go t.startTopicRouter(logger)
		}
	}
}

func (t *TopicDiscoverer) stopReaderFileLoggers() {
	for _, topic := range t.topics {
		topic.F.termChan <- true
	}
}

func (t *TopicDiscoverer) hupReaderFileLoggers() {
	for _, topic := range t.topics {
		topic.F.hupChan <- true
	}
}

func (t *TopicDiscoverer) watch(addrs []string, sync bool) {
	ticker := time.Tick(*topicPollRate)
	for {
		select {
		case <-ticker:
			if sync {
				t.syncTopics(addrs)
			}
		case <-t.termChan:
			t.stopReaderFileLoggers()
			t.wg.Wait()
			return
		case <-t.hupChan:
			t.hupReaderFileLoggers()
		}
	}
}

func main() {
	flag.Parse()

	if *showVersion {
		fmt.Printf("nsq_to_file v%s\n", util.BINARY_VERSION)
		return
	}

	if *channel == "" {
		log.Fatalf("--channel is required")
	}

	var topicsFromNSQLookupd bool

	if len(nsqdTCPAddrs) == 0 && len(lookupdHTTPAddrs) == 0 {
		log.Fatalf("--nsqd-tcp-address or --lookupd-http-address required.")
	}
	if len(nsqdTCPAddrs) != 0 && len(lookupdHTTPAddrs) != 0 {
		log.Fatalf("use --nsqd-tcp-address or --lookupd-http-address not both")
	}

	if *gzipLevel < 1 || *gzipLevel > 9 {
		log.Fatalf("invalid --gzip-level value (%d), should be 1-9", *gzipLevel)
	}

	// TODO: remove, deprecated
	if hasArg("gzip-compression") {
		log.Printf("WARNING: --gzip-compression is deprecated in favor of --gzip-level")
		switch *gzipCompression {
		case 1:
			*gzipLevel = gzip.BestSpeed
		case 2:
			*gzipLevel = gzip.BestCompression
		case 3:
			*gzipLevel = gzip.DefaultCompression
		default:
			log.Fatalf("invalid --gzip-compression value (%d), should be 1,2,3", *gzipCompression)
		}
	}

	discoverer := newTopicDiscoverer()

	signal.Notify(discoverer.hupChan, syscall.SIGHUP)
	signal.Notify(discoverer.termChan, syscall.SIGINT, syscall.SIGTERM)

	if len(topics) < 1 {
		if len(lookupdHTTPAddrs) < 1 {
			log.Fatalf("use --topic to list at least one topic to subscribe to or specify at least one --lookupd-http-address to subscribe to all its topics")
		}
		topicsFromNSQLookupd = true
		var err error
		topics, err = lookupd.GetLookupdTopics(lookupdHTTPAddrs)
		if err != nil {
			log.Fatalf("ERROR: could not retrieve topic list: %s", err)
		}
	}

	for _, topic := range topics {
		logger, err := newReaderFileLogger(topic)
		if err != nil {
			log.Fatalf("ERROR: couldn't create logger for topic %s: %s", topic, err.Error())
		}
		discoverer.topics[topic] = logger
		go discoverer.startTopicRouter(logger)
	}

	discoverer.watch(lookupdHTTPAddrs, topicsFromNSQLookupd)
}
<|endoftext|>


<commit_before>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package e2e

import (
	"fmt"
	"os"
	"strings"
	"testing"
	"time"

	"github.com/coreos/etcd/pkg/testutil"
)

func TestCtlV3Set(t *testing.T) {
	testCtlV3Set(t, &configNoTLS, 3*time.Second, false)
}

func TestCtlV3SetZeroTimeout(t *testing.T) {
	testCtlV3Set(t, &configNoTLS, 0, false)
}

func TestCtlV3SetTimeout(t *testing.T) {
	testCtlV3Set(t, &configNoTLS, time.Nanosecond, false)
}

func TestCtlV3SetPeerTLS(t *testing.T) {
	testCtlV3Set(t, &configPeerTLS, 3*time.Second, false)
}

func TestCtlV3SetQuorum(t *testing.T) {
	testCtlV3Set(t, &configNoTLS, 3*time.Second, true)
}

func TestCtlV3SetQuorumZeroTimeout(t *testing.T) {
	testCtlV3Set(t, &configNoTLS, 0, true)
}

func TestCtlV3SetQuorumTimeout(t *testing.T) {
	testCtlV3Set(t, &configNoTLS, time.Nanosecond, true)
}

func TestCtlV3SetPeerTLSQuorum(t *testing.T) {
	testCtlV3Set(t, &configPeerTLS, 3*time.Second, true)
}

func testCtlV3Set(t *testing.T, cfg *etcdProcessClusterConfig, dialTimeout time.Duration, quorum bool) {
	defer testutil.AfterTest(t)

	os.Setenv("ETCDCTL_API", "3")
	epc := setupCtlV3Test(t, cfg, quorum)
	defer func() {
		os.Unsetenv("ETCDCTL_API")
		if errC := epc.Close(); errC != nil {
			t.Fatalf("error closing etcd processes (%v)", errC)
		}
	}()

	key, value := "foo", "bar"

	errc := make(chan error, 1)
	expectTimeout := dialTimeout > 0 && dialTimeout <= time.Nanosecond
	go func() {
		defer close(errc)
		if err := ctlV3Put(epc, key, value, dialTimeout); err != nil {
			if expectTimeout && isGRPCTimedout(err) {
				errc <- fmt.Errorf("put error (%v)", err)
				return
			}
		}
		if err := ctlV3Get(epc, key, value, dialTimeout, quorum); err != nil {
			if expectTimeout && isGRPCTimedout(err) {
				errc <- fmt.Errorf("get error (%v)", err)
				return
			}
		}
	}()

	select {
	case <-time.After(2*dialTimeout + time.Second):
		if dialTimeout > 0 {
			t.Fatalf("test timed out for %v", dialTimeout)
		}
	case err := <-errc:
		if err != nil {
			t.Fatal(err)
		}
	}
}

func ctlV3PrefixArgs(clus *etcdProcessCluster, dialTimeout time.Duration) []string {
	if len(clus.proxies()) > 0 { // TODO: add proxy check as in v2
		panic("v3 proxy not implemented")
	}

	endpoints := ""
	if backends := clus.backends(); len(backends) != 0 {
		es := []string{}
		for _, b := range backends {
			es = append(es, stripSchema(b.cfg.acurl))
		}
		endpoints = strings.Join(es, ",")
	}
	cmdArgs := []string{"../bin/etcdctl", "--endpoints", endpoints, "--dial-timeout", dialTimeout.String()}
	if clus.cfg.clientTLS == clientTLS {
		cmdArgs = append(cmdArgs, "--cacert", caPath, "--cert", certPath, "--key", privateKeyPath)
	}
	return cmdArgs
}

func ctlV3Put(clus *etcdProcessCluster, key, value string, dialTimeout time.Duration) error {
	cmdArgs := append(ctlV3PrefixArgs(clus, dialTimeout), "put", key, value)
	return spawnWithExpect(cmdArgs, "OK")
}

func ctlV3Get(clus *etcdProcessCluster, key, value string, dialTimeout time.Duration, quorum bool) error {
	cmdArgs := append(ctlV3PrefixArgs(clus, dialTimeout), "get", key)
	if !quorum {
		cmdArgs = append(cmdArgs, "--consistency", "s")
	}
	return spawnWithExpects(cmdArgs, key, value)
}

func setupCtlV3Test(t *testing.T, cfg *etcdProcessClusterConfig, quorum bool) *etcdProcessCluster {
	mustEtcdctl(t)
	if !quorum {
		cfg = configStandalone(*cfg)
	}
	copied := *cfg
	epc, err := newEtcdProcessCluster(&copied)
	if err != nil {
		t.Fatalf("could not start etcd process cluster (%v)", err)
	}
	return epc
}

func isGRPCTimedout(err error) bool {
	return strings.Contains(err.Error(), "grpc: timed out trying to connect")
}

func stripSchema(s string) string {
	if strings.HasPrefix(s, "http://") {
		s = strings.Replace(s, "http://", "", -1)
	}
	if strings.HasPrefix(s, "https://") {
		s = strings.Replace(s, "https://", "", -1)
	}
	return s
}
<commit_msg>e2e: add basic v3 watch test<commit_after>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package e2e

import (
	"fmt"
	"os"
	"strconv"
	"strings"
	"testing"
	"time"

	"github.com/coreos/etcd/pkg/testutil"
)

func TestCtlV3Put(t *testing.T)              { testCtl(t, putTest) }
func TestCtlV3PutTimeout(t *testing.T)       { testCtl(t, putTest, withDialTimeout(0)) }
func TestCtlV3PutTimeoutQuorum(t *testing.T) { testCtl(t, putTest, withDialTimeout(0), withQuorum()) }
func TestCtlV3PutAutoTLS(t *testing.T)       { testCtl(t, putTest, withCfg(configAutoTLS)) }
func TestCtlV3PutPeerTLS(t *testing.T)       { testCtl(t, putTest, withCfg(configPeerTLS)) }
func TestCtlV3PutClientTLS(t *testing.T)     { testCtl(t, putTest, withCfg(configClientTLS)) }

func TestCtlV3Watch(t *testing.T)        { testCtl(t, watchTest) }
func TestCtlV3WatchAutoTLS(t *testing.T) { testCtl(t, watchTest, withCfg(configAutoTLS)) }
func TestCtlV3WatchPeerTLS(t *testing.T) { testCtl(t, watchTest, withCfg(configPeerTLS)) }

// TODO: Watch with client TLS is not working
// func TestCtlV3WatchClientTLS(t *testing.T) {
// 	testCtl(t, watchTest, withCfg(configClientTLS))
// }

func TestCtlV3WatchInteractive(t *testing.T) { testCtl(t, watchTest, withInteractive()) }
func TestCtlV3WatchInteractiveAutoTLS(t *testing.T) {
	testCtl(t, watchTest, withInteractive(), withCfg(configAutoTLS))
}
func TestCtlV3WatchInteractivePeerTLS(t *testing.T) {
	testCtl(t, watchTest, withInteractive(), withCfg(configPeerTLS))
}

// TODO: Watch with client TLS is not working
// func TestCtlV3WatchInteractiveClientTLS(t *testing.T) {
// 	testCtl(t, watchTest, withInteractive(), withCfg(configClientTLS))
// }

type ctlCtx struct {
	t   *testing.T
	cfg etcdProcessClusterConfig
	epc *etcdProcessCluster

	errc        chan error
	dialTimeout time.Duration

	quorum        bool
	interactive   bool
	watchRevision int
}

type ctlOption func(*ctlCtx)

func (cx *ctlCtx) applyOpts(opts []ctlOption) {
	for _, opt := range opts {
		opt(cx)
	}
}

func withCfg(cfg etcdProcessClusterConfig) ctlOption {
	return func(cx *ctlCtx) { cx.cfg = cfg }
}

func withDialTimeout(timeout time.Duration) ctlOption {
	return func(cx *ctlCtx) { cx.dialTimeout = timeout }
}

func withQuorum() ctlOption {
	return func(cx *ctlCtx) { cx.quorum = true }
}

func withInteractive() ctlOption {
	return func(cx *ctlCtx) { cx.interactive = true }
}

func withWatchRevision(rev int) ctlOption {
	return func(cx *ctlCtx) { cx.watchRevision = rev }
}

func setupCtlV3Test(t *testing.T, cfg etcdProcessClusterConfig, quorum bool) *etcdProcessCluster {
	mustEtcdctl(t)
	if !quorum {
		cfg = *configStandalone(cfg)
	}
	epc, err := newEtcdProcessCluster(&cfg)
	if err != nil {
		t.Fatalf("could not start etcd process cluster (%v)", err)
	}
	return epc
}

func testCtl(t *testing.T, testFunc func(ctlCtx), opts ...ctlOption) {
	defer testutil.AfterTest(t)

	var (
		defaultDialTimeout   = 7 * time.Second
		defaultWatchRevision = 1
	)
	ret := ctlCtx{
		t:             t,
		cfg:           configNoTLS,
		errc:          make(chan error, 1),
		dialTimeout:   defaultDialTimeout,
		watchRevision: defaultWatchRevision,
	}
	ret.applyOpts(opts)

	os.Setenv("ETCDCTL_API", "3")
	ret.epc = setupCtlV3Test(ret.t, ret.cfg, ret.quorum)

	defer func() {
		os.Unsetenv("ETCDCTL_API")
		if errC := ret.epc.Close(); errC != nil {
			t.Fatalf("error closing etcd processes (%v)", errC)
		}
	}()

	go testFunc(ret)

	select {
	case <-time.After(2*ret.dialTimeout + time.Second):
		if ret.dialTimeout > 0 {
			t.Fatalf("test timed out for %v", ret.dialTimeout)
		}
	case err := <-ret.errc:
		if err != nil {
			t.Fatal(err)
		}
	}
	return
}

func putTest(cx ctlCtx) {
	key, value := "foo", "bar"

	defer close(cx.errc)

	if err := ctlV3Put(cx, key, value); err != nil {
		if cx.dialTimeout > 0 && isGRPCTimedout(err) {
			cx.errc <- fmt.Errorf("put error (%v)", err)
			return
		}
	}
	if err := ctlV3Get(cx, key, value); err != nil {
		if cx.dialTimeout > 0 && isGRPCTimedout(err) {
			cx.errc <- fmt.Errorf("get error (%v)", err)
			return
		}
	}
}

func watchTest(cx ctlCtx) {
	key, value := "foo", "bar"

	defer close(cx.errc)

	go func() {
		if err := ctlV3Put(cx, key, value); err != nil {
			cx.t.Fatal(err)
		}
	}()

	if err := ctlV3Watch(cx, key, value); err != nil {
		if cx.dialTimeout > 0 && isGRPCTimedout(err) {
			cx.errc <- fmt.Errorf("watch error (%v)", err)
			return
		}
	}
}

func ctlV3PrefixArgs(clus *etcdProcessCluster, dialTimeout time.Duration) []string {
	if len(clus.proxies()) > 0 { // TODO: add proxy check as in v2
		panic("v3 proxy not implemented")
	}

	endpoints := ""
	if backends := clus.backends(); len(backends) != 0 {
		es := []string{}
		for _, b := range backends {
			es = append(es, stripSchema(b.cfg.acurl))
		}
		endpoints = strings.Join(es, ",")
	}
	cmdArgs := []string{"../bin/etcdctl", "--endpoints", endpoints, "--dial-timeout", dialTimeout.String()}
	if clus.cfg.clientTLS == clientTLS {
		cmdArgs = append(cmdArgs, "--cacert", caPath, "--cert", certPath, "--key", privateKeyPath)
	}
	return cmdArgs
}

func ctlV3Put(cx ctlCtx, key, value string) error {
	cmdArgs := append(ctlV3PrefixArgs(cx.epc, cx.dialTimeout), "put", key, value)
	return spawnWithExpect(cmdArgs, "OK")
}

func ctlV3Get(cx ctlCtx, key, value string) error {
	cmdArgs := append(ctlV3PrefixArgs(cx.epc, cx.dialTimeout), "get", key)
	if !cx.quorum {
		cmdArgs = append(cmdArgs, "--consistency", "s")
	}
	return spawnWithExpects(cmdArgs, key, value)
}

func ctlV3Watch(cx ctlCtx, key, value string) error {
	cmdArgs := append(ctlV3PrefixArgs(cx.epc, cx.dialTimeout), "watch")
	if !cx.interactive {
		if cx.watchRevision > 0 {
			cmdArgs = append(cmdArgs, "--rev", strconv.Itoa(cx.watchRevision))
		}
		cmdArgs = append(cmdArgs, key)
		return spawnWithExpects(cmdArgs, key, value)
	}
	cmdArgs = append(cmdArgs, "--interactive")
	proc, err := spawnCmd(cmdArgs)
	if err != nil {
		return err
	}
	watchLine := fmt.Sprintf("watch %s", key)
	if cx.watchRevision > 0 {
		watchLine = fmt.Sprintf("watch %s --rev %d", key, cx.watchRevision)
	}
	if err = proc.SendLine(watchLine); err != nil {
		return err
	}
	_, err = proc.Expect(key)
	if err != nil {
		return err
	}
	_, err = proc.Expect(value)
	if err != nil {
		return err
	}
	return proc.Close()
}

func isGRPCTimedout(err error) bool {
	return strings.Contains(err.Error(), "grpc: timed out trying to connect")
}

func stripSchema(s string) string {
	if strings.HasPrefix(s, "http://") {
		s = strings.Replace(s, "http://", "", -1)
	}
	if strings.HasPrefix(s, "https://") {
		s = strings.Replace(s, "https://", "", -1)
	}
	return s
}
<|endoftext|>


<commit_before>package terraform

// The main version number that is being run at the moment.
const Version = "0.6.1"

// A pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release
// such as "dev" (in development), "beta", "rc1", etc.
const VersionPrerelease = ""
<commit_msg>release: clean up after v0.6.1<commit_after>package terraform

// The main version number that is being run at the moment.
const Version = "0.6.2"

// A pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release
// such as "dev" (in development), "beta", "rc1", etc.
const VersionPrerelease = "dev"
<|endoftext|>


<commit_before>package terraform

import (
	"fmt"

	"github.com/hashicorp/go-version"
)

// The main version number that is being run at the moment.
const Version = "0.8.0"

// A pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release
// such as "dev" (in development), "beta", "rc1", etc.
const VersionPrerelease = "dev"

// SemVersion is an instance of version.Version. This has the secondary
// benefit of verifying during tests and init time that our version is a
// proper semantic version, which should always be the case.
var SemVersion = version.Must(version.NewVersion(Version))

// VersionHeader is the header name used to send the current terraform version
// in http requests.
const VersionHeader = "Terraform-Version"

func VersionString() string {
	if VersionPrerelease != "" {
		return fmt.Sprintf("%s-%s", Version, VersionPrerelease)
	}
	return Version
}
<commit_msg>v0.8.0-beta2<commit_after>package terraform

import (
	"fmt"

	"github.com/hashicorp/go-version"
)

// The main version number that is being run at the moment.
const Version = "0.8.0"

// A pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release
// such as "dev" (in development), "beta", "rc1", etc.
const VersionPrerelease = "beta2"

// SemVersion is an instance of version.Version. This has the secondary
// benefit of verifying during tests and init time that our version is a
// proper semantic version, which should always be the case.
var SemVersion = version.Must(version.NewVersion(Version))

// VersionHeader is the header name used to send the current terraform version
// in http requests.
const VersionHeader = "Terraform-Version"

func VersionString() string {
	if VersionPrerelease != "" {
		return fmt.Sprintf("%s-%s", Version, VersionPrerelease)
	}
	return Version
}
<|endoftext|>


<commit_before>package egl

import (
	"fmt"
	"log"
	"unsafe"

	"github.com/xlab/android-go/android"
)

type DisplayHandle struct {
	display Display
	surface Surface
	context Context

	Width  int
	Height int
}

func (d *DisplayHandle) EGLDisplay() Display {
	return d.display
}

func (d *DisplayHandle) EGLSurface() Surface {
	return d.surface
}

func (d *DisplayHandle) EGLContext() Context {
	return d.context
}

func (d *DisplayHandle) Destroy() {
	if d == nil {
		return
	}
	MakeCurrent(d.display, NoSurface, NoSurface, NoContext)
	DestroyContext(d.display, d.context)
	DestroySurface(d.display, d.surface)
	Terminate(d.display)
	if err := Error(); err != nil {
		log.Println("EGL error:", err)
	}
}

// NewDisplayHandle initializes EGL display/surface/context and returns a handle object or error.
// Use expectedConfig to specify the desired EGL config constraints like:
//
//  map[int32]int32{
//  	egl.SurfaceType: egl.WindowBit,
//  	egl.RedSize:   8,
//  	egl.GreenSize: 8,
//  	egl.BlueSize:  8,
//  	egl.AlphaSize: 8,
//  	egl.DepthSize: 24,
//  }
func NewDisplayHandle(window *android.NativeWindow, expectedConfig map[int32]int32) (*DisplayHandle, error) {
	display := GetDisplay(DefaultDisplay)
	if Initialize(display, nil, nil) == False {
		err := fmt.Errorf("eglInitialize failed: %v", Error())
		return nil, err
	}

	var count int32
	GetConfigs(display, nil, 0, &count)
	configs := make([]Config, count)
	GetConfigs(display, configs, count, &count)
	var format int32
	var foundConfig = -1
	var v int32
	if expectedConfig == nil {
		expectedConfig = map[int32]int32{}
	}
	for i, cfg := range configs {
		GetConfigAttrib(display, cfg, SurfaceType, &v)
		if e := expectedConfig[SurfaceType]; e > 0 && v&e != e {
			continue
		}
		GetConfigAttrib(display, cfg, RedSize, &v)
		if e := expectedConfig[RedSize]; e > 0 && v != e {
			continue
		}
		GetConfigAttrib(display, cfg, GreenSize, &v)
		if e := expectedConfig[GreenSize]; e > 0 && v != e {
			continue
		}
		GetConfigAttrib(display, cfg, BlueSize, &v)
		if e := expectedConfig[BlueSize]; e > 0 && v != e {
			continue
		}
		GetConfigAttrib(display, cfg, AlphaSize, &v)
		if e := expectedConfig[AlphaSize]; e > 0 && v != e {
			continue
		}
		GetConfigAttrib(display, cfg, DepthSize, &v)
		if e := expectedConfig[DepthSize]; e > 0 && v != e {
			continue
		}
		// gotcha!
		foundConfig = i
		// NativeVisualId is an attribute of the Config that is
		// guaranteed to be accepted by android.NativeWindowSetBuffersGeometry().
		// As soon as we picked a Config, we can safely reconfigure the
		// NativeWindow buffers to match, using NativeVisualId.
		GetConfigAttrib(display, cfg, NativeVisualId, &format)
	}
	if foundConfig < 0 {
		Terminate(display)
		err := fmt.Errorf("failed to find EGL config for %#v", expectedConfig)
		return nil, err
	}
	android.NativeWindowSetBuffersGeometry(window, 0, 0, format)
	windowPtr := NativeWindowType(unsafe.Pointer(window))
	surface := CreateWindowSurface(display, configs[foundConfig], windowPtr, nil)
	context := CreateContext(display, configs[foundConfig], NoContext, []int32{
		ContextClientVersion, 3.0, None, // create GL ES 3.0 context
	})
	if MakeCurrent(display, surface, surface, context) == False {
		DestroyContext(display, context)
		DestroySurface(display, surface)
		Terminate(display)
		err := fmt.Errorf("eglMakeCurrent failed: %v", Error())
		return nil, err
	}
	handle := &DisplayHandle{
		display: display,
		surface: surface,
		context: context,
	}
	handle.UpdateDimensions()
	return handle, nil
}

func (d *DisplayHandle) UpdateDimensions() {
	var width, height int32
	QuerySurface(d.display, d.surface, Width, &width)
	QuerySurface(d.display, d.surface, Height, &height)
	d.Width = int(width)
	d.Height = int(height)
}

func (d *DisplayHandle) SwapBuffers() {
	SwapBuffers(d.display, d.surface)
}
<commit_msg>Support request of egl.ContextClientVersion in expectedConfig<commit_after>package egl

import (
	"fmt"
	"log"
	"unsafe"

	"github.com/xlab/android-go/android"
)

type DisplayHandle struct {
	display Display
	surface Surface
	context Context

	Width  int
	Height int
}

func (d *DisplayHandle) EGLDisplay() Display {
	return d.display
}

func (d *DisplayHandle) EGLSurface() Surface {
	return d.surface
}

func (d *DisplayHandle) EGLContext() Context {
	return d.context
}

func (d *DisplayHandle) Destroy() {
	if d == nil {
		return
	}
	MakeCurrent(d.display, NoSurface, NoSurface, NoContext)
	DestroyContext(d.display, d.context)
	DestroySurface(d.display, d.surface)
	Terminate(d.display)
	if err := Error(); err != nil {
		log.Println("EGL error:", err)
	}
}

// NewDisplayHandle initializes EGL display/surface/context and returns a handle object or error.
// Use expectedConfig to specify the desired EGL config constraints like:
//
//  map[int32]int32{
//  	egl.SurfaceType: egl.WindowBit,
//  	egl.RedSize:   8,
//  	egl.GreenSize: 8,
//  	egl.BlueSize:  8,
//  	egl.AlphaSize: 8,
//  	egl.DepthSize: 24,
//  }
func NewDisplayHandle(window *android.NativeWindow, expectedConfig map[int32]int32) (*DisplayHandle, error) {
	display := GetDisplay(DefaultDisplay)
	if Initialize(display, nil, nil) == False {
		err := fmt.Errorf("eglInitialize failed: %v", Error())
		return nil, err
	}

	var count int32
	GetConfigs(display, nil, 0, &count)
	configs := make([]Config, count)
	GetConfigs(display, configs, count, &count)
	var format int32
	var foundConfig = -1
	var v int32
	if expectedConfig == nil {
		expectedConfig = map[int32]int32{}
	}
	for i, cfg := range configs {
		GetConfigAttrib(display, cfg, SurfaceType, &v)
		if e := expectedConfig[SurfaceType]; e > 0 && v&e != e {
			continue
		}
		GetConfigAttrib(display, cfg, RedSize, &v)
		if e := expectedConfig[RedSize]; e > 0 && v != e {
			continue
		}
		GetConfigAttrib(display, cfg, GreenSize, &v)
		if e := expectedConfig[GreenSize]; e > 0 && v != e {
			continue
		}
		GetConfigAttrib(display, cfg, BlueSize, &v)
		if e := expectedConfig[BlueSize]; e > 0 && v != e {
			continue
		}
		GetConfigAttrib(display, cfg, AlphaSize, &v)
		if e := expectedConfig[AlphaSize]; e > 0 && v != e {
			continue
		}
		GetConfigAttrib(display, cfg, DepthSize, &v)
		if e := expectedConfig[DepthSize]; e > 0 && v != e {
			continue
		}
		// gotcha!
		foundConfig = i
		// NativeVisualId is an attribute of the Config that is
		// guaranteed to be accepted by android.NativeWindowSetBuffersGeometry().
		// As soon as we picked a Config, we can safely reconfigure the
		// NativeWindow buffers to match, using NativeVisualId.
		GetConfigAttrib(display, cfg, NativeVisualId, &format)
	}
	if foundConfig < 0 {
		Terminate(display)
		err := fmt.Errorf("failed to find EGL config for %#v", expectedConfig)
		return nil, err
	}
	android.NativeWindowSetBuffersGeometry(window, 0, 0, format)
	windowPtr := NativeWindowType(unsafe.Pointer(window))
	surface := CreateWindowSurface(display, configs[foundConfig], windowPtr, nil)
	var ctxRequest []int32
	if ctxVer := expectedConfig[ContextClientVersion]; ctxVer > 0 {
		ctxRequest = []int32{ContextClientVersion, ctxVer, None}
	}
	context := CreateContext(display, configs[foundConfig], NoContext, ctxRequest)
	if MakeCurrent(display, surface, surface, context) == False {
		DestroyContext(display, context)
		DestroySurface(display, surface)
		Terminate(display)
		err := fmt.Errorf("eglMakeCurrent failed: %v", Error())
		return nil, err
	}
	handle := &DisplayHandle{
		display: display,
		surface: surface,
		context: context,
	}
	handle.UpdateDimensions()
	return handle, nil
}

func (d *DisplayHandle) UpdateDimensions() {
	var width, height int32
	QuerySurface(d.display, d.surface, Width, &width)
	QuerySurface(d.display, d.surface, Height, &height)
	d.Width = int(width)
	d.Height = int(height)
}

func (d *DisplayHandle) SwapBuffers() {
	SwapBuffers(d.display, d.surface)
}
<|endoftext|>


<commit_before>// run

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Test heap sampling logic.

package main

import (
	"fmt"
	"math"
	"runtime"
)

var a16 *[16]byte
var a512 *[512]byte
var a256 *[256]byte
var a1k *[1024]byte
var a64k *[64 * 1024]byte

// This test checks that heap sampling produces reasonable
// results. Note that heap sampling uses randomization, so the results
// vary for run to run. This test only checks that the resulting
// values appear reasonable.
func main() {
	return // TODO: fix this flaky test; golang.org/issue/13098

	const countInterleaved = 10000
	allocInterleaved(countInterleaved)
	checkAllocations(getMemProfileRecords(), "main.allocInterleaved", countInterleaved, []int64{256 * 1024, 1024, 256 * 1024, 512, 256 * 1024, 256})

	const count = 100000
	alloc(count)
	checkAllocations(getMemProfileRecords(), "main.alloc", count, []int64{1024, 512, 256})
}

// allocInterleaved stress-tests the heap sampling logic by
// interleaving large and small allocations.
func allocInterleaved(n int) {
	for i := 0; i < n; i++ {
		// Test verification depends on these lines being contiguous.
		a64k = new([64 * 1024]byte)
		a1k = new([1024]byte)
		a64k = new([64 * 1024]byte)
		a512 = new([512]byte)
		a64k = new([64 * 1024]byte)
		a256 = new([256]byte)
	}
}

// alloc performs only small allocations for sanity testing.
func alloc(n int) {
	for i := 0; i < n; i++ {
		// Test verification depends on these lines being contiguous.
		a1k = new([1024]byte)
		a512 = new([512]byte)
		a256 = new([256]byte)
	}
}

// checkAllocations validates that the profile records collected for
// the named function are consistent with count contiguous allocations
// of the specified sizes.
func checkAllocations(records []runtime.MemProfileRecord, fname string, count int64, size []int64) {
	a := allocObjects(records, fname)
	firstLine := 0
	for ln := range a {
		if firstLine == 0 || firstLine > ln {
			firstLine = ln
		}
	}
	var totalcount int64
	for i, w := range size {
		ln := firstLine + i
		s := a[ln]
		checkValue(fname, ln, "objects", count, s.objects)
		checkValue(fname, ln, "bytes", count*w, s.bytes)
		totalcount += s.objects
	}
	// Check the total number of allocations, to ensure some sampling occurred.
	if totalwant := count * int64(len(size)); totalcount <= 0 || totalcount > totalwant*1024 {
		panic(fmt.Sprintf("%s want total count > 0 && <= %d, got %d", fname, totalwant*1024, totalcount))
	}
}

// checkValue checks an unsampled value against a range.
func checkValue(fname string, ln int, name string, want, got int64) {
	if got < 0 || got > 1024*want {
		panic(fmt.Sprintf("%s:%d want %s >= 0 && <= %d, got %d", fname, ln, name, 1024*want, got))
	}
}

func getMemProfileRecords() []runtime.MemProfileRecord {
	// Find out how many records there are (MemProfile(nil, true)),
	// allocate that many records, and get the data.
	// There's a race—more records might be added between
	// the two calls—so allocate a few extra records for safety
	// and also try again if we're very unlucky.
	// The loop should only execute one iteration in the common case.
	var p []runtime.MemProfileRecord
	n, ok := runtime.MemProfile(nil, true)
	for {
		// Allocate room for a slightly bigger profile,
		// in case a few more entries have been added
		// since the call to MemProfile.
		p = make([]runtime.MemProfileRecord, n+50)
		n, ok = runtime.MemProfile(p, true)
		if ok {
			p = p[0:n]
			break
		}
		// Profile grew; try again.
	}
	return p
}

type allocStat struct {
	bytes, objects int64
}

// allocObjects examines the profile records for the named function
// and returns the allocation stats aggregated by source line number.
func allocObjects(records []runtime.MemProfileRecord, function string) map[int]allocStat {
	a := make(map[int]allocStat)
	for _, r := range records {
		for _, s := range r.Stack0 {
			if s == 0 {
				break
			}
			if f := runtime.FuncForPC(s); f != nil {
				name := f.Name()
				_, line := f.FileLine(s)
				if name == function {
					allocStat := a[line]
					allocStat.bytes += r.AllocBytes
					allocStat.objects += r.AllocObjects
					a[line] = allocStat
				}
			}
		}
	}
	for line, stats := range a {
		objects, bytes := scaleHeapSample(stats.objects, stats.bytes, int64(runtime.MemProfileRate))
		a[line] = allocStat{bytes, objects}
	}
	return a
}

// scaleHeapSample unsamples heap allocations.
// Taken from src/cmd/pprof/internal/profile/legacy_profile.go
func scaleHeapSample(count, size, rate int64) (int64, int64) {
	if count == 0 || size == 0 {
		return 0, 0
	}

	if rate <= 1 {
		// if rate==1 all samples were collected so no adjustment is needed.
		// if rate<1 treat as unknown and skip scaling.
		return count, size
	}

	avgSize := float64(size) / float64(count)
	scale := 1 / (1 - math.Exp(-avgSize/float64(rate)))

	return int64(float64(count) * scale), int64(float64(size) * scale)
}
<commit_msg>test: force heap profile update in heapsampling.go test<commit_after>// run

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Test heap sampling logic.

package main

import (
	"fmt"
	"math"
	"runtime"
)

var a16 *[16]byte
var a512 *[512]byte
var a256 *[256]byte
var a1k *[1024]byte
var a64k *[64 * 1024]byte

// This test checks that heap sampling produces reasonable
// results. Note that heap sampling uses randomization, so the results
// vary for run to run. This test only checks that the resulting
// values appear reasonable.
func main() {
	const countInterleaved = 10000
	allocInterleaved(countInterleaved)
	checkAllocations(getMemProfileRecords(), "main.allocInterleaved", countInterleaved, []int64{256 * 1024, 1024, 256 * 1024, 512, 256 * 1024, 256})

	const count = 100000
	alloc(count)
	checkAllocations(getMemProfileRecords(), "main.alloc", count, []int64{1024, 512, 256})
}

// allocInterleaved stress-tests the heap sampling logic by
// interleaving large and small allocations.
func allocInterleaved(n int) {
	for i := 0; i < n; i++ {
		// Test verification depends on these lines being contiguous.
		a64k = new([64 * 1024]byte)
		a1k = new([1024]byte)
		a64k = new([64 * 1024]byte)
		a512 = new([512]byte)
		a64k = new([64 * 1024]byte)
		a256 = new([256]byte)
	}
}

// alloc performs only small allocations for sanity testing.
func alloc(n int) {
	for i := 0; i < n; i++ {
		// Test verification depends on these lines being contiguous.
		a1k = new([1024]byte)
		a512 = new([512]byte)
		a256 = new([256]byte)
	}
}

// checkAllocations validates that the profile records collected for
// the named function are consistent with count contiguous allocations
// of the specified sizes.
func checkAllocations(records []runtime.MemProfileRecord, fname string, count int64, size []int64) {
	a := allocObjects(records, fname)
	firstLine := 0
	for ln := range a {
		if firstLine == 0 || firstLine > ln {
			firstLine = ln
		}
	}
	var totalcount int64
	for i, w := range size {
		ln := firstLine + i
		s := a[ln]
		checkValue(fname, ln, "objects", count, s.objects)
		checkValue(fname, ln, "bytes", count*w, s.bytes)
		totalcount += s.objects
	}
	// Check the total number of allocations, to ensure some sampling occurred.
	if totalwant := count * int64(len(size)); totalcount <= 0 || totalcount > totalwant*1024 {
		panic(fmt.Sprintf("%s want total count > 0 && <= %d, got %d", fname, totalwant*1024, totalcount))
	}
}

// checkValue checks an unsampled value against a range.
func checkValue(fname string, ln int, name string, want, got int64) {
	if got < 0 || got > 1024*want {
		panic(fmt.Sprintf("%s:%d want %s >= 0 && <= %d, got %d", fname, ln, name, 1024*want, got))
	}
}

func getMemProfileRecords() []runtime.MemProfileRecord {
	// Force the runtime to update the object and byte counts.
	runtime.GC()

	// Find out how many records there are (MemProfile(nil, true)),
	// allocate that many records, and get the data.
	// There's a race—more records might be added between
	// the two calls—so allocate a few extra records for safety
	// and also try again if we're very unlucky.
	// The loop should only execute one iteration in the common case.
	var p []runtime.MemProfileRecord
	n, ok := runtime.MemProfile(nil, true)
	for {
		// Allocate room for a slightly bigger profile,
		// in case a few more entries have been added
		// since the call to MemProfile.
		p = make([]runtime.MemProfileRecord, n+50)
		n, ok = runtime.MemProfile(p, true)
		if ok {
			p = p[0:n]
			break
		}
		// Profile grew; try again.
	}
	return p
}

type allocStat struct {
	bytes, objects int64
}

// allocObjects examines the profile records for the named function
// and returns the allocation stats aggregated by source line number.
func allocObjects(records []runtime.MemProfileRecord, function string) map[int]allocStat {
	a := make(map[int]allocStat)
	for _, r := range records {
		for _, s := range r.Stack0 {
			if s == 0 {
				break
			}
			if f := runtime.FuncForPC(s); f != nil {
				name := f.Name()
				_, line := f.FileLine(s)
				if name == function {
					allocStat := a[line]
					allocStat.bytes += r.AllocBytes
					allocStat.objects += r.AllocObjects
					a[line] = allocStat
				}
			}
		}
	}
	for line, stats := range a {
		objects, bytes := scaleHeapSample(stats.objects, stats.bytes, int64(runtime.MemProfileRate))
		a[line] = allocStat{bytes, objects}
	}
	return a
}

// scaleHeapSample unsamples heap allocations.
// Taken from src/cmd/pprof/internal/profile/legacy_profile.go
func scaleHeapSample(count, size, rate int64) (int64, int64) {
	if count == 0 || size == 0 {
		return 0, 0
	}

	if rate <= 1 {
		// if rate==1 all samples were collected so no adjustment is needed.
		// if rate<1 treat as unknown and skip scaling.
		return count, size
	}

	avgSize := float64(size) / float64(count)
	scale := 1 / (1 - math.Exp(-avgSize/float64(rate)))

	return int64(float64(count) * scale), int64(float64(size) * scale)
}
<|endoftext|>


<commit_before>// Copyright 2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package helpers

import (
	"fmt"
	"time"
)

var (
	// HelperTimeout is a predefined timeout value for commands.
	HelperTimeout time.Duration = 300 // WithTimeout helper translates it to seconds
)

const (
	// BasePath is the path in the Vagrant VMs to which the test directory
	// is mounted
	BasePath = "/vagrant/"

	// ManifestBase tells ginkgo suite where to look for manifests
	K8sManifestBase = "k8sT/manifests"

	// VM / Test suite constants.
	K8s     = "k8s"
	K8s1    = "k8s1"
	K8s2    = "k8s2"
	Runtime = "runtime"

	Enabled  = "enabled"
	Disabled = "disabled"
	Total    = "total"

	// PolicyEnforcement represents the PolicyEnforcement configuration option
	// for the Cilium agent.
	PolicyEnforcement = "PolicyEnforcement"

	// PolicyEnforcementDefault represents the default PolicyEnforcement mode
	// for Cilium.
	PolicyEnforcementDefault = "default"

	// PolicyEnforcementAlways represents the PolicyEnforcement mode
	// for Cilium in which traffic is denied by default even when no policy
	// is imported.
	PolicyEnforcementAlways = "always"

	// PolicyEnforcementNever represents the PolicyEnforcement mode
	// for Cilium in which traffic is always allowed even if there is a policy
	// selecting endpoints.
	PolicyEnforcementNever = "never"

	// Docker Image names

	// CiliumDockerNetwork is the name of the Docker network which Cilium manages.
	CiliumDockerNetwork = "cilium-net"

	// NetperfImage is the Docker image used for performance testing
	NetperfImage = "tgraf/netperf"

	// HttpdImage is the image used for starting an HTTP server.
	HttpdImage = "cilium/demo-httpd"

	// Names of commonly used containers in tests.

	Httpd1 = "httpd1"
	Httpd2 = "httpd2"
	Httpd3 = "httpd3"
	App1   = "app1"
	App2   = "app2"
	App3   = "app3"
	Client = "client"
	Server = "server"
	Host   = "host"

	// Container lifecycle actions.
	Create = "create"
	Delete = "delete"

	// IP Address families.
	IPv4 = "IPv4"
	IPv6 = "IPv6"

	// Configuration options for endpoints. Copied from endpoint/endpoint.go
	// TODO: these should be converted into types for use in configuration
	// functions instead of using basic strings.

	OptionAllowToHost         = "AllowToHost"
	OptionAllowToWorld        = "AllowToWorld"
	OptionConntrackAccounting = "ConntrackAccounting"
	OptionConntrackLocal      = "ConntrackLocal"
	OptionConntrack           = "Conntrack"
	OptionDebug               = "Debug"
	OptionDropNotify          = "DropNotification"
	OptionTraceNotify         = "TraceNotification"
	OptionNAT46               = "NAT46"
	OptionIngressPolicy       = "IngressPolicy"
	OptionEgressPolicy        = "EgressPolicy"
	OptionIngress             = "ingress"
	OptionEgress              = "egress"
	OptionNone                = "none"
	OptionDisabled            = "Disabled"
	OptionEnabled             = "Enabled"

	PingCount          = 5
	CurlConnectTimeout = 5

	DefaultNamespace    = "default"
	KubeSystemNamespace = "kube-system"

	TestResultsPath = "test_results/"
	RunDir          = "/var/run/cilium"
	LibDir          = "/var/lib/cilium"

	DaemonName             = "cilium"
	CiliumDockerDaemonName = "cilium-docker"
	AgentDaemon            = "cilium-agent"

	GeneratedHTMLManifest   = "html.yaml"
	GeneratedServerManifest = "server.yaml"
	GeneratedClientManifest = "client.yaml"
)

var ciliumCLICommands = map[string]string{
	"cilium endpoint list -o json":   "endpoint_list_txt",
	"cilium service list -o json":    "service_list.txt",
	"cilium config":                  "config.txt",
	"sudo cilium bpf lb list":        "bpf_lb_list.txt",
	"sudo cilium bpf ct list global": "bpf_ct_list.txt",
	"sudo cilium bpf tunnel list":    "bpf_tunnel_list.txt",
	"cilium policy get":              "policy_get.txt",
	"cilium status":                  "status.txt",
}

// ciliumKubCLICommands these commands are the same as `ciliumCLICommands` but
// it'll run inside a container and it does not have sudo support
var ciliumKubCLICommands = map[string]string{
	"cilium endpoint list -o json": "endpoint_list_txt",
	"cilium service list -o json":  "service_list.txt",
	"cilium config":                "config.txt",
	"cilium bpf lb list":           "bpf_lb_list.txt",
	"cilium bpf ct list global":    "bpf_ct_list.txt",
	"cilium bpf tunnel list":       "bpf_tunnel_list.txt",
	"cilium policy get":            "policy_get.txt",
	"cilium status":                "status.txt",
}

//GetFilePath returns the absolute path of the provided filename
func GetFilePath(filename string) string {
	return fmt.Sprintf("%s%s", BasePath, filename)
}

// K8s1VMName is the name of the Kubernetes master node when running K8s tests.
func K8s1VMName() string {
	return fmt.Sprintf("k8s1-%s", GetCurrentK8SEnv())
}

// K8s2VMName is the name of the Kubernetes worker node when running K8s tests.
func K8s2VMName() string {
	return fmt.Sprintf("k8s2-%s", GetCurrentK8SEnv())
}
<commit_msg>test/helpers: change filename to have .txt extension<commit_after>// Copyright 2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package helpers

import (
	"fmt"
	"time"
)

var (
	// HelperTimeout is a predefined timeout value for commands.
	HelperTimeout time.Duration = 300 // WithTimeout helper translates it to seconds
)

const (
	// BasePath is the path in the Vagrant VMs to which the test directory
	// is mounted
	BasePath = "/vagrant/"

	// ManifestBase tells ginkgo suite where to look for manifests
	K8sManifestBase = "k8sT/manifests"

	// VM / Test suite constants.
	K8s     = "k8s"
	K8s1    = "k8s1"
	K8s2    = "k8s2"
	Runtime = "runtime"

	Enabled  = "enabled"
	Disabled = "disabled"
	Total    = "total"

	// PolicyEnforcement represents the PolicyEnforcement configuration option
	// for the Cilium agent.
	PolicyEnforcement = "PolicyEnforcement"

	// PolicyEnforcementDefault represents the default PolicyEnforcement mode
	// for Cilium.
	PolicyEnforcementDefault = "default"

	// PolicyEnforcementAlways represents the PolicyEnforcement mode
	// for Cilium in which traffic is denied by default even when no policy
	// is imported.
	PolicyEnforcementAlways = "always"

	// PolicyEnforcementNever represents the PolicyEnforcement mode
	// for Cilium in which traffic is always allowed even if there is a policy
	// selecting endpoints.
	PolicyEnforcementNever = "never"

	// Docker Image names

	// CiliumDockerNetwork is the name of the Docker network which Cilium manages.
	CiliumDockerNetwork = "cilium-net"

	// NetperfImage is the Docker image used for performance testing
	NetperfImage = "tgraf/netperf"

	// HttpdImage is the image used for starting an HTTP server.
	HttpdImage = "cilium/demo-httpd"

	// Names of commonly used containers in tests.

	Httpd1 = "httpd1"
	Httpd2 = "httpd2"
	Httpd3 = "httpd3"
	App1   = "app1"
	App2   = "app2"
	App3   = "app3"
	Client = "client"
	Server = "server"
	Host   = "host"

	// Container lifecycle actions.
	Create = "create"
	Delete = "delete"

	// IP Address families.
	IPv4 = "IPv4"
	IPv6 = "IPv6"

	// Configuration options for endpoints. Copied from endpoint/endpoint.go
	// TODO: these should be converted into types for use in configuration
	// functions instead of using basic strings.

	OptionAllowToHost         = "AllowToHost"
	OptionAllowToWorld        = "AllowToWorld"
	OptionConntrackAccounting = "ConntrackAccounting"
	OptionConntrackLocal      = "ConntrackLocal"
	OptionConntrack           = "Conntrack"
	OptionDebug               = "Debug"
	OptionDropNotify          = "DropNotification"
	OptionTraceNotify         = "TraceNotification"
	OptionNAT46               = "NAT46"
	OptionIngressPolicy       = "IngressPolicy"
	OptionEgressPolicy        = "EgressPolicy"
	OptionIngress             = "ingress"
	OptionEgress              = "egress"
	OptionNone                = "none"
	OptionDisabled            = "Disabled"
	OptionEnabled             = "Enabled"

	PingCount          = 5
	CurlConnectTimeout = 5

	DefaultNamespace    = "default"
	KubeSystemNamespace = "kube-system"

	TestResultsPath = "test_results/"
	RunDir          = "/var/run/cilium"
	LibDir          = "/var/lib/cilium"

	DaemonName             = "cilium"
	CiliumDockerDaemonName = "cilium-docker"
	AgentDaemon            = "cilium-agent"

	GeneratedHTMLManifest   = "html.yaml"
	GeneratedServerManifest = "server.yaml"
	GeneratedClientManifest = "client.yaml"
)

var ciliumCLICommands = map[string]string{
	"cilium endpoint list -o json":   "endpoint_list.txt",
	"cilium service list -o json":    "service_list.txt",
	"cilium config":                  "config.txt",
	"sudo cilium bpf lb list":        "bpf_lb_list.txt",
	"sudo cilium bpf ct list global": "bpf_ct_list.txt",
	"sudo cilium bpf tunnel list":    "bpf_tunnel_list.txt",
	"cilium policy get":              "policy_get.txt",
	"cilium status":                  "status.txt",
}

// ciliumKubCLICommands these commands are the same as `ciliumCLICommands` but
// it'll run inside a container and it does not have sudo support
var ciliumKubCLICommands = map[string]string{
	"cilium endpoint list -o json": "endpoint_list.txt",
	"cilium service list -o json":  "service_list.txt",
	"cilium config":                "config.txt",
	"cilium bpf lb list":           "bpf_lb_list.txt",
	"cilium bpf ct list global":    "bpf_ct_list.txt",
	"cilium bpf tunnel list":       "bpf_tunnel_list.txt",
	"cilium policy get":            "policy_get.txt",
	"cilium status":                "status.txt",
}

//GetFilePath returns the absolute path of the provided filename
func GetFilePath(filename string) string {
	return fmt.Sprintf("%s%s", BasePath, filename)
}

// K8s1VMName is the name of the Kubernetes master node when running K8s tests.
func K8s1VMName() string {
	return fmt.Sprintf("k8s1-%s", GetCurrentK8SEnv())
}

// K8s2VMName is the name of the Kubernetes worker node when running K8s tests.
func K8s2VMName() string {
	return fmt.Sprintf("k8s2-%s", GetCurrentK8SEnv())
}
<|endoftext|>


<commit_before>package sync

import (
	"database/sql"
	"fmt"
	"github.com/daemonl/go_lib/databath"
	"log"
	"reflect"
	"strings"
)

func doErr(err error) {
	if err != nil {
		panic(err)
	}
}

type TableStatus struct {
	Name            string
	Engine          string
	Version         *uint32
	Row_format      *string
	Rows            *uint64
	Avg_row_length  *uint64
	Data_length     *uint64
	Max_data_length *uint64
	Index_length    *uint64
	Data_free       *uint64
	Auto_increment  *uint64
	Create_time     *string
	Update_time     *string
	Check_time      *string
	Collation       *string
	Checksum        *string
	Create_options  *string
	Comment         *string
}

type Column struct {
	Field   string
	Type    string
	Null    string
	Key     *string
	Default *string
	Extra   *string
}

var execString string = ""

func (c *Column) GetString() string {
	built := c.Type
	if c.Null == "NO" {
		built += " NOT NULL"
	} else {
		built += " NULL"
	}
	if c.Extra != nil {
		built += " " + *c.Extra
	}
	built = strings.TrimSpace(built)
	return strings.ToUpper(built)
}

func ScanToStruct(res *sql.Rows, obj interface{}) error {

	rv := reflect.ValueOf(obj)
	rt := reflect.TypeOf(obj)

	if reflect.Indirect(rv).Kind().String() != "struct" {
		panic("KIND NOT STRUCT" + rv.Kind().String())
	}

	valueElm := rv.Elem()

	maxElements := rt.Elem().NumField()
	scanVals := make([]interface{}, maxElements, maxElements)
	for i := 0; i < maxElements; i++ {

		interf := valueElm.Field(i).Addr().Interface()
		scanVals[i] = interf
	}
	err := res.Scan(scanVals...)
	if err != nil {
		return err
	}
	return nil
}

func MustExecF(now bool, db *sql.DB, format string, parameters ...interface{}) {
	q := fmt.Sprintf(format, parameters...)
	log.Println("EXEC: " + q)
	if now {
		_, err := db.Exec(q)
		doErr(err)
	} else {
		execString += fmt.Sprintf("%s;\n", q)

	}

	//log.Printf("RES: %d %d;", res.RowsAffected(), res.LastInsertId())
}

func SyncDb(db *sql.DB, model *databath.Model, now bool) {

	// CREATE DATABASE IF NOT EXISTS #{config.db.database}
	// USE #{config.db.database}
	// Probably won't work - the connection is set to a database.

	res, err := db.Query(`SHOW TABLE STATUS WHERE Engine != 'InnoDB'`)
	doErr(err)

	for res.Next() {
		table := TableStatus{}
		err := ScanToStruct(res, &table)
		doErr(err)
		MustExecF(now, db, "ALTER TABLE %s ENGINE = 'InnoDB'", table.Name)
	}
	res.Close()

	for name, collection := range model.Collections {
		log.Printf("COLLECTION: %s\n", name)
		res, err := db.Query(`SHOW TABLE STATUS WHERE Name = ?`, name)
		doErr(err)
		if res.Next() {
			log.Println("UPDATE TABLE")
			// UPDATE!

			for colName, field := range collection.Fields {
				showRes, err := db.Query(`SHOW COLUMNS FROM `+name+` WHERE Field = ?`, colName)

				doErr(err)
				if showRes.Next() {
					col := Column{}
					err := ScanToStruct(showRes, &col)
					doErr(err)
					colStr := col.GetString()
					modelStr := field.GetMysqlDef()
					if colStr != modelStr {
						log.Printf("'%s' '%s'\n", colStr, modelStr)
						MustExecF(now, db, "ALTER TABLE %s CHANGE COLUMN %s %s %s",
							name, colName, colName, modelStr)
					}
				} else {
					MustExecF(now, db, "ALTER TABLE %s ADD `%s` %s", name, colName, field.GetMysqlDef())
				}
				showRes.Close()
			}

		} else {
			// CRAETE!
			log.Println("CRAETE TABLE")
			params := make([]string, 0, 0)

			for name, field := range collection.Fields {
				params = append(params, name+" "+field.GetMysqlDef())
			}

			params = append(params, "PRIMARY KEY (`id`)")

			MustExecF(now, db, "CREATE TABLE %s (%s)", name, strings.Join(params, ", "))
		}
		res.Close()
	}
	fmt.Println("==========")
	fmt.Println(execString)
}
<commit_msg>Sync fix<commit_after>package sync

import (
	"database/sql"
	"fmt"
	"github.com/daemonl/go_lib/databath"
	"log"
	"reflect"
	"strings"
)

func doErr(err error) {
	if err != nil {
		panic(err)
	}
}

type TableStatus struct {
	Name            string
	Engine          string
	Version         *uint32
	Row_format      *string
	Rows            *uint64
	Avg_row_length  *uint64
	Data_length     *uint64
	Max_data_length *uint64
	Index_length    *uint64
	Data_free       *uint64
	Auto_increment  *uint64
	Create_time     *string
	Update_time     *string
	Check_time      *string
	Collation       *string
	Checksum        *string
	Create_options  *string
	Comment         *string
}

type Column struct {
	Field   string
	Type    string
	Null    string
	Key     *string
	Default *string
	Extra   *string
}

var execString string = ""

func (c *Column) GetString() string {
	built := c.Type
	if c.Null == "NO" {
		built += " NOT NULL"
	} else {
		built += " NULL"
	}
	if c.Extra != nil {
		built += " " + *c.Extra
	}
	built = strings.TrimSpace(built)
	return strings.ToUpper(built)
}

func ScanToStruct(res *sql.Rows, obj interface{}) error {

	rv := reflect.ValueOf(obj)
	rt := reflect.TypeOf(obj)

	if reflect.Indirect(rv).Kind().String() != "struct" {
		panic("KIND NOT STRUCT" + rv.Kind().String())
	}

	valueElm := rv.Elem()

	maxElements := rt.Elem().NumField()
	scanVals := make([]interface{}, maxElements, maxElements)
	for i := 0; i < maxElements; i++ {

		interf := valueElm.Field(i).Addr().Interface()
		scanVals[i] = interf
	}
	err := res.Scan(scanVals...)
	if err != nil {
		return err
	}
	return nil
}

func MustExecF(now bool, db *sql.DB, format string, parameters ...interface{}) {
	q := fmt.Sprintf(format, parameters...)
	log.Println("EXEC: " + q)
	if now {
		_, err := db.Exec(q)
		doErr(err)
	} else {
		execString += fmt.Sprintf("%s;\n", q)

	}

	//log.Printf("RES: %d %d;", res.RowsAffected(), res.LastInsertId())
}

func SyncDb(db *sql.DB, model *databath.Model, now bool) {

	// CREATE DATABASE IF NOT EXISTS #{config.db.database}
	// USE #{config.db.database}
	// Probably won't work - the connection is set to a database.

	res, err := db.Query(`SHOW TABLE STATUS WHERE Engine != 'InnoDB'`)
	doErr(err)

	for res.Next() {
		table := TableStatus{}
		err := ScanToStruct(res, &table)
		doErr(err)
		MustExecF(now, db, "ALTER TABLE %s ENGINE = 'InnoDB'", table.Name)
	}
	res.Close()

	for name, collection := range model.Collections {
		log.Printf("COLLECTION: %s\n", name)
		res, err := db.Query(`SHOW TABLE STATUS WHERE Name = ?`, name)
		doErr(err)
		if res.Next() {
			log.Println("UPDATE TABLE")
			// UPDATE!

			for colName, field := range collection.Fields {
				showRes, err := db.Query(`SHOW COLUMNS FROM `+name+` WHERE Field = ?`, colName)

				doErr(err)
				if showRes.Next() {
					col := Column{}
					err := ScanToStruct(showRes, &col)
					doErr(err)
					colStr := col.GetString()
					modelStr := field.GetMysqlDef()
					if colStr != modelStr {
						log.Printf("'%s' '%s'\n", colStr, modelStr)
						MustExecF(now, db, "ALTER TABLE %s CHANGE COLUMN %s %s %s",
							name, colName, colName, modelStr)
					}
				} else {
					MustExecF(now, db, "ALTER TABLE %s ADD `%s` %s", name, colName, field.GetMysqlDef())
				}
				showRes.Close()
			}

		} else {
			// CRAETE!
			log.Println("CRAETE TABLE")
			params := make([]string, 0, 0)

			for name, field := range collection.Fields {
				params = append(params, fmt.Sprintf("`%s` %s", name, field.GetMysqlDef()))
			}

			params = append(params, "PRIMARY KEY (`id`)")

			MustExecF(now, db, "CREATE TABLE %s (%s)", name, strings.Join(params, ", "))
		}
		res.Close()
	}
	fmt.Println("==========")
	fmt.Println(execString)
}
<|endoftext|>


<commit_before>// +build !providerless

/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package routetableclient

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"net/http"
	"testing"

	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network"
	"github.com/Azure/go-autorest/autorest"
	"github.com/Azure/go-autorest/autorest/to"
	"github.com/golang/mock/gomock"
	"github.com/stretchr/testify/assert"

	azclients "k8s.io/legacy-cloud-providers/azure/clients"
	"k8s.io/legacy-cloud-providers/azure/clients/armclient"
	"k8s.io/legacy-cloud-providers/azure/clients/armclient/mockarmclient"
)

func TestGetNotFound(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Network/routeTables/rt1"
	response := &http.Response{
		StatusCode: http.StatusNotFound,
		Body:       ioutil.NopCloser(bytes.NewReader([]byte("{}"))),
	}
	armClient := mockarmclient.NewMockInterface(ctrl)
	armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(response, nil).Times(1)
	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)

	rtClient := getTestRouteTableClient(armClient)
	expected := network.RouteTable{Response: autorest.Response{}}
	result, rerr := rtClient.Get(context.TODO(), "rg", "rt1", "")
	assert.Equal(t, expected, result)
	assert.NotNil(t, rerr)
	assert.Equal(t, http.StatusNotFound, rerr.HTTPStatusCode)
}

func TestGetInternalError(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Network/routeTables/rt1"
	response := &http.Response{
		StatusCode: http.StatusInternalServerError,
		Body:       ioutil.NopCloser(bytes.NewReader([]byte("{}"))),
	}
	armClient := mockarmclient.NewMockInterface(ctrl)
	armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(response, nil).Times(1)
	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)

	rtClient := getTestRouteTableClient(armClient)
	expected := network.RouteTable{Response: autorest.Response{}}
	result, rerr := rtClient.Get(context.TODO(), "rg", "rt1", "")
	assert.Equal(t, expected, result)
	assert.NotNil(t, rerr)
	assert.Equal(t, http.StatusInternalServerError, rerr.HTTPStatusCode)
}

func TestCreateOrUpdate(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	lb := getTestRouteTable("rt1")
	armClient := mockarmclient.NewMockInterface(ctrl)
	response := &http.Response{
		StatusCode: http.StatusOK,
		Body:       ioutil.NopCloser(bytes.NewReader([]byte(""))),
	}
	armClient.EXPECT().PutResourceWithDecorators(gomock.Any(), to.String(lb.ID), lb, gomock.Any()).Return(response, nil).Times(1)
	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)

	rtClient := getTestRouteTableClient(armClient)
	rerr := rtClient.CreateOrUpdate(context.TODO(), "rg", "rt1", lb, "")
	assert.Nil(t, rerr)
}

func getTestRouteTable(name string) network.RouteTable {
	return network.RouteTable{
		ID:       to.StringPtr(fmt.Sprintf("/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Network/routeTables/%s", name)),
		Name:     to.StringPtr(name),
		Location: to.StringPtr("eastus"),
	}
}

func getTestRouteTableClient(armClient armclient.Interface) *Client {
	rateLimiterReader, rateLimiterWriter := azclients.NewRateLimiter(&azclients.RateLimitConfig{})
	return &Client{
		armClient:         armClient,
		subscriptionID:    "subscriptionID",
		rateLimiterReader: rateLimiterReader,
		rateLimiterWriter: rateLimiterWriter,
	}
}
<commit_msg>Improves unittest CC for azure_routetableclient<commit_after>// +build !providerless

/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package routetableclient

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"net/http"
	"testing"
	"time"

	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network"
	"github.com/Azure/go-autorest/autorest"
	"github.com/Azure/go-autorest/autorest/to"
	"github.com/golang/mock/gomock"
	"github.com/stretchr/testify/assert"

	"k8s.io/client-go/util/flowcontrol"
	azclients "k8s.io/legacy-cloud-providers/azure/clients"
	"k8s.io/legacy-cloud-providers/azure/clients/armclient"
	"k8s.io/legacy-cloud-providers/azure/clients/armclient/mockarmclient"
	"k8s.io/legacy-cloud-providers/azure/retry"
)

// 2065-01-24 05:20:00 +0000 UTC
func getFutureTime() time.Time {
	return time.Unix(3000000000, 0)
}

func TestNew(t *testing.T) {
	config := &azclients.ClientConfig{
		SubscriptionID:          "sub",
		ResourceManagerEndpoint: "endpoint",
		Location:                "eastus",
		RateLimitConfig: &azclients.RateLimitConfig{
			CloudProviderRateLimit:            true,
			CloudProviderRateLimitQPS:         0.5,
			CloudProviderRateLimitBucket:      1,
			CloudProviderRateLimitQPSWrite:    0.5,
			CloudProviderRateLimitBucketWrite: 1,
		},
		Backoff: &retry.Backoff{Steps: 1},
	}

	routetableClient := New(config)
	assert.Equal(t, "sub", routetableClient.subscriptionID)
	assert.NotEmpty(t, routetableClient.rateLimiterReader)
	assert.NotEmpty(t, routetableClient.rateLimiterWriter)
}

func TestGet(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Network/routeTables/rt1"
	response := &http.Response{
		StatusCode: http.StatusOK,
		Body:       ioutil.NopCloser(bytes.NewReader([]byte("{}"))),
	}

	armClient := mockarmclient.NewMockInterface(ctrl)
	armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(response, nil).Times(1)
	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)

	routetableClient := getTestRouteTableClient(armClient)
	expected := network.RouteTable{}
	expected.Response = autorest.Response{Response: response}
	result, rerr := routetableClient.Get(context.TODO(), "rg", "rt1", "")
	assert.Equal(t, expected, result)
	assert.Nil(t, rerr)
}

func TestGetNotFound(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Network/routeTables/rt1"
	response := &http.Response{
		StatusCode: http.StatusNotFound,
		Body:       ioutil.NopCloser(bytes.NewReader([]byte("{}"))),
	}
	armClient := mockarmclient.NewMockInterface(ctrl)
	armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(response, nil).Times(1)
	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)

	rtClient := getTestRouteTableClient(armClient)
	expected := network.RouteTable{Response: autorest.Response{}}
	result, rerr := rtClient.Get(context.TODO(), "rg", "rt1", "")
	assert.Equal(t, expected, result)
	assert.NotNil(t, rerr)
	assert.Equal(t, http.StatusNotFound, rerr.HTTPStatusCode)
}

func TestGetInternalError(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Network/routeTables/rt1"
	response := &http.Response{
		StatusCode: http.StatusInternalServerError,
		Body:       ioutil.NopCloser(bytes.NewReader([]byte("{}"))),
	}
	armClient := mockarmclient.NewMockInterface(ctrl)
	armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(response, nil).Times(1)
	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)

	rtClient := getTestRouteTableClient(armClient)
	expected := network.RouteTable{Response: autorest.Response{}}
	result, rerr := rtClient.Get(context.TODO(), "rg", "rt1", "")
	assert.Equal(t, expected, result)
	assert.NotNil(t, rerr)
	assert.Equal(t, http.StatusInternalServerError, rerr.HTTPStatusCode)
}

func TestGetNeverRateLimiter(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	rtGetErr := &retry.Error{
		RawError:  fmt.Errorf("azure cloud provider rate limited(%s) for operation %q", "read", "RouteTableGet"),
		Retriable: true,
	}

	armClient := mockarmclient.NewMockInterface(ctrl)

	routetableClient := getTestRouteTableClientWithNeverRateLimiter(armClient)
	expected := network.RouteTable{}
	result, rerr := routetableClient.Get(context.TODO(), "rg", "rt1", "")
	assert.Equal(t, expected, result)
	assert.Equal(t, rtGetErr, rerr)
}

func TestGetRetryAfterReader(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	rtGetErr := &retry.Error{
		RawError:   fmt.Errorf("azure cloud provider throttled for operation %s with reason %q", "RouteTableGet", "client throttled"),
		Retriable:  true,
		RetryAfter: getFutureTime(),
	}

	armClient := mockarmclient.NewMockInterface(ctrl)

	routetableClient := getTestRouteTableClientWithRetryAfterReader(armClient)
	expected := network.RouteTable{}
	result, rerr := routetableClient.Get(context.TODO(), "rg", "rt1", "")
	assert.Equal(t, expected, result)
	assert.Equal(t, rtGetErr, rerr)
}

func TestGetThrottle(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Network/routeTables/rt1"
	response := &http.Response{
		StatusCode: http.StatusTooManyRequests,
		Body:       ioutil.NopCloser(bytes.NewReader([]byte("{}"))),
	}
	throttleErr := &retry.Error{
		HTTPStatusCode: http.StatusTooManyRequests,
		RawError:       fmt.Errorf("error"),
		Retriable:      true,
		RetryAfter:     time.Unix(100, 0),
	}
	armClient := mockarmclient.NewMockInterface(ctrl)
	armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(response, throttleErr).Times(1)
	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)

	routetableClient := getTestRouteTableClient(armClient)
	result, rerr := routetableClient.Get(context.TODO(), "rg", "rt1", "")
	assert.Empty(t, result)
	assert.Equal(t, throttleErr, rerr)
}

func TestCreateOrUpdate(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	rt1 := getTestRouteTable("rt1")
	armClient := mockarmclient.NewMockInterface(ctrl)
	response := &http.Response{
		StatusCode: http.StatusOK,
		Body:       ioutil.NopCloser(bytes.NewReader([]byte(""))),
	}
	armClient.EXPECT().PutResourceWithDecorators(gomock.Any(), to.String(rt1.ID), rt1, gomock.Any()).Return(response, nil).Times(1)
	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)

	rtClient := getTestRouteTableClient(armClient)
	rerr := rtClient.CreateOrUpdate(context.TODO(), "rg", "rt1", rt1, "*")
	assert.Nil(t, rerr)
}

func TestCreateOrUpdateWithNeverRateLimiter(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	rcCreateOrUpdatetErr := &retry.Error{
		RawError:  fmt.Errorf("azure cloud provider rate limited(%s) for operation %q", "write", "RouteTableCreateOrUpdate"),
		Retriable: true,
	}

	rt1 := getTestRouteTable("rt1")
	armClient := mockarmclient.NewMockInterface(ctrl)

	routetableClient := getTestRouteTableClientWithNeverRateLimiter(armClient)
	rerr := routetableClient.CreateOrUpdate(context.TODO(), "rg", "rt1", rt1, "")
	assert.Equal(t, rcCreateOrUpdatetErr, rerr)
}

func TestCreateOrUpdateRetryAfterReader(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	rcCreateOrUpdateErr := &retry.Error{
		RawError:   fmt.Errorf("azure cloud provider throttled for operation %s with reason %q", "RouteTableCreateOrUpdate", "client throttled"),
		Retriable:  true,
		RetryAfter: getFutureTime(),
	}

	rt1 := getTestRouteTable("rt1")
	armClient := mockarmclient.NewMockInterface(ctrl)

	routetableClient := getTestRouteTableClientWithRetryAfterReader(armClient)
	rerr := routetableClient.CreateOrUpdate(context.TODO(), "rg", "rt1", rt1, "")
	assert.NotNil(t, rerr)
	assert.Equal(t, rcCreateOrUpdateErr, rerr)
}

func TestCreateOrUpdateThrottle(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	response := &http.Response{
		StatusCode: http.StatusTooManyRequests,
		Body:       ioutil.NopCloser(bytes.NewReader([]byte("{}"))),
	}
	throttleErr := &retry.Error{
		HTTPStatusCode: http.StatusTooManyRequests,
		RawError:       fmt.Errorf("error"),
		Retriable:      true,
		RetryAfter:     time.Unix(100, 0),
	}

	rt1 := getTestRouteTable("rt1")
	armClient := mockarmclient.NewMockInterface(ctrl)
	armClient.EXPECT().PutResourceWithDecorators(gomock.Any(), to.String(rt1.ID), rt1, gomock.Any()).Return(response, throttleErr).Times(1)
	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)

	routetableClient := getTestRouteTableClient(armClient)
	rerr := routetableClient.CreateOrUpdate(context.TODO(), "rg", "rt1", rt1, "")
	assert.Equal(t, throttleErr, rerr)
}

func TestCreateOrUpdateWithCreateOrUpdateResponderError(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	rt1 := getTestRouteTable("rt1")
	armClient := mockarmclient.NewMockInterface(ctrl)
	response := &http.Response{
		StatusCode: http.StatusNotFound,
		Body:       ioutil.NopCloser(bytes.NewReader([]byte(""))),
	}

	armClient.EXPECT().PutResourceWithDecorators(gomock.Any(), to.String(rt1.ID), rt1, gomock.Any()).Return(response, nil).Times(1)
	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)

	routetableClient := getTestRouteTableClient(armClient)
	rerr := routetableClient.CreateOrUpdate(context.TODO(), "rg", "rt1", rt1, "")
	assert.NotNil(t, rerr)
}

func getTestRouteTable(name string) network.RouteTable {
	return network.RouteTable{
		ID:       to.StringPtr(fmt.Sprintf("/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Network/routeTables/%s", name)),
		Name:     to.StringPtr(name),
		Location: to.StringPtr("eastus"),
	}
}

func getTestRouteTableClient(armClient armclient.Interface) *Client {
	rateLimiterReader, rateLimiterWriter := azclients.NewRateLimiter(&azclients.RateLimitConfig{})
	return &Client{
		armClient:         armClient,
		subscriptionID:    "subscriptionID",
		rateLimiterReader: rateLimiterReader,
		rateLimiterWriter: rateLimiterWriter,
	}
}

func getTestRouteTableClientWithNeverRateLimiter(armClient armclient.Interface) *Client {
	rateLimiterReader := flowcontrol.NewFakeNeverRateLimiter()
	rateLimiterWriter := flowcontrol.NewFakeNeverRateLimiter()
	return &Client{
		armClient:         armClient,
		subscriptionID:    "subscriptionID",
		rateLimiterReader: rateLimiterReader,
		rateLimiterWriter: rateLimiterWriter,
	}
}

func getTestRouteTableClientWithRetryAfterReader(armClient armclient.Interface) *Client {
	rateLimiterReader := flowcontrol.NewFakeAlwaysRateLimiter()
	rateLimiterWriter := flowcontrol.NewFakeAlwaysRateLimiter()
	return &Client{
		armClient:         armClient,
		subscriptionID:    "subscriptionID",
		rateLimiterReader: rateLimiterReader,
		rateLimiterWriter: rateLimiterWriter,
		RetryAfterReader:  getFutureTime(),
		RetryAfterWriter:  getFutureTime(),
	}
}
<|endoftext|>


<commit_before>package main

import (
	"cred-alert/ingestor"
	"cred-alert/metrics"
	"cred-alert/queue"
	"fmt"
	"net/http"
	"os"

	"code.cloudfoundry.org/lager"
	flags "github.com/jessevdk/go-flags"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/http_server"
	"github.com/tedsuo/ifrit/sigmon"
)

type Opts struct {
	Port     uint16 `short:"p" long:"port" description:"the port to listen on" default:"8080" env:"PORT" value-name:"PORT"`
	Endpoint string `long:"endpoint" description:"the endpoint to forward tasks to" env:"ENDPOINT" value-name:"URL" required:"true"`

	GitHub struct {
		WebhookToken string `short:"w" long:"webhook-token" description:"github webhook secret token" env:"GITHUB_WEBHOOK_SECRET_KEY" value-name:"TOKEN" required:"true"`
	} `group:"GitHub Options"`

	Metrics struct {
		DatadogAPIKey string `long:"datadog-api-key" description:"key to emit to datadog" env:"DATADOG_API_KEY" value-name:"KEY"`
		Environment   string `long:"environment" description:"environment tag for metrics" env:"ENVIRONMENT" value-name:"NAME" default:"development"`
	} `group:"Metrics Options"`
}

func main() {
	var opts Opts

	logger := lager.NewLogger("revok-ingestor")
	logger.RegisterSink(lager.NewWriterSink(os.Stdout, lager.INFO))

	logger.Debug("starting")

	_, err := flags.ParseArgs(&opts, os.Args)
	if err != nil {
		logger.Fatal("failed", err)
		os.Exit(1)
	}

	emitter := metrics.BuildEmitter(opts.Metrics.DatadogAPIKey, opts.Metrics.Environment)
	generator := queue.NewGenerator()

	enqueuer := queue.NewHTTPEnqueuer(logger, opts.Endpoint)
	in := ingestor.NewIngestor(enqueuer, emitter, "revok", generator)

	router := http.NewServeMux()
	router.Handle("/webhook", ingestor.Handler(logger, in, opts.GitHub.WebhookToken))

	members := []grouper.Member{
		{"api", http_server.New(fmt.Sprintf(":%d", opts.Port), router)},
	}

	runner := sigmon.New(grouper.NewParallel(os.Interrupt, members))

	serverLogger := logger.Session("server", lager.Data{
		"port": opts.Port,
	})
	serverLogger.Info("starting")
	err = <-ifrit.Invoke(runner).Wait()
	if err != nil {
		serverLogger.Error("failed", err)
	}
}
<commit_msg>WebhookToken -> WebhookSecretToken<commit_after>package main

import (
	"cred-alert/ingestor"
	"cred-alert/metrics"
	"cred-alert/queue"
	"fmt"
	"net/http"
	"os"

	"code.cloudfoundry.org/lager"
	flags "github.com/jessevdk/go-flags"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/http_server"
	"github.com/tedsuo/ifrit/sigmon"
)

type Opts struct {
	Port     uint16 `short:"p" long:"port" description:"the port to listen on" default:"8080" env:"PORT" value-name:"PORT"`
	Endpoint string `long:"endpoint" description:"the endpoint to forward tasks to" env:"ENDPOINT" value-name:"URL" required:"true"`

	GitHub struct {
		WebhookSecretToken string `short:"w" long:"github-webhook-secret-token" description:"github webhook secret token" env:"GITHUB_WEBHOOK_SECRET_TOKEN" value-name:"TOKEN" required:"true"`
	} `group:"GitHub Options"`

	Metrics struct {
		DatadogAPIKey string `long:"datadog-api-key" description:"key to emit to datadog" env:"DATADOG_API_KEY" value-name:"KEY"`
		Environment   string `long:"environment" description:"environment tag for metrics" env:"ENVIRONMENT" value-name:"NAME" default:"development"`
	} `group:"Metrics Options"`
}

func main() {
	var opts Opts

	logger := lager.NewLogger("revok-ingestor")
	logger.RegisterSink(lager.NewWriterSink(os.Stdout, lager.INFO))

	logger.Debug("starting")

	_, err := flags.ParseArgs(&opts, os.Args)
	if err != nil {
		logger.Fatal("failed", err)
		os.Exit(1)
	}

	emitter := metrics.BuildEmitter(opts.Metrics.DatadogAPIKey, opts.Metrics.Environment)
	generator := queue.NewGenerator()

	enqueuer := queue.NewHTTPEnqueuer(logger, opts.Endpoint)
	in := ingestor.NewIngestor(enqueuer, emitter, "revok", generator)

	router := http.NewServeMux()
	router.Handle("/webhook", ingestor.Handler(logger, in, opts.GitHub.WebhookSecretToken))

	members := []grouper.Member{
		{"api", http_server.New(fmt.Sprintf(":%d", opts.Port), router)},
	}

	runner := sigmon.New(grouper.NewParallel(os.Interrupt, members))

	serverLogger := logger.Session("server", lager.Data{
		"port": opts.Port,
	})
	serverLogger.Info("starting")
	err = <-ifrit.Invoke(runner).Wait()
	if err != nil {
		serverLogger.Error("failed", err)
	}
}
<|endoftext|>