<commit_before>package spy

import (
	"time"
	"math/rand"
	"fmt"
	"github.com/Workiva/go-datastructures/queue"
	"github.com/ridewindx/crumb/dnscache"
)

type IFetcher interface {
	Fetch(req *Request, spider ISpider) (*Response, *Request, error)
}

type Fetcher struct {
	Handlers map[string]FetcherHandler
	TotalConcurrency int
	DomainConcurrency int
	IpConcurrency int
	Delay time.Duration
	RandomizeDelay bool
	*FetcherMiddlewareManager
	slots map[string]*Slot
	dnscache *dnscache.Resolver
	active map[*Request]struct{}
	*rand.Rand
}

type FetcherHandler interface {
	Fetch(request *Request, spider ISpider) (*Response, error)
	Close()
}

type fetchresult struct {
	rep *Response
	err error
}

type task struct {
	request *Request
	result chan *fetchresult
}

type Slot struct {
	concurrency int
	tasks       chan *task
	active      int
	lastSeen    time.Time
}

func NewFetcher() *Fetcher {
	return &Fetcher{
		Rand: rand.New(rand.NewSource(time.Now().UTC().UnixNano())),
	}
}

func (f *Fetcher) Open(spider ISpider) {

}

func (f *Fetcher) Close(spider ISpider) {
	for _, handler := range f.Handlers {
		handler.Close()
	}
}

func (f *Fetcher) Fetch(req *Request, spider ISpider) (*Response, *Request, error) {
	f.active[req] = struct{}{}
	defer delete(f.active, req)

	return f.FetcherMiddlewareManager.Fetch(f.fetchRequest, req, spider)
}

func (f *Fetcher) NeedsBackout() bool {
	return len(f.active) >= f.TotalConcurrency
}

func (f *Fetcher) fetchRequest(req *Request, spider ISpider) (*Response, error) {
	key := f.getSlotKey(req)
	slot, ok := f.slots[key]
	if !ok {
		slot = f.addSlot(key, spider)
	}

	result := make(chan *fetchresult)

	slot.tasks <- &task{req, result}

	r := <- result
	return r.rep, r.err
}

func (f *Fetcher) work(slot *Slot, spider ISpider) {
	for {
		now := time.Now()
		delay := f.computedDelay(spider)
		if delay > 0 {
			penalty := delay - now.Sub(slot.lastSeen)
			if penalty > 0 {
				time.Sleep(penalty)
				continue
			}
		}

		for len(slot.tasks) > 0 && (slot.concurrency-slot.active) > 0 {
			task := <-slot.tasks

			slot.lastSeen = time.Now()
			go f.fetch(slot, task, spider)

			if delay > 0 {
				break
			}
		}
	}
}

func (f *Fetcher) fetch(slot *Slot, task *task, spider ISpider) {
	slot.active++
	defer func() {
		slot.active--
	}()

	req := task.request
	var rep *Response
	var err error
	scheme := req.Request.URL.Scheme // TODO: not only http url
	handler, ok := f.Handlers[scheme]
	if ok {
		rep, err = handler.Fetch(req, spider)
	} else {
		err = fmt.Errorf("unsupported URL scheme '%s'", scheme)
	}
	task.result <- &fetchresult{rep, err}
}

func (f *Fetcher) computedDelay(spider ISpider) time.Duration {
	var delay time.Duration
	if d := spider.FetchDelay(); d > 0 {
		delay = time.Duration(d)
	} else {
		delay = f.Delay
	}

	if f.RandomizeDelay {
		return time.Duration(0.5*delay + f.Rand.Int63n(int64(delay)))
	} else {
		return delay
	}
}

func (f *Fetcher) getSlotKey(req *Request) string {
	var key string
	if k, ok := req.Meta["downloadSlot"]; ok {
		key = k.(string)
	} else {
		key = req.URL.Host // TODO: strip port
		if f.IpConcurrency {
			k, err := f.dnscache.FetchOneString(key)
			if err == nil {
				key = k
			}
		}
		req.Meta["downloadSlot"] = key
	}
	return key
}

func (f *Fetcher) addSlot(key string, spider ISpider) *Slot {
	slot := &Slot{}
	slot.concurrency = spider.ConcurrentRequests()
	if slot.concurrency == 0 {
		if f.IpConcurrency > 0 {
			slot.concurrency = f.IpConcurrency
		} else {
			slot.concurrency = f.DomainConcurrency
		}
	}
	slot.tasks = make(chan *task, slot.concurrency)

	go f.work(slot, spider)

	f.slots[key] = slot
	return slot
}

type FetcherMiddleware interface {}

type FetchingRequestProcessor interface {
	/* ProcessRequest is called for each request that goes through the fetcher middleware.
	It should either: return nils, return a Response object, return a Request object, or return an IgnoreRequest error.

	If it returns nils, will continue processing this request, executing all other middlewares,
	until, finally, the appropriate fetcher handler performs the request performed (and its response downloaded).

	If it returns a Response object, won’t bother calling any other ProcessRequest or ProcessError methods,
	or the appropriate fetcher handler; it’ll return that response. The ProcessResponse methods of installed middleware
	is always called on every response.

	If it returns a Request object, will stop calling ProcessRequest methods and reschedule the returned request.
	Once the newly returned request is performed, the appropriate middleware chain will again be called on the downloaded response.

	If it returns an IgnoreRequest error, the ProcessError methods of installed downloader middleware will be called.
	If none of them handle the error, the callback function of the request (Request.Callback) is called.
	If no code handles the returned error, it is ignored and not logged (unlike other error).
	 */
	ProcessRequest(request *Request, spider ISpider) (*Response, *Request, error)
}

type FetchingResponseProcessor interface {
	/* ProcessResponse should either: return a Response object, return a Request object or return an IgnoreRequest error.

	If it returns a Response (it could be the same given response, or a brand-new one), that response will continue to be
	processed with the ProcessResponse method of the next middleware in the chain.

	If it returns a Request object, the middleware chain is halted and the returned request is rescheduled to be performed
	in the future. This is the same behavior as if a request is returned from ProcessRequest.

	If it returns an IgnoreRequest error, the callback function of the request (Request.Callback) is called.
	If no code handles the returned error, it is ignored and not logged (unlike other error).
	 */
	ProcessResponse(response *Response, request *Request, spider ISpider) (*Response, *Request, error)
}

type FetchingErrorProcessor interface {
	/* ProcessError is called when a fetcher handler or a ProcessRequest (from a fetcher middleware) returns an error.
	It should return: either nils, a Response object, or a Request object.

	If it returns nils, will continue processing this error, executing any other ProcessError methods of installed
	middleware, until no middleware is left.

	If it returns a Response object, the ProcessResponse methods chain of installed middleware is started,
	and won’t bother calling any other ProcessError methods of middleware.

	If it returns a Request object, the returned request is rescheduled to be performed in the future.
	This stops the execution of ProcessError methods of the middleware the same as returning a response would.
	 */
	ProcessError(err error, request *Request, spider ISpider) (*Response, *Request)
}

type FetcherMiddlewareManager struct {
	middlewares []FetcherMiddleware
	requestProcessors []FetchingRequestProcessor
	responseProcessors []FetchingResponseProcessor
	errorProcessors []FetchingErrorProcessor
}

func (fmm *FetcherMiddlewareManager) Register(middleware FetcherMiddleware) {
	fmm.middlewares = append(fmm.middlewares, middleware)

	if p, ok := middleware.(FetchingRequestProcessor); ok {
		fmm.requestProcessors = append(fmm.requestProcessors, p)
	}
	if p, ok := middleware.(FetchingResponseProcessor); ok {
		fmm.responseProcessors = append([]FetchingResponseProcessor{p}, fmm.responseProcessors...)
	}
	if p, ok := middleware.(FetchingErrorProcessor); ok {
		fmm.errorProcessors = append([]FetchingErrorProcessor{p}, fmm.errorProcessors...)
	}
}

func (fmm *FetcherMiddlewareManager) Fetch(fetchFunc func(*Request, ISpider) (*Response, error), request *Request, spider ISpider) (*Response, *Request, error) {
	var rep *Response
	var req *Request
	var err error

	var none = true
	for _, processor := range fmm.requestProcessors { // chain handles request
		rep, req, err = processor.ProcessRequest(request, spider)
		if rep != nil || req != nil || err != nil {
			none = false
			break
		}
	}

	if none {
		rep, err = fetchFunc(request, spider) // really fetch the request
		assert(rep != nil || err != nil)
	}

	if err != nil {
		for _, processor := range fmm.errorProcessors { // chain handles error
			rep, req = processor.ProcessError(err, request, spider)
			if rep != nil || req != nil {
				err = nil // ProcessError eliminated error
				break
			}
		}
	}

	assert(rep != nil || req != nil || err != nil)

	if err != nil {
		return nil, nil, err
	}

	if req != nil {
		return nil, req, nil // reschedule the request
	}

	for _, processor := range fmm.responseProcessors { // chain handles response
		rep, req, err = processor.ProcessResponse(rep, request, spider)
		if req != nil {
			return nil, req, nil // reschedule the request
		}
		if err != nil {
			return nil, nil, err
		}
		assert(rep != nil)
	}

	return rep, nil, nil
}
<commit_msg>Update<commit_after>package spy

import (
	"time"
	"math/rand"
	"fmt"
	"sync"
	"github.com/ridewindx/crumb/dnscache"
)

type IFetcher interface {
	Fetch(req *Request, spider ISpider) (*Response, *Request, error)
}

type Fetcher struct {
	Handlers          map[string]FetcherHandler
	TotalConcurrency  int
	DomainConcurrency int
	IpConcurrency     int
	Delay             time.Duration
	RandomizeDelay    bool
	*FetcherMiddlewareManager
	slots             map[string]*Slot
	dnscache          *dnscache.Resolver
	active            map[*Request]struct{}
	*rand.Rand
	mutex             *sync.RWMutex
	closed            chan struct{}
	waitGroup *sync.WaitGroup
}

type FetcherHandler interface {
	Fetch(request *Request, spider ISpider) (*Response, error)
	Close()
}

type fetchresult struct {
	rep *Response
	err error
}

type task struct {
	request *Request
	result chan *fetchresult
}

type Slot struct {
	concurrency int
	tasks       chan *task
	active      int
	lastSeen    time.Time
	closed chan struct{}
}

func NewFetcher() *Fetcher {
	return &Fetcher{
		Rand: rand.New(rand.NewSource(time.Now().UTC().UnixNano())),
	}
}

func (f *Fetcher) Open(spider ISpider) {

}

func (f *Fetcher) Close(spider ISpider) {
	for _, handler := range f.Handlers {
		handler.Close()
	}

	close(f.closed)
	for _, slot := range f.slots {
		close(slot.closed)
	}

	f.waitGroup.Wait()
}

func (f *Fetcher) Fetch(req *Request, spider ISpider) (*Response, *Request, error) {
	f.active[req] = struct{}{}
	defer delete(f.active, req)

	return f.FetcherMiddlewareManager.Fetch(f.fetchRequest, req, spider)
}

func (f *Fetcher) NeedsBackout() bool {
	return len(f.active) >= f.TotalConcurrency
}

func (f *Fetcher) fetchRequest(req *Request, spider ISpider) (*Response, error) {
	key := f.getSlotKey(req)
	f.mutex.RLock()
	slot, ok := f.slots[key]
	f.mutex.RUnlock()
	if !ok {
		slot = f.addSlot(key, spider)
	}

	result := make(chan *fetchresult)

	slot.tasks <- &task{req, result}

	r := <- result
	return r.rep, r.err
}

func (f *Fetcher) work(slot *Slot, spider ISpider) {
	f.waitGroup.Add(1)
	defer f.waitGroup.Done()

	for {
		select {
		case <-slot.closed:
			return

		default:
			now := time.Now()
			delay := f.computedDelay(spider)
			if delay > 0 {
				penalty := delay - now.Sub(slot.lastSeen)
				if penalty > 0 {
					time.Sleep(penalty)
					continue
				}
			}

			for len(slot.tasks) > 0 && (slot.concurrency-slot.active) > 0 {
				task := <-slot.tasks

				slot.lastSeen = time.Now()
				f.waitGroup.Add(1)
				go f.fetch(slot, task, spider)

				if delay > 0 {
					break
				}
			}
		}
	}
}

func (f *Fetcher) fetch(slot *Slot, task *task, spider ISpider) {
	defer f.waitGroup.Done()

	slot.active++
	defer func() {
		slot.active--
	}()

	req := task.request
	var rep *Response
	var err error
	scheme := req.Request.URL.Scheme // TODO: not only http url
	handler, ok := f.Handlers[scheme]
	if ok {
		rep, err = handler.Fetch(req, spider)
	} else {
		err = fmt.Errorf("unsupported URL scheme '%s'", scheme)
	}
	task.result <- &fetchresult{rep, err}
}

func (f *Fetcher) computedDelay(spider ISpider) time.Duration {
	var delay time.Duration
	if d := spider.FetchDelay(); d > 0 {
		delay = time.Duration(d)
	} else {
		delay = f.Delay
	}

	if f.RandomizeDelay {
		return time.Duration(0.5*delay + f.Rand.Int63n(int64(delay)))
	} else {
		return delay
	}
}

func (f *Fetcher) getSlotKey(req *Request) string {
	var key string
	if k, ok := req.Meta["downloadSlot"]; ok {
		key = k.(string)
	} else {
		key = req.URL.Host // TODO: strip port
		if f.IpConcurrency {
			k, err := f.dnscache.FetchOneString(key)
			if err == nil {
				key = k
			}
		}
		req.Meta["downloadSlot"] = key
	}
	return key
}

func (f *Fetcher) addSlot(key string, spider ISpider) *Slot {
	slot := &Slot{
		concurrency: spider.ConcurrentRequests(),
		closed: make(chan struct{}),
	}
	if slot.concurrency == 0 {
		if f.IpConcurrency > 0 {
			slot.concurrency = f.IpConcurrency
		} else {
			slot.concurrency = f.DomainConcurrency
		}
	}
	slot.tasks = make(chan *task, slot.concurrency)

	go f.work(slot, spider)

	f.mutex.Lock()
	f.slots[key] = slot
	f.mutex.Unlock()

	return slot
}

func (f *Fetcher) purgeSlots() {
	ticker := time.NewTicker(time.Minute)
	for {
		select {
		case <-f.closed:
			ticker.Stop()
			return

		case <-ticker.C:
			idleSlots := make(map[string]*Slot)
			f.mutex.RLock()
			for key, slot := range f.slots {
				if len(slot.tasks) == 0 && slot.active == 0 {
					idleSlots[key] = slot
				}
			}
			f.mutex.RUnlock()

			f.mutex.Lock()
			for key, slot := range idleSlots {
				delete(f.slots, key)
				close(slot.closed)
			}
			f.mutex.RUnlock()
		}
	}
}

type FetcherMiddleware interface {}

type FetchingRequestProcessor interface {
	/* ProcessRequest is called for each request that goes through the fetcher middleware.
	It should either: return nils, return a Response object, return a Request object, or return an IgnoreRequest error.

	If it returns nils, will continue processing this request, executing all other middlewares,
	until, finally, the appropriate fetcher handler performs the request performed (and its response downloaded).

	If it returns a Response object, won’t bother calling any other ProcessRequest or ProcessError methods,
	or the appropriate fetcher handler; it’ll return that response. The ProcessResponse methods of installed middleware
	is always called on every response.

	If it returns a Request object, will stop calling ProcessRequest methods and reschedule the returned request.
	Once the newly returned request is performed, the appropriate middleware chain will again be called on the downloaded response.

	If it returns an IgnoreRequest error, the ProcessError methods of installed downloader middleware will be called.
	If none of them handle the error, the callback function of the request (Request.Callback) is called.
	If no code handles the returned error, it is ignored and not logged (unlike other error).
	 */
	ProcessRequest(request *Request, spider ISpider) (*Response, *Request, error)
}

type FetchingResponseProcessor interface {
	/* ProcessResponse should either: return a Response object, return a Request object or return an IgnoreRequest error.

	If it returns a Response (it could be the same given response, or a brand-new one), that response will continue to be
	processed with the ProcessResponse method of the next middleware in the chain.

	If it returns a Request object, the middleware chain is halted and the returned request is rescheduled to be performed
	in the future. This is the same behavior as if a request is returned from ProcessRequest.

	If it returns an IgnoreRequest error, the callback function of the request (Request.Callback) is called.
	If no code handles the returned error, it is ignored and not logged (unlike other error).
	 */
	ProcessResponse(response *Response, request *Request, spider ISpider) (*Response, *Request, error)
}

type FetchingErrorProcessor interface {
	/* ProcessError is called when a fetcher handler or a ProcessRequest (from a fetcher middleware) returns an error.
	It should return: either nils, a Response object, or a Request object.

	If it returns nils, will continue processing this error, executing any other ProcessError methods of installed
	middleware, until no middleware is left.

	If it returns a Response object, the ProcessResponse methods chain of installed middleware is started,
	and won’t bother calling any other ProcessError methods of middleware.

	If it returns a Request object, the returned request is rescheduled to be performed in the future.
	This stops the execution of ProcessError methods of the middleware the same as returning a response would.
	 */
	ProcessError(err error, request *Request, spider ISpider) (*Response, *Request)
}

type FetcherMiddlewareManager struct {
	middlewares []FetcherMiddleware
	requestProcessors []FetchingRequestProcessor
	responseProcessors []FetchingResponseProcessor
	errorProcessors []FetchingErrorProcessor
}

func (fmm *FetcherMiddlewareManager) Register(middleware FetcherMiddleware) {
	fmm.middlewares = append(fmm.middlewares, middleware)

	if p, ok := middleware.(FetchingRequestProcessor); ok {
		fmm.requestProcessors = append(fmm.requestProcessors, p)
	}
	if p, ok := middleware.(FetchingResponseProcessor); ok {
		fmm.responseProcessors = append([]FetchingResponseProcessor{p}, fmm.responseProcessors...)
	}
	if p, ok := middleware.(FetchingErrorProcessor); ok {
		fmm.errorProcessors = append([]FetchingErrorProcessor{p}, fmm.errorProcessors...)
	}
}

func (fmm *FetcherMiddlewareManager) Fetch(fetchFunc func(*Request, ISpider) (*Response, error), request *Request, spider ISpider) (*Response, *Request, error) {
	var rep *Response
	var req *Request
	var err error

	var none = true
	for _, processor := range fmm.requestProcessors { // chain handles request
		rep, req, err = processor.ProcessRequest(request, spider)
		if rep != nil || req != nil || err != nil {
			none = false
			break
		}
	}

	if none {
		rep, err = fetchFunc(request, spider) // really fetch the request
		assert(rep != nil || err != nil)
	}

	if err != nil {
		for _, processor := range fmm.errorProcessors { // chain handles error
			rep, req = processor.ProcessError(err, request, spider)
			if rep != nil || req != nil {
				err = nil // ProcessError eliminated error
				break
			}
		}
	}

	assert(rep != nil || req != nil || err != nil)

	if err != nil {
		return nil, nil, err
	}

	if req != nil {
		return nil, req, nil // reschedule the request
	}

	for _, processor := range fmm.responseProcessors { // chain handles response
		rep, req, err = processor.ProcessResponse(rep, request, spider)
		if req != nil {
			return nil, req, nil // reschedule the request
		}
		if err != nil {
			return nil, nil, err
		}
		assert(rep != nil)
	}

	return rep, nil, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/alexjohnj/flasher/tbutils"
	"github.com/codegangsta/cli"
	"github.com/nsf/termbox-go"
	"log"
	"os"
)

func main() {
	app := cli.NewApp()
	app.Name = "flasher"
	app.Version = "0.3.0-DEV"
	app.Author = "Alex Jackson"
	app.Email = "alex@alexj.org"

	app.Commands = []cli.Command{
		{
			Name:      "flash",
			ShortName: "f",
			Usage:     "flasher flash [flashcard-file.json]",
			Action:    cliFlash,
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "no-shuffle, n",
					Usage: "Presents flashcards in the order they are written in the source JSON file.",
				},
			},
		},
		{
			Name:      "info",
			ShortName: "i",
			Usage:     "flasher info [flashcard-file.json]",
			Action:    cliInfo,
		},
	}
	app.Run(os.Args)
}

func cliFlash(c *cli.Context) {
	// Load flashcards

	if len(c.Args()) != 1 {
		log.Printf("Incorrect usage\n")
		cli.ShowCommandHelp(c, "flash")
		os.Exit(1)
	}

	flashcardStack := new(cardStack)
	err := flashcardStack.loadFlashcardStack(c.Args()[0])

	if err != nil {
		log.Fatal(err)
	}

	if !c.Bool("no-shuffle") {
		flashcardStack.shuffle()
	}

	// Init termbox
	err = termbox.Init()
	if err != nil {
		panic(err)
	}
	defer termbox.Close()
	drawAll(flashcardStack)

	//Main Run loop
mainloop:
	for {
		switch event := termbox.PollEvent(); event.Type {
		case termbox.EventKey:
			switch event.Key {
			case termbox.KeyEsc, termbox.KeyCtrlC:
				break mainloop
			case termbox.KeyEnter, termbox.KeyArrowRight:
				flashcardStack.advanceStack()
				drawAll(flashcardStack)
			case termbox.KeyBackspace2, termbox.KeyArrowLeft:
				flashcardStack.revertStack()
				drawAll(flashcardStack)
			}

			switch event.Ch {
			case 'q':
				break mainloop

			case 'l':
				flashcardStack.advanceStack()
				drawAll(flashcardStack)

			case 'h':
				flashcardStack.revertStack()
				drawAll(flashcardStack)

			case 'r':
				if flashcardStack.atEndOfStack() {
					flashcardStack.StackIndex = 0
					flashcardStack.ShowAnswer = false
					drawAll(flashcardStack)
				}

			case 'x':
				if flashcardStack.atEndOfStack() {
					flashcardStack.shuffle()
					flashcardStack.StackIndex = 0
					flashcardStack.ShowAnswer = false
					drawAll(flashcardStack)
				}
			}

		case termbox.EventResize:
			drawAll(flashcardStack)
		}

		drawAll(flashcardStack)
	}
}

func cliInfo(c *cli.Context) {
	if len(c.Args()) != 1 {
		log.Printf("Incorrect usage\n")
		cli.ShowCommandHelp(c, "info")
		os.Exit(1)
	}

	flashcardStack := new(cardStack)
	err := flashcardStack.loadFlashcardStack(c.Args()[0])

	if err != nil {
		log.Fatalf("%s is an invalid file: %s", c.Args()[0], err.Error())
	}

	fmt.Printf("Deck Name: %s\nAuthor: %s\nNumber of Cards: %d\n", flashcardStack.Title, flashcardStack.Author, len(flashcardStack.Flashcards))
}

func drawAll(stack *cardStack) {
	termbox.Clear(termbox.ColorDefault, termbox.ColorDefault)
	w, h := termbox.Size()

	// Draw termbox border
	termbox.SetCell(0, 0, '+', termbox.ColorDefault, termbox.ColorDefault)
	termbox.SetCell(w-1, 0, '+', termbox.ColorDefault, termbox.ColorDefault)
	termbox.SetCell(0, h-1, '+', termbox.ColorDefault, termbox.ColorDefault)
	termbox.SetCell(w-1, h-1, '+', termbox.ColorDefault, termbox.ColorDefault)

	for x := 1; x < w-1; x++ {
		termbox.SetCell(x, 0, '-', termbox.ColorDefault, termbox.ColorDefault)
		termbox.SetCell(x, h-1, '-', termbox.ColorDefault, termbox.ColorDefault)
	}

	// Draw the Stack's title
	titleXCoord := tbutils.CalculateXCenterCoord(stack.Title)
	tbutils.DrawText(titleXCoord, 1, stack.Title)

	// Draw the current card
	currentQuestion := stack.getCurrentFlashcard()
	currentQuestion.drawQuestion()
	if stack.ShowAnswer {
		currentQuestion.drawAnswer()
		// Draw the Q/A divider
		for x := 0; x < w; x++ {
			termbox.SetCell(x, (3 * h / 8), '-', termbox.ColorBlue, termbox.ColorDefault)
		}
	}

	// Draw end of stack message
	if stack.atEndOfStack() {
		endOfStackMessageLine1 := "End of Stack..."
		endOfStackMessageLine2 := "(q)Quit (r)Restart (x)Reshuffle & Restart."
		eosXCoord1, eosYCoord1 := tbutils.CalculateXCenterCoord(endOfStackMessageLine1), (3 * (h / 4))
		eosXCoord2, eosYCoord2 := tbutils.CalculateXCenterCoord(endOfStackMessageLine2), (3*h/4)+1
		tbutils.DrawText(eosXCoord1, eosYCoord1, endOfStackMessageLine1)
		tbutils.DrawText(eosXCoord2, eosYCoord2, endOfStackMessageLine2)
	}

	// Draw the current index
	indexStr := fmt.Sprintf("(%d/%d)", stack.StackIndex+1, len(stack.Flashcards))
	indexXCoord, indexYCoord := tbutils.CalculateXCenterCoord(indexStr), h-1
	tbutils.DrawText(indexXCoord, indexYCoord, indexStr)

	// Write out the back buffer
	termbox.Flush()
}
<commit_msg>Finalised version number<commit_after>package main

import (
	"fmt"
	"github.com/alexjohnj/flasher/tbutils"
	"github.com/codegangsta/cli"
	"github.com/nsf/termbox-go"
	"log"
	"os"
)

func main() {
	app := cli.NewApp()
	app.Name = "flasher"
	app.Version = "0.3.0"
	app.Author = "Alex Jackson"
	app.Email = "alex@alexj.org"

	app.Commands = []cli.Command{
		{
			Name:      "flash",
			ShortName: "f",
			Usage:     "flasher flash [flashcard-file.json]",
			Action:    cliFlash,
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "no-shuffle, n",
					Usage: "Presents flashcards in the order they are written in the source JSON file.",
				},
			},
		},
		{
			Name:      "info",
			ShortName: "i",
			Usage:     "flasher info [flashcard-file.json]",
			Action:    cliInfo,
		},
	}
	app.Run(os.Args)
}

func cliFlash(c *cli.Context) {
	// Load flashcards

	if len(c.Args()) != 1 {
		log.Printf("Incorrect usage\n")
		cli.ShowCommandHelp(c, "flash")
		os.Exit(1)
	}

	flashcardStack := new(cardStack)
	err := flashcardStack.loadFlashcardStack(c.Args()[0])

	if err != nil {
		log.Fatal(err)
	}

	if !c.Bool("no-shuffle") {
		flashcardStack.shuffle()
	}

	// Init termbox
	err = termbox.Init()
	if err != nil {
		panic(err)
	}
	defer termbox.Close()
	drawAll(flashcardStack)

	//Main Run loop
mainloop:
	for {
		switch event := termbox.PollEvent(); event.Type {
		case termbox.EventKey:
			switch event.Key {
			case termbox.KeyEsc, termbox.KeyCtrlC:
				break mainloop
			case termbox.KeyEnter, termbox.KeyArrowRight:
				flashcardStack.advanceStack()
				drawAll(flashcardStack)
			case termbox.KeyBackspace2, termbox.KeyArrowLeft:
				flashcardStack.revertStack()
				drawAll(flashcardStack)
			}

			switch event.Ch {
			case 'q':
				break mainloop

			case 'l':
				flashcardStack.advanceStack()
				drawAll(flashcardStack)

			case 'h':
				flashcardStack.revertStack()
				drawAll(flashcardStack)

			case 'r':
				if flashcardStack.atEndOfStack() {
					flashcardStack.StackIndex = 0
					flashcardStack.ShowAnswer = false
					drawAll(flashcardStack)
				}

			case 'x':
				if flashcardStack.atEndOfStack() {
					flashcardStack.shuffle()
					flashcardStack.StackIndex = 0
					flashcardStack.ShowAnswer = false
					drawAll(flashcardStack)
				}
			}

		case termbox.EventResize:
			drawAll(flashcardStack)
		}

		drawAll(flashcardStack)
	}
}

func cliInfo(c *cli.Context) {
	if len(c.Args()) != 1 {
		log.Printf("Incorrect usage\n")
		cli.ShowCommandHelp(c, "info")
		os.Exit(1)
	}

	flashcardStack := new(cardStack)
	err := flashcardStack.loadFlashcardStack(c.Args()[0])

	if err != nil {
		log.Fatalf("%s is an invalid file: %s", c.Args()[0], err.Error())
	}

	fmt.Printf("Deck Name: %s\nAuthor: %s\nNumber of Cards: %d\n", flashcardStack.Title, flashcardStack.Author, len(flashcardStack.Flashcards))
}

func drawAll(stack *cardStack) {
	termbox.Clear(termbox.ColorDefault, termbox.ColorDefault)
	w, h := termbox.Size()

	// Draw termbox border
	termbox.SetCell(0, 0, '+', termbox.ColorDefault, termbox.ColorDefault)
	termbox.SetCell(w-1, 0, '+', termbox.ColorDefault, termbox.ColorDefault)
	termbox.SetCell(0, h-1, '+', termbox.ColorDefault, termbox.ColorDefault)
	termbox.SetCell(w-1, h-1, '+', termbox.ColorDefault, termbox.ColorDefault)

	for x := 1; x < w-1; x++ {
		termbox.SetCell(x, 0, '-', termbox.ColorDefault, termbox.ColorDefault)
		termbox.SetCell(x, h-1, '-', termbox.ColorDefault, termbox.ColorDefault)
	}

	// Draw the Stack's title
	titleXCoord := tbutils.CalculateXCenterCoord(stack.Title)
	tbutils.DrawText(titleXCoord, 1, stack.Title)

	// Draw the current card
	currentQuestion := stack.getCurrentFlashcard()
	currentQuestion.drawQuestion()
	if stack.ShowAnswer {
		currentQuestion.drawAnswer()
		// Draw the Q/A divider
		for x := 0; x < w; x++ {
			termbox.SetCell(x, (3 * h / 8), '-', termbox.ColorBlue, termbox.ColorDefault)
		}
	}

	// Draw end of stack message
	if stack.atEndOfStack() {
		endOfStackMessageLine1 := "End of Stack..."
		endOfStackMessageLine2 := "(q)Quit (r)Restart (x)Reshuffle & Restart."
		eosXCoord1, eosYCoord1 := tbutils.CalculateXCenterCoord(endOfStackMessageLine1), (3 * (h / 4))
		eosXCoord2, eosYCoord2 := tbutils.CalculateXCenterCoord(endOfStackMessageLine2), (3*h/4)+1
		tbutils.DrawText(eosXCoord1, eosYCoord1, endOfStackMessageLine1)
		tbutils.DrawText(eosXCoord2, eosYCoord2, endOfStackMessageLine2)
	}

	// Draw the current index
	indexStr := fmt.Sprintf("(%d/%d)", stack.StackIndex+1, len(stack.Flashcards))
	indexXCoord, indexYCoord := tbutils.CalculateXCenterCoord(indexStr), h-1
	tbutils.DrawText(indexXCoord, indexYCoord, indexStr)

	// Write out the back buffer
	termbox.Flush()
}
<|endoftext|>


<commit_before>package graval

import (
	"bufio"
	"crypto/rand"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"io"
	"net"
	"path/filepath"
	"strconv"
	"strings"
)

const (
	welcomeMessage = "Welcome to the Go FTP Server"
)

type ftpConn struct {
	conn          *net.TCPConn
	controlReader *bufio.Reader
	controlWriter *bufio.Writer
	dataConn      ftpDataSocket
	driver        FTPDriver
	logger        *ftpLogger
	sessionId     string
	namePrefix    string
	reqUser       string
	user          string
	renameFrom    string
	minDataPort   int
	maxDataPort   int
}

// NewftpConn constructs a new object that will handle the FTP protocol over
// an active net.TCPConn. The TCP connection should already be open before
// it is handed to this functions. driver is an instance of FTPDriver that
// will handle all auth and persistence details.
func newftpConn(tcpConn *net.TCPConn, driver FTPDriver, minPort int, maxPort int) *ftpConn {
	c := new(ftpConn)
	c.namePrefix = "/"
	c.conn = tcpConn
	c.controlReader = bufio.NewReader(tcpConn)
	c.controlWriter = bufio.NewWriter(tcpConn)
	c.driver = driver
	c.sessionId = newSessionId()
	c.logger = newFtpLogger(c.sessionId)
	c.minDataPort = minPort
	c.maxDataPort = maxPort
	return c
}

// returns a random 20 char string that can be used as a unique session ID
func newSessionId() string {
	hash := sha256.New()
	_, err := io.CopyN(hash, rand.Reader, 50)
	if err != nil {
		return "????????????????????"
	}
	md := hash.Sum(nil)
	mdStr := hex.EncodeToString(md)
	return mdStr[0:20]
}

// Serve starts an endless loop that reads FTP commands from the client and
// responds appropriately. terminated is a channel that will receive a true
// message when the connection closes. This loop will be running inside a
// goroutine, so use this channel to be notified when the connection can be
// cleaned up.
func (ftpConn *ftpConn) Serve() {
	ftpConn.logger.Print("Connection Established")
	// send welcome
	ftpConn.writeMessage(220, welcomeMessage)
	// read commands
	for {
		line, err := ftpConn.controlReader.ReadString('\n')
		if err != nil {
			break
		}
		ftpConn.receiveLine(line)
	}
	ftpConn.logger.Print("Connection Terminated")
}

// Close will manually close this connection, even if the client isn't ready.
func (ftpConn *ftpConn) Close() {
	ftpConn.conn.Close()
	if ftpConn.dataConn != nil {
		ftpConn.dataConn.Close()
	}
}

// receiveLine accepts a single line FTP command and co-ordinates an
// appropriate response.
func (ftpConn *ftpConn) receiveLine(line string) {
	command, param := ftpConn.parseLine(line)
	ftpConn.logger.PrintCommand(command, param)
	cmdObj := commands[command]
	if cmdObj == nil {
		ftpConn.writeMessage(500, "Command not found")
		return
	}
	if cmdObj.RequireParam() && param == "" {
		ftpConn.writeMessage(553, "action aborted, required param missing")
	} else if cmdObj.RequireAuth() && ftpConn.user == "" {
		ftpConn.writeMessage(530, "not logged in")
	} else {
		cmdObj.Execute(ftpConn, param)
	}
}

func (ftpConn *ftpConn) parseLine(line string) (string, string) {
	params := strings.SplitN(strings.Trim(line, "\r\n"), " ", 2)
	if len(params) == 1 {
		return params[0], ""
	}
	return params[0], strings.TrimSpace(params[1])
}

// writeMessage will send a standard FTP response back to the client.
func (ftpConn *ftpConn) writeMessage(code int, message string) (wrote int, err error) {
	ftpConn.logger.PrintResponse(code, message)
	line := fmt.Sprintf("%d %s\r\n", code, message)
	wrote, err = ftpConn.controlWriter.WriteString(line)
	ftpConn.controlWriter.Flush()
	return
}

// buildPath takes a client supplied path or filename and generates a safe
// absolute path within their account sandbox.
//
//    buildpath("/")
//    => "/"
//    buildpath("one.txt")
//    => "/one.txt"
//    buildpath("/files/two.txt")
//    => "/files/two.txt"
//    buildpath("files/two.txt")
//    => "files/two.txt"
//    buildpath("/../../../../etc/passwd")
//    => "/etc/passwd"
//
// The driver implementation is responsible for deciding how to treat this path.
// Obviously they MUST NOT just read the path off disk. The probably want to
// prefix the path with something to scope the users access to a sandbox.
func (ftpConn *ftpConn) buildPath(filename string) (fullPath string) {
	if len(filename) > 0 && filename[0:1] == "/" {
		fullPath = filepath.Clean(filename)
	} else if len(filename) > 0 && filename != "-a" {
		fullPath = filepath.Clean(ftpConn.namePrefix + "/" + filename)
	} else {
		fullPath = filepath.Clean(ftpConn.namePrefix)
	}
	fullPath = strings.Replace(fullPath, "//", "/", -1)
	return
}

// sendOutofbandData will send a string to the client via the currently open
// data socket. Assumes the socket is open and ready to be used.
func (ftpConn *ftpConn) sendOutofbandData(data string) {
	bytes := len(data)
	ftpConn.dataConn.Write([]byte(data))
	ftpConn.dataConn.Close()
	message := "Closing data connection, sent " + strconv.Itoa(bytes) + " bytes"
	ftpConn.writeMessage(226, message)
}
<commit_msg>Recover panic per FTP connection<commit_after>package graval

import (
	"bufio"
	"crypto/rand"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"io"
	"net"
	"path/filepath"
	"strconv"
	"strings"
)

const (
	welcomeMessage = "Welcome to the Go FTP Server"
)

type ftpConn struct {
	conn          *net.TCPConn
	controlReader *bufio.Reader
	controlWriter *bufio.Writer
	dataConn      ftpDataSocket
	driver        FTPDriver
	logger        *ftpLogger
	sessionId     string
	namePrefix    string
	reqUser       string
	user          string
	renameFrom    string
	minDataPort   int
	maxDataPort   int
}

// NewftpConn constructs a new object that will handle the FTP protocol over
// an active net.TCPConn. The TCP connection should already be open before
// it is handed to this functions. driver is an instance of FTPDriver that
// will handle all auth and persistence details.
func newftpConn(tcpConn *net.TCPConn, driver FTPDriver, minPort int, maxPort int) *ftpConn {
	c := new(ftpConn)
	c.namePrefix = "/"
	c.conn = tcpConn
	c.controlReader = bufio.NewReader(tcpConn)
	c.controlWriter = bufio.NewWriter(tcpConn)
	c.driver = driver
	c.sessionId = newSessionId()
	c.logger = newFtpLogger(c.sessionId)
	c.minDataPort = minPort
	c.maxDataPort = maxPort
	return c
}

// returns a random 20 char string that can be used as a unique session ID
func newSessionId() string {
	hash := sha256.New()
	_, err := io.CopyN(hash, rand.Reader, 50)
	if err != nil {
		return "????????????????????"
	}
	md := hash.Sum(nil)
	mdStr := hex.EncodeToString(md)
	return mdStr[0:20]
}

// Serve starts an endless loop that reads FTP commands from the client and
// responds appropriately. terminated is a channel that will receive a true
// message when the connection closes. This loop will be running inside a
// goroutine, so use this channel to be notified when the connection can be
// cleaned up.
func (ftpConn *ftpConn) Serve() {
	defer func() {
		if r := recover(); r != nil {
			ftpConn.logger.Printf("Recovered in ftpConn Serve: %s", r)
		}

		ftpConn.Close()
	}()

	ftpConn.logger.Print("Connection Established")
	// send welcome
	ftpConn.writeMessage(220, welcomeMessage)
	// read commands
	for {
		line, err := ftpConn.controlReader.ReadString('\n')
		if err != nil {
			break
		}
		ftpConn.receiveLine(line)
	}
	ftpConn.logger.Print("Connection Terminated")
}

// Close will manually close this connection, even if the client isn't ready.
func (ftpConn *ftpConn) Close() {
	ftpConn.conn.Close()
	if ftpConn.dataConn != nil {
		ftpConn.dataConn.Close()
	}
}

// receiveLine accepts a single line FTP command and co-ordinates an
// appropriate response.
func (ftpConn *ftpConn) receiveLine(line string) {
	command, param := ftpConn.parseLine(line)
	ftpConn.logger.PrintCommand(command, param)
	cmdObj := commands[command]
	if cmdObj == nil {
		ftpConn.writeMessage(500, "Command not found")
		return
	}
	if cmdObj.RequireParam() && param == "" {
		ftpConn.writeMessage(553, "action aborted, required param missing")
	} else if cmdObj.RequireAuth() && ftpConn.user == "" {
		ftpConn.writeMessage(530, "not logged in")
	} else {
		cmdObj.Execute(ftpConn, param)
	}
}

func (ftpConn *ftpConn) parseLine(line string) (string, string) {
	params := strings.SplitN(strings.Trim(line, "\r\n"), " ", 2)
	if len(params) == 1 {
		return params[0], ""
	}
	return params[0], strings.TrimSpace(params[1])
}

// writeMessage will send a standard FTP response back to the client.
func (ftpConn *ftpConn) writeMessage(code int, message string) (wrote int, err error) {
	ftpConn.logger.PrintResponse(code, message)
	line := fmt.Sprintf("%d %s\r\n", code, message)
	wrote, err = ftpConn.controlWriter.WriteString(line)
	ftpConn.controlWriter.Flush()
	return
}

// buildPath takes a client supplied path or filename and generates a safe
// absolute path within their account sandbox.
//
//    buildpath("/")
//    => "/"
//    buildpath("one.txt")
//    => "/one.txt"
//    buildpath("/files/two.txt")
//    => "/files/two.txt"
//    buildpath("files/two.txt")
//    => "files/two.txt"
//    buildpath("/../../../../etc/passwd")
//    => "/etc/passwd"
//
// The driver implementation is responsible for deciding how to treat this path.
// Obviously they MUST NOT just read the path off disk. The probably want to
// prefix the path with something to scope the users access to a sandbox.
func (ftpConn *ftpConn) buildPath(filename string) (fullPath string) {
	if len(filename) > 0 && filename[0:1] == "/" {
		fullPath = filepath.Clean(filename)
	} else if len(filename) > 0 && filename != "-a" {
		fullPath = filepath.Clean(ftpConn.namePrefix + "/" + filename)
	} else {
		fullPath = filepath.Clean(ftpConn.namePrefix)
	}
	fullPath = strings.Replace(fullPath, "//", "/", -1)
	return
}

// sendOutofbandData will send a string to the client via the currently open
// data socket. Assumes the socket is open and ready to be used.
func (ftpConn *ftpConn) sendOutofbandData(data string) {
	bytes := len(data)
	ftpConn.dataConn.Write([]byte(data))
	ftpConn.dataConn.Close()
	message := "Closing data connection, sent " + strconv.Itoa(bytes) + " bytes"
	ftpConn.writeMessage(226, message)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"

	"code.cloudfoundry.org/auction/simulation/simulationrep"
	"code.cloudfoundry.org/cflager"
	executorfakes "code.cloudfoundry.org/executor/fakes"
	"code.cloudfoundry.org/rep"
	"code.cloudfoundry.org/rep/evacuation/evacuation_context/fake_evacuation_context"
	rephandlers "code.cloudfoundry.org/rep/handlers"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/http_server"
	"github.com/tedsuo/ifrit/sigmon"
	"github.com/tedsuo/rata"
)

var memoryMB = flag.Int("memoryMB", 100, "total available memory in MB")
var diskMB = flag.Int("diskMB", 100, "total available disk in MB")
var containers = flag.Int("containers", 100, "total available containers")
var repGuid = flag.String("repGuid", "", "rep-guid")
var httpAddr = flag.String("httpAddr", "", "http server addres")
var stack = flag.String("stack", "", "stack")
var zone = flag.String("zone", "Z0", "availability zone")

func main() {
	cflager.AddFlags(flag.CommandLine)
	flag.Parse()

	if *repGuid == "" {
		panic("need rep-guid")
	}

	if *httpAddr == "" {
		panic("need http addr")
	}

	simulationRep := simulationrep.New(*stack, *zone, rep.Resources{
		MemoryMB:   int32(*memoryMB),
		DiskMB:     int32(*diskMB),
		Containers: *containers,
	}, []string{})

	logger, _ := cflager.New("repnode-http")

	fakeExecutorClient := new(executorfakes.FakeClient)
	fakeEvacuatable := new(fake_evacuation_context.FakeEvacuatable)

	handlers := rephandlers.New(simulationRep, fakeExecutorClient, fakeEvacuatable, logger.Session(*repGuid))
	router, err := rata.NewRouter(rep.Routes, handlers)
	if err != nil {
		log.Fatalln("failed to make router:", err)
	}
	httpServer := http_server.New(*httpAddr, router)

	monitor := ifrit.Invoke(sigmon.New(httpServer))
	fmt.Println("rep node listening")
	err = <-monitor.Wait()
	if err != nil {
		println("EXITED WITH ERROR: ", err.Error())
	}
}
<commit_msg>cco/cflager -> cco/lager/lagerflags<commit_after>package main

import (
	"flag"
	"fmt"
	"log"

	"code.cloudfoundry.org/auction/simulation/simulationrep"
	executorfakes "code.cloudfoundry.org/executor/fakes"
	"code.cloudfoundry.org/lager/lagerflags"
	"code.cloudfoundry.org/rep"
	"code.cloudfoundry.org/rep/evacuation/evacuation_context/fake_evacuation_context"
	rephandlers "code.cloudfoundry.org/rep/handlers"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/http_server"
	"github.com/tedsuo/ifrit/sigmon"
	"github.com/tedsuo/rata"
)

var memoryMB = flag.Int("memoryMB", 100, "total available memory in MB")
var diskMB = flag.Int("diskMB", 100, "total available disk in MB")
var containers = flag.Int("containers", 100, "total available containers")
var repGuid = flag.String("repGuid", "", "rep-guid")
var httpAddr = flag.String("httpAddr", "", "http server addres")
var stack = flag.String("stack", "", "stack")
var zone = flag.String("zone", "Z0", "availability zone")

func main() {
	lagerflags.AddFlags(flag.CommandLine)
	flag.Parse()

	if *repGuid == "" {
		panic("need rep-guid")
	}

	if *httpAddr == "" {
		panic("need http addr")
	}

	simulationRep := simulationrep.New(*stack, *zone, rep.Resources{
		MemoryMB:   int32(*memoryMB),
		DiskMB:     int32(*diskMB),
		Containers: *containers,
	}, []string{})

	logger, _ := lagerflags.New("repnode-http")

	fakeExecutorClient := new(executorfakes.FakeClient)
	fakeEvacuatable := new(fake_evacuation_context.FakeEvacuatable)

	handlers := rephandlers.New(simulationRep, fakeExecutorClient, fakeEvacuatable, logger.Session(*repGuid))
	router, err := rata.NewRouter(rep.Routes, handlers)
	if err != nil {
		log.Fatalln("failed to make router:", err)
	}
	httpServer := http_server.New(*httpAddr, router)

	monitor := ifrit.Invoke(sigmon.New(httpServer))
	fmt.Println("rep node listening")
	err = <-monitor.Wait()
	if err != nil {
		println("EXITED WITH ERROR: ", err.Error())
	}
}
<|endoftext|>


<commit_before>package carDirectory

import (
	"fmt"
	"os"
	"strconv"
	"strings"

	"github.com/bullettrain-sh/bullettrain-go-core/ansi"
)

const carPaint = "white:blue"

// Directory car
type Car struct {
	paint string
	// Current directory
	Pwd string
}

// GetPaint returns the calculated end paint string for the car.
func (c *Car) GetPaint() string {
	if c.paint = os.Getenv("BULLETTRAIN_CAR_DIRECTORY_PAINT"); c.paint == "" {
		c.paint = carPaint
	}

	return c.paint
}

// CanShow decides if this car needs to be displayed.
func (c *Car) CanShow() bool {
	s := true
	if e := os.Getenv("BULLETTRAIN_CAR_DIRECTORY_SHOW"); e == "false" {
		s = false
	}

	return s
}

// Render builds and passes the end product of a completely composed car onto
// the channel.
func (c *Car) Render(out chan<- string) {
	defer close(out)

	dir := c.Pwd

	if os.Getenv("HOME") == dir {
		dir = "~"
	} else {
		ps := string(os.PathSeparator)

		// Calculate max directory elements to display.
		max_length := 3
		if e := os.Getenv("BULLETTRAIN_CAR_DIRECTORY_MAX_LENGHT"); e != "" {
			ml, err := strconv.Atoi(e)
			if err == nil {
				max_length = ml
			}
		}

		// Allow to override the default three dots by some other string.
		depth_indicator := "..."
		di, di_defined := os.LookupEnv("BULLETTRAIN_CAR_DIRECTORY_DEPTH_INDICATOR")
		if di_defined {
			depth_indicator = di
		}

		// Compose directory segments.
		dirs := strings.Split(dir, ps)
		if max_length > 0 && len(dirs) > max_length+1 {
			f := len(dirs) - max_length
			p := dirs[f:]
			dir = fmt.Sprintf("%s%s", depth_indicator, strings.Join(p, ps))
		}

		if s := os.Getenv("BULLETTRAIN_CAR_DIRECTORY_PATH_SEPARATOR"); s != "" {
			dir = strings.Replace(dir, ps, s, -1)
		}
	}

	out <- ansi.Color(fmt.Sprintf("%s", dir), c.GetPaint())
}

// GetSeparatorPaint overrides the Fg/Bg colours of the right hand side
// separator through ENV variables.
func (c *Car) GetSeparatorPaint() string {
	return os.Getenv("BULLETTRAIN_CAR_DIRECTORY_SEPARATOR_PAINT")
}

// GetSeparatorSymbol overrides the symbol of the right hand side
// separator through ENV variables.
func (c *Car) GetSeparatorSymbol() string {
	return os.Getenv("BULLETTRAIN_CAR_DIRECTORY_SEPARATOR_SYMBOL")
}
<commit_msg>Show "~" when it's the parent.<commit_after>package carDirectory

import (
	"fmt"
	"os"
	"strconv"
	"strings"

	"github.com/bullettrain-sh/bullettrain-go-core/ansi"
)

const carPaint = "white:blue"

// Directory car
type Car struct {
	paint string
	// Current directory
	Pwd string
}

// GetPaint returns the calculated end paint string for the car.
func (c *Car) GetPaint() string {
	if c.paint = os.Getenv("BULLETTRAIN_CAR_DIRECTORY_PAINT"); c.paint == "" {
		c.paint = carPaint
	}

	return c.paint
}

// CanShow decides if this car needs to be displayed.
func (c *Car) CanShow() bool {
	s := true
	if e := os.Getenv("BULLETTRAIN_CAR_DIRECTORY_SHOW"); e == "false" {
		s = false
	}

	return s
}

// Render builds and passes the end product of a completely composed car onto
// the channel.
func (c *Car) Render(out chan<- string) {
	defer close(out)

	dir := c.Pwd

	if strings.HasPrefix(dir, os.Getenv("HOME")) {
		dir = strings.Replace(dir, os.Getenv("HOME"), "~", 1)
	}

	ps := string(os.PathSeparator)

	// Calculate max directory elements to display.
	max_length := 3
	if e := os.Getenv("BULLETTRAIN_CAR_DIRECTORY_MAX_LENGHT"); e != "" {
		ml, err := strconv.Atoi(e)
		if err == nil {
			max_length = ml
		}
	}

	// Allow to override the default three dots by some other string.
	depth_indicator := "..."
	di, di_defined := os.LookupEnv("BULLETTRAIN_CAR_DIRECTORY_DEPTH_INDICATOR")
	if di_defined {
		depth_indicator = di
	}

	// Compose directory segments.
	dirs := strings.Split(dir, ps)
	if max_length > 0 && len(dirs) > max_length+1 {
		f := len(dirs) - max_length
		p := dirs[f:]
		dir = fmt.Sprintf("%s%s", depth_indicator, strings.Join(p, ps))
	}

	if s := os.Getenv("BULLETTRAIN_CAR_DIRECTORY_PATH_SEPARATOR"); s != "" {
		dir = strings.Replace(dir, ps, s, -1)
	}

	out <- ansi.Color(fmt.Sprintf("%s", dir), c.GetPaint())
}

// GetSeparatorPaint overrides the Fg/Bg colours of the right hand side
// separator through ENV variables.
func (c *Car) GetSeparatorPaint() string {
	return os.Getenv("BULLETTRAIN_CAR_DIRECTORY_SEPARATOR_PAINT")
}

// GetSeparatorSymbol overrides the symbol of the right hand side
// separator through ENV variables.
func (c *Car) GetSeparatorSymbol() string {
	return os.Getenv("BULLETTRAIN_CAR_DIRECTORY_SEPARATOR_SYMBOL")
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"syscall"

	"github.com/lomik/carbon-clickhouse/carbon"
	"github.com/lomik/carbon-clickhouse/helper/RowBinary"
	"github.com/lomik/zapwriter"
	"go.uber.org/zap"

	_ "net/http/pprof"
)

// Version of carbon-clickhouse
const Version = "0.7.0-alpha"

func httpServe(addr string) (func(), error) {
	tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
	if err != nil {
		return nil, err
	}

	listener, err := net.ListenTCP("tcp", tcpAddr)
	if err != nil {
		return nil, err
	}

	go http.Serve(listener, nil)
	return func() { listener.Close() }, nil
}

func main() {
	var err error

	/* CONFIG start */

	configFile := flag.String("config", "/etc/carbon-clickhouse/carbon-clickhouse.conf", "Filename of config")
	printDefaultConfig := flag.Bool("config-print-default", false, "Print default config")
	checkConfig := flag.Bool("check-config", false, "Check config and exit")
	printVersion := flag.Bool("version", false, "Print version")
	cat := flag.String("cat", "", "Print RowBinary file in TabSeparated format")
	bincat := flag.String("recover", "", "Read all good records from corrupted data file. Write binary data to stdout")

	flag.Parse()

	if *printVersion {
		fmt.Print(Version)
		return
	}

	if *cat != "" {
		reader, err := RowBinary.NewReader(*cat)
		if err != nil {
			log.Fatal(err)
		}

		for {
			metric, err := reader.ReadRecord()
			if err != nil {
				if err == io.EOF {
					return
				}
				log.Fatal(err)
			}

			fmt.Printf("%s\t%#v\t%d\t%s\t%d\n",
				string(metric),
				reader.Value(),
				reader.Timestamp(),
				reader.DaysString(),
				reader.Version(),
			)
		}
	}

	if *bincat != "" {
		reader, err := RowBinary.NewReader(*bincat)
		if err != nil {
			log.Fatal(err)
		}

		io.Copy(os.Stdout, reader)
		return
	}

	if *printDefaultConfig {
		if err = carbon.PrintDefaultConfig(); err != nil {
			log.Fatal(err)
		}
		return
	}

	app := carbon.New(*configFile)

	if err = app.ParseConfig(); err != nil {
		log.Fatal(err)
	}

	// config parsed successfully. Exit in check-only mode
	if *checkConfig {
		return
	}

	cfg := app.Config

	if err = zapwriter.ApplyConfig(cfg.Logging); err != nil {
		log.Fatal(err)
	}

	mainLogger := zapwriter.Logger("main")

	/* CONFIG end */

	// pprof
	if cfg.Pprof.Enabled {
		_, err = httpServe(cfg.Pprof.Listen)
		if err != nil {
			mainLogger.Fatal("pprof listen failed", zap.Error(err))
		}
	}

	if err = app.Start(); err != nil {
		mainLogger.Fatal("app start failed", zap.Error(err))
	} else {
		mainLogger.Info("app started")
	}

	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, syscall.SIGUSR1, syscall.SIGUSR2, syscall.SIGHUP)

		for {
			s := <-c
			switch s {
			case syscall.SIGUSR1:
				mainLogger.Info("SIGUSR1 received. Clear tree cache")
				app.Reset()
			case syscall.SIGUSR2:
				mainLogger.Info("SIGUSR2 received. Ignoring")
			case syscall.SIGHUP:
				mainLogger.Info("SIGHUP received. Ignoring")
			}
		}
	}()

	app.Loop()

	mainLogger.Info("app stopped")
}
<commit_msg>0.7.0-alpha2<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"syscall"

	"github.com/lomik/carbon-clickhouse/carbon"
	"github.com/lomik/carbon-clickhouse/helper/RowBinary"
	"github.com/lomik/zapwriter"
	"go.uber.org/zap"

	_ "net/http/pprof"
)

// Version of carbon-clickhouse
const Version = "0.7.0-alpha2"

func httpServe(addr string) (func(), error) {
	tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
	if err != nil {
		return nil, err
	}

	listener, err := net.ListenTCP("tcp", tcpAddr)
	if err != nil {
		return nil, err
	}

	go http.Serve(listener, nil)
	return func() { listener.Close() }, nil
}

func main() {
	var err error

	/* CONFIG start */

	configFile := flag.String("config", "/etc/carbon-clickhouse/carbon-clickhouse.conf", "Filename of config")
	printDefaultConfig := flag.Bool("config-print-default", false, "Print default config")
	checkConfig := flag.Bool("check-config", false, "Check config and exit")
	printVersion := flag.Bool("version", false, "Print version")
	cat := flag.String("cat", "", "Print RowBinary file in TabSeparated format")
	bincat := flag.String("recover", "", "Read all good records from corrupted data file. Write binary data to stdout")

	flag.Parse()

	if *printVersion {
		fmt.Print(Version)
		return
	}

	if *cat != "" {
		reader, err := RowBinary.NewReader(*cat)
		if err != nil {
			log.Fatal(err)
		}

		for {
			metric, err := reader.ReadRecord()
			if err != nil {
				if err == io.EOF {
					return
				}
				log.Fatal(err)
			}

			fmt.Printf("%s\t%#v\t%d\t%s\t%d\n",
				string(metric),
				reader.Value(),
				reader.Timestamp(),
				reader.DaysString(),
				reader.Version(),
			)
		}
	}

	if *bincat != "" {
		reader, err := RowBinary.NewReader(*bincat)
		if err != nil {
			log.Fatal(err)
		}

		io.Copy(os.Stdout, reader)
		return
	}

	if *printDefaultConfig {
		if err = carbon.PrintDefaultConfig(); err != nil {
			log.Fatal(err)
		}
		return
	}

	app := carbon.New(*configFile)

	if err = app.ParseConfig(); err != nil {
		log.Fatal(err)
	}

	// config parsed successfully. Exit in check-only mode
	if *checkConfig {
		return
	}

	cfg := app.Config

	if err = zapwriter.ApplyConfig(cfg.Logging); err != nil {
		log.Fatal(err)
	}

	mainLogger := zapwriter.Logger("main")

	/* CONFIG end */

	// pprof
	if cfg.Pprof.Enabled {
		_, err = httpServe(cfg.Pprof.Listen)
		if err != nil {
			mainLogger.Fatal("pprof listen failed", zap.Error(err))
		}
	}

	if err = app.Start(); err != nil {
		mainLogger.Fatal("app start failed", zap.Error(err))
	} else {
		mainLogger.Info("app started")
	}

	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, syscall.SIGUSR1, syscall.SIGUSR2, syscall.SIGHUP)

		for {
			s := <-c
			switch s {
			case syscall.SIGUSR1:
				mainLogger.Info("SIGUSR1 received. Clear tree cache")
				app.Reset()
			case syscall.SIGUSR2:
				mainLogger.Info("SIGUSR2 received. Ignoring")
			case syscall.SIGHUP:
				mainLogger.Info("SIGHUP received. Ignoring")
			}
		}
	}()

	app.Loop()

	mainLogger.Info("app stopped")
}
<|endoftext|>


<commit_before>package eveapi

// CCP basic XML Frame
type xmlAPIFrame struct {
	Version     int        `xml:"eveapi>version"`
	CurrentTime EVEXMLTime `xml:"currentTime"`
	CachedUntil EVEXMLTime `xml:"cachedUntil"`
}
<commit_msg>Add CallList and APIKeyInfo xml endpoints<commit_after>package eveapi

import (
	"fmt"

	"golang.org/x/oauth2"
)

// CCP basic XML Frame
type xmlAPIFrame struct {
	Version     int        `xml:"eveapi>version"`
	CurrentTime EVEXMLTime `xml:"currentTime"`
	CachedUntil EVEXMLTime `xml:"cachedUntil"`
}

type EVEAPICallListXML struct {
	xmlAPIFrame
	Entries []struct {
		Kind string `xml:"name,attr"`
		Rows []struct {
			Name        string `xml:"name,attr"`
			Description string `xml:"description,attr"`
			GroupID     int32  `xml:"groupID,attr"`
			AccessMask  int64  `xml:"accessMask,attr"`
			Type        string `xml:"type,attr"`
		} `xml:"row"`
	} `xml:"result>rowset"`
}

func (c *EVEAPIClient) CallList(auth oauth2.TokenSource) (*EVEAPICallListXML, error) {
	w := &EVEAPICallListXML{}
	tok, err := auth.Token()
	if err != nil {
		return nil, err
	}
	url := c.base.XML + fmt.Sprintf("api/CallList.xml.aspx?accessToken=%s", tok.AccessToken)
	_, err = c.doXML("GET", url, nil, w)
	if err != nil {
		return nil, err
	}
	return w, nil
}

type EVEAPIKeyInfoXML struct {
	xmlAPIFrame
	Key struct {
		AccessMask int64      `xml:"accessMask,attr"`
		Type       string     `xml:"type,attr"`
		Expires    EVEXMLTime `xml:"expires,attr"`
		Info       struct {
			CharacterID     int64  `xml:"characterID,attr"`
			CharacterName   string `xml:"characterName,attr"`
			CorporationID   int64  `xml:"corporationID,attr"`
			CorporationName string `xml:"corporationName,attr"`
			AllianceID      int64  `xml:"allianceID,attr"`
			AllianceName    string `xml:"allianceName,attr"`
			FactionID       int64  `xml:"factionID,attr"`
			FactionName     string `xml:"factionName,attr"`
		} `xml:"rowset>row"`
	} `xml:"result>key"`
}

func (c *EVEAPIClient) APIKeyInfo(auth oauth2.TokenSource) (*EVEAPIKeyInfoXML, error) {
	w := &EVEAPIKeyInfoXML{}
	tok, err := auth.Token()
	if err != nil {
		return nil, err
	}
	url := c.base.XML + fmt.Sprintf("account/APIKeyInfo.xml.aspx?accessToken=%s&accessType=corporation", tok.AccessToken)
	_, err = c.doXML("GET", url, nil, w)
	if err != nil {
		return nil, err
	}
	return w, nil
}
<|endoftext|>


<commit_before>package haproxyconfigparser

import (
	"testing"

	"fmt"

	"github.com/stretchr/testify/assert"
)

func TestRegisterEvent(t *testing.T) {
	backend := 0
	RegisterEvent("backend", "*", func(eventType EventType, line string, parser Parser) {
		backend++
	})

	frontend := 0
	RegisterEvent("frontend", "*", func(eventType EventType, line string, parser Parser) {
		frontend++
	})

	global := 0
	RegisterEvent("global", "*", func(eventType EventType, line string, parser Parser) {
		global++
	})

	_, err := ParseFromFile("testdata/haproxy.cfg")
	if err != nil {
		t.Errorf("Failed to parse data: %s", err)
	}
	assert.Equal(t, 6, global, "Expected different number of events for global.")
	assert.Equal(t, 16, backend, "Expected different number of events for backend.")
	assert.Equal(t, 21, frontend, "Expected different number of events for frontend.")
}

func ExampleRegisterEvent() {
	RegisterEvent("backend", "*", func(eventType EventType, line string, parser Parser) {
		if eventType == START_SECTION {
			fmt.Printf("EventType=%s, line=%s\n", eventType, line)
		}
	})

	_, err := ParseFromFile("testdata/haproxy.cfg")
	if err != nil {
		fmt.Errorf("Failed to parse data: %s", err)
	}
	//Output:
	//EventType=start_section, line=backend profileEditingService_20000
	//EventType=start_section, line=backend accountCreationService_10000
}
<commit_msg>Fixed a warning level issue in the RegisterEvent example<commit_after>package haproxyconfigparser

import (
	"testing"

	"fmt"

	"github.com/stretchr/testify/assert"
)

func TestRegisterEvent(t *testing.T) {
	backend := 0
	RegisterEvent("backend", "*", func(eventType EventType, line string, parser Parser) {
		backend++
	})

	frontend := 0
	RegisterEvent("frontend", "*", func(eventType EventType, line string, parser Parser) {
		frontend++
	})

	global := 0
	RegisterEvent("global", "*", func(eventType EventType, line string, parser Parser) {
		global++
	})

	_, err := ParseFromFile("testdata/haproxy.cfg")
	if err != nil {
		t.Errorf("Failed to parse data: %s", err)
	}
	assert.Equal(t, 6, global, "Expected different number of events for global.")
	assert.Equal(t, 16, backend, "Expected different number of events for backend.")
	assert.Equal(t, 21, frontend, "Expected different number of events for frontend.")
}

func ExampleRegisterEvent() {
	RegisterEvent("backend", "*", func(eventType EventType, line string, parser Parser) {
		if eventType == START_SECTION {
			fmt.Printf("EventType=%s, line=%s\n", eventType, line)
		}
	})

	_, err := ParseFromFile("testdata/haproxy.cfg")
	if err != nil {
		fmt.Printf("Failed to parse data: %s", err)
	}
	//Output:
	//EventType=start_section, line=backend profileEditingService_20000
	//EventType=start_section, line=backend accountCreationService_10000
}
<|endoftext|>


<commit_before>package promhttputil

import (
	"fmt"
	"reflect"

	"github.com/prometheus/common/model"
)

func ValueAddLabelSet(a model.Value, l model.LabelSet) error {
	switch aTyped := a.(type) {
	case model.Vector:
		for _, item := range aTyped {
			for k, v := range l {
				item.Metric[k] = v
			}
		}

	case model.Matrix:
		for _, item := range aTyped {
			for k, v := range l {
				item.Metric[k] = v
			}
		}
	}

	return nil

}

// TODO: always make copies? Now we sometimes return one, or make a copy, or do nothing
// Merge 2 values and
func MergeValues(a, b model.Value) (model.Value, error) {
	if a.Type() != b.Type() {
		return nil, fmt.Errorf("Error!")
	}

	switch aTyped := a.(type) {
	// TODO: more logic? for now we assume both are correct if they exist
	// In the case where it is a single datapoint, we're going to assume that
	// either is valid, we just need one
	case *model.Scalar:
		bTyped := b.(*model.Scalar)

		if aTyped.Value != 0 && aTyped.Timestamp != 0 {
			return aTyped, nil
		} else {
			return bTyped, nil
		}

	// In the case where it is a single datapoint, we're going to assume that
	// either is valid, we just need one
	case *model.String:
		bTyped := b.(*model.String)

		if aTyped.Value != "" && aTyped.Timestamp != 0 {
			return aTyped, nil
		} else {
			return bTyped, nil
		}

	// List of *model.Sample -- only 1 value (guaranteed same timestamp)
	case model.Vector:
		bTyped := b.(model.Vector)

		newValue := make(model.Vector, 0, len(aTyped)+len(bTyped))
		fingerPrintMap := make(map[model.Fingerprint]int)

		addItem := func(item *model.Sample) {
			finger := item.Metric.Fingerprint()

			// If we've seen this fingerPrint before, lets make sure that a value exists
			if index, ok := fingerPrintMap[finger]; ok {
				// TODO: better? For now we only replace if we have no value (which seems reasonable)
				if newValue[index].Value == model.SampleValue(0) {
					newValue[index].Value = item.Value
				}
			} else {
				newValue = append(newValue, item)
				fingerPrintMap[finger] = len(newValue) - 1
			}
		}

		for _, item := range aTyped {
			addItem(item)
		}

		for _, item := range bTyped {
			addItem(item)
		}
		return newValue, nil

	case model.Matrix:
		bTyped := b.(model.Matrix)

		newValue := make(model.Matrix, 0, len(aTyped)+len(bTyped))
		fingerPrintMap := make(map[model.Fingerprint]int)

		addStream := func(stream *model.SampleStream) {
			finger := stream.Metric.Fingerprint()

			// If we've seen this fingerPrint before, lets make sure that a value exists
			if index, ok := fingerPrintMap[finger]; ok {
				// TODO: check this error? For now the only one is sig collision, which we check
				newValue[index], _ = MergeSampleStream(newValue[index], stream)
			} else {
				newValue = append(newValue, stream)
				fingerPrintMap[finger] = len(newValue) - 1
			}
		}

		for _, item := range aTyped {
			addStream(item)
		}

		for _, item := range bTyped {
			addStream(item)
		}
		return newValue, nil
	}

	return nil, fmt.Errorf("Unknown type! %v", reflect.TypeOf(a))
}

func MergeSampleStream(a, b *model.SampleStream) (*model.SampleStream, error) {
	if a.Metric.Fingerprint() != b.Metric.Fingerprint() {
		return nil, fmt.Errorf("Cannot merge mismatch fingerprints")
	}

	// TODO: really there should be a library method for this in prometheus IMO
	// At this point we have 2 sorted lists of datapoints which we need to merge
	seenTimes := make(map[model.Time]struct{})
	newValues := make([]model.SamplePair, 0, len(a.Values)+len(b.Values))

	ai := 0 // Offset in a
	bi := 0 // Offset in b

	for {
		if ai >= len(a.Values) && bi >= len(b.Values) {
			break
		}

		var item model.SamplePair

		if ai < len(a.Values) { // If a exists
			if bi < len(b.Values) {
				// both items
				if a.Values[ai].Timestamp < b.Values[bi].Timestamp {
					item = a.Values[ai]
					ai++
				} else {
					item = b.Values[bi]
					bi++
				}
			} else {
				// Only A
				item = a.Values[ai]
				ai++
			}
		} else {
			if bi < len(b.Values) {
				// Only B
				item = b.Values[bi]
				bi++
			}
		}
		// If we've already seen this timestamp, skip
		if _, ok := seenTimes[item.Timestamp]; ok {
			continue
		}

		// Otherwise, lets add it
		newValues = append(newValues, item)
		seenTimes[item.Timestamp] = struct{}{}
	}

	return &model.SampleStream{
		Metric: a.Metric,
		Values: newValues,
	}, nil
}
<commit_msg>Add hacky anti-affinity when merging matrix values<commit_after>package promhttputil

import (
	"fmt"
	"reflect"

	"github.com/prometheus/common/model"
)

func ValueAddLabelSet(a model.Value, l model.LabelSet) error {
	switch aTyped := a.(type) {
	case model.Vector:
		for _, item := range aTyped {
			for k, v := range l {
				item.Metric[k] = v
			}
		}

	case model.Matrix:
		for _, item := range aTyped {
			for k, v := range l {
				item.Metric[k] = v
			}
		}
	}

	return nil

}

// TODO: always make copies? Now we sometimes return one, or make a copy, or do nothing
// Merge 2 values and
func MergeValues(a, b model.Value) (model.Value, error) {
	if a.Type() != b.Type() {
		return nil, fmt.Errorf("Error!")
	}

	switch aTyped := a.(type) {
	// TODO: more logic? for now we assume both are correct if they exist
	// In the case where it is a single datapoint, we're going to assume that
	// either is valid, we just need one
	case *model.Scalar:
		bTyped := b.(*model.Scalar)

		if aTyped.Value != 0 && aTyped.Timestamp != 0 {
			return aTyped, nil
		} else {
			return bTyped, nil
		}

	// In the case where it is a single datapoint, we're going to assume that
	// either is valid, we just need one
	case *model.String:
		bTyped := b.(*model.String)

		if aTyped.Value != "" && aTyped.Timestamp != 0 {
			return aTyped, nil
		} else {
			return bTyped, nil
		}

	// List of *model.Sample -- only 1 value (guaranteed same timestamp)
	case model.Vector:
		bTyped := b.(model.Vector)

		newValue := make(model.Vector, 0, len(aTyped)+len(bTyped))
		fingerPrintMap := make(map[model.Fingerprint]int)

		addItem := func(item *model.Sample) {
			finger := item.Metric.Fingerprint()

			// If we've seen this fingerPrint before, lets make sure that a value exists
			if index, ok := fingerPrintMap[finger]; ok {
				// TODO: better? For now we only replace if we have no value (which seems reasonable)
				if newValue[index].Value == model.SampleValue(0) {
					newValue[index].Value = item.Value
				}
			} else {
				newValue = append(newValue, item)
				fingerPrintMap[finger] = len(newValue) - 1
			}
		}

		for _, item := range aTyped {
			addItem(item)
		}

		for _, item := range bTyped {
			addItem(item)
		}
		return newValue, nil

	case model.Matrix:
		bTyped := b.(model.Matrix)

		newValue := make(model.Matrix, 0, len(aTyped)+len(bTyped))
		fingerPrintMap := make(map[model.Fingerprint]int)

		addStream := func(stream *model.SampleStream) {
			finger := stream.Metric.Fingerprint()

			// If we've seen this fingerPrint before, lets make sure that a value exists
			if index, ok := fingerPrintMap[finger]; ok {
				// TODO: check this error? For now the only one is sig collision, which we check
				newValue[index], _ = MergeSampleStream(newValue[index], stream)
			} else {
				newValue = append(newValue, stream)
				fingerPrintMap[finger] = len(newValue) - 1
			}
		}

		for _, item := range aTyped {
			addStream(item)
		}

		for _, item := range bTyped {
			addStream(item)
		}
		return newValue, nil
	}

	return nil, fmt.Errorf("Unknown type! %v", reflect.TypeOf(a))
}

func MergeSampleStream(a, b *model.SampleStream) (*model.SampleStream, error) {
	if a.Metric.Fingerprint() != b.Metric.Fingerprint() {
		return nil, fmt.Errorf("Cannot merge mismatch fingerprints")
	}

	// TODO: really there should be a library method for this in prometheus IMO
	// At this point we have 2 sorted lists of datapoints which we need to merge
	seenTimes := make(map[model.Time]struct{})
	newValues := make([]model.SamplePair, 0, len(a.Values)+len(b.Values))

	ai := 0 // Offset in a
	bi := 0 // Offset in b

	// When combining series from 2 different prometheus hosts we can run into some problems
	// with clock skew (from a variety of sources). The primary one I've run into is issues
	// with the time that prometheus stores. Since the time associated with the datapoint is
	// the *start* time of the scrape, there can be quite a lot of time (which can vary
	// dramatically between hosts) for the exporter to return. In an attempt to mitigate
	// this problem we're going to *not* merge any datapoint within 5s of another point
	// we have. This means we can tolerate 2.5s on either side (which can be used by either
	// clock skew or from this scrape skew).

	// TODO: config
	antiAffinityBuffer := model.TimeFromUnix(5) // 5s
	var lastTime model.Time

	for {
		if ai >= len(a.Values) && bi >= len(b.Values) {
			break
		}

		var item model.SamplePair

		if ai < len(a.Values) { // If a exists
			if bi < len(b.Values) {
				// both items
				if a.Values[ai].Timestamp < b.Values[bi].Timestamp {
					item = a.Values[ai]
					ai++
				} else {
					item = b.Values[bi]
					bi++
				}
			} else {
				// Only A
				item = a.Values[ai]
				ai++
			}
		} else {
			if bi < len(b.Values) {
				// Only B
				item = b.Values[bi]
				bi++
			}
		}
		// If we've already seen this timestamp, skip
		if _, ok := seenTimes[item.Timestamp]; ok {
			continue
		}

		if lastTime == 0 {
			lastTime = item.Timestamp
		}

		if item.Timestamp-lastTime < antiAffinityBuffer {
			continue
		}
		lastTime = item.Timestamp

		// Otherwise, lets add it
		newValues = append(newValues, item)
		seenTimes[item.Timestamp] = struct{}{}
	}

	return &model.SampleStream{
		Metric: a.Metric,
		Values: newValues,
	}, nil
}
<|endoftext|>


<commit_before>package ics

import (
	"strings"
	"time"
)

type completed struct {
	dateTime
}

func (p *parser) readCompletedProperty() (property, error) {
	v, err := p.readValue()
	if err != nil {
		return nil, err
	}
	var c completed
	c.Time, err = time.ParseInLocation("20060102T150405Z", v, time.UTC)
	if err != nil {
		return nil, err
	}
	return c, nil
}

func (c completed) Validate() bool {
	return c.Location() == time.UTC
}

func (c completed) Data() propertyData {
	return propertyData{
		Name:  completedp,
		Value: c.String(),
	}
}

type dateTimeEnd struct {
	dateTime
}

func (p *parser) readDateTimeOrTime() (t dateTime, err error) {
	as, err := p.readAttributes(tzidparam, valuetypeparam)
	if err != nil {
		return t, err
	}
	var (
		l        *time.Location
		justDate bool
	)
	if tzid, ok := as[tzidparam]; ok {
		l, err = time.LoadLocation(tzid.String())
		if err != nil {
			return t, err
		}
	}
	if v, ok := as[valuetypeparam]; ok {
		val := v.(value)
		switch val {
		case valueDate:
			justDate = true
		case valueDateTime:
			justDate = false
		default:
			return t, ErrUnsupportedValue
		}
	}
	v, err := p.readValue()
	if err != nil {
		return t, err
	}
	if justDate {
		t, err = parseDate(v)
	} else {
		t, err = parseDateTime(v, l)
	}
	return t, err
}

func dateTimeOrTimeData(name string, d dateTime) propertyData {
	params := make(map[string]attribute)
	if d.justDate {
		params[valuetypeparam] = valueDate
	}
	if d.Location() != time.UTC {
		params[tzidparam] = d.Location().String()
	}
	return propertyData{
		Name:   name,
		Params: params,
		Value:  d.String(),
	}
}

func (p *parser) readDateTimeEndProperty() (property, error) {
	t, err := p.readDateTimeOrTime()
	if err != nil {
		return nil, err
	}
	return dateTimeEnd{t}, nil
}

func (d dateTimeEnd) Validate() bool {
	return true
}

func (d dateTimeEnd) Date() propertyData {
	return dateTimeOrTimeData(dtendp, d.dateTime)
}

type dateTimeDue struct {
	dateTime dateTime
}

func (p *parser) readDateTimeDueProperty() (property, error) {
	t, err := p.readDateTimeOrTime()
	if err != nil {
		return nil, err
	}
	return dateTimeDue{t}, nil
}

func (d dateTimeDue) Validate() bool {
	return true
}

func (d dateTimeDue) Data() propertyData {
	return dateTimeOrTimeData(duep, d.dateTime)
}

type dateTimeStart struct {
	dateTime dateTime
}

func (p *parser) readDateTimeStartProperty() (property, error) {
	t, err := p.readDateTimeOrTime()
	if err != nil {
		return nil, err
	}
	return dateTimeStart{t}, nil
}

func (d dateTimeStart) Validate() bool {
	return true
}

func (d dateTimeStart) Data() propertyData {
	return dateTimeOrTimeData(dtstartp, d.dateTime)
}

type duration struct {
	time.Duration
}

func (p *parser) readDurationProperty() (property, error) {
	v, err := p.readValue()
	if err != nil {
		return nil, err
	}
	var d duration
	d.Duration, err = parseDuration(v)
	if err != nil {
		return nil, err
	}
	return d, nil
}

func (d duration) Validate() bool {
	return true
}

func (d duration) Data() propertyData {
	return propertyData{
		Name:  durationp,
		Value: durationString(d.Duration),
	}
}

type freeBusyTime struct {
	Typ     freeBusy
	Periods []period
}

type period struct {
	FixedDuration bool
	Start, End    dateTime
}

func parsePeriods(v string, l *time.Location) ([]period, error) {
	periods := make([]period, 0, 1)

	for _, pd := range textSplit(v, ',') {
		parts := strings.Split(pd, "/")
		if len(parts) != 2 {
			return nil, ErrUnsupportedValue
		}
		if parts[0][len(parts[0])-1] != 'Z' {
			return nil, ErrUnsupportedValue
		}
		start, err := parseDateTime(parts[0], l)
		if err != nil {
			return nil, err
		}
		var (
			end           dateTime
			fixedDuration bool
		)
		if parts[1][len(parts[1])-1] == 'Z' {
			end, err = parseDateTime(parts[1], l)
			if err != nil {
				return nil, err
			}
		} else {
			d, err := parseDuration(parts[1])
			if err != nil {
				return nil, err
			}
			if d < 0 {
				return nil, ErrUnsupportedValue
			}
			end = start.Add(d)
			fixedDuration = true
		}
		periods = append(periods, period{fixedDuration, start, end})
	}
	return periods, nil
}

func (p *parser) readFreeBusyTimeProperty() (property, error) {
	as, err := p.readAttributes(fbtypeparam)
	if err != nil {
		return nil, err
	}
	var fb freeBusy
	if f, ok := as[fbtypeparam]; ok {
		fb = f.(freeBusy)
	}
	v, err := p.readValue()
	if err != nil {
		return nil, err
	}
	periods, err := parsePeriods(v, nil)
	if err != nil {
		return nil, err
	}
	return freeBusyTime{
		Typ:     fb,
		Periods: periods,
	}, nil
}

func (f freeBusyTime) Validate() bool {
	return f.Typ >= fbBusy && f.Typ <= fbBusyTentative
}

func (f freeBusyTime) Data() propertyData {
	params := make(map[string]attribute)
	params[fbtypeparam] = f.Typ
	var val []byte
	for _, period := range f.Periods {
		val = append(val, ',')
		val = append(val, period.Start.String()...)
		val = append(val, '/')
		if period.FixedDuration {
			val = append(val, durationString(period.End.Sub(period.Start.Time))...)
		} else {
			val = append(val, period.End.String()...)
		}
	}
	return propertyData{
		Name:   freebusyp,
		Params: params,
		Value:  string(val),
	}
}

const (
	TTOpaque timeTransparency = iota
	TTTransparent
)

type timeTransparency int

func (p *parser) readTimeTransparencyProperty() (property, error) {
	v, err := p.readValue()
	if err != nil {
		return nil, err
	}
	switch v {
	case "OPAQUE":
		return TTOpaque, nil
	case "TRANSPARENT":
		return TTTransparent, nil
	default:
		return nil, ErrUnsupportedValue
	}
}

func (t timeTransparency) String() string {
	switch t {
	case TTOpaque:
		return "OPAQUE"
	case TTTransparent:
		return "TRANSPARENT"
	default:
		return "UNKNOWN"
	}
}

func (t timeTransparency) Validate() bool {
	switch t {
	case TTOpaque, TTTransparent:
		return true
	default:
		return false
	}
}

func (t timeTransparency) Data() propertyData {
	return propertyData{
		Name:  transpp,
		Value: t.String(),
	}
}
<commit_msg>moved period byte'ing into separate method<commit_after>package ics

import (
	"strings"
	"time"
)

type completed struct {
	dateTime
}

func (p *parser) readCompletedProperty() (property, error) {
	v, err := p.readValue()
	if err != nil {
		return nil, err
	}
	var c completed
	c.Time, err = time.ParseInLocation("20060102T150405Z", v, time.UTC)
	if err != nil {
		return nil, err
	}
	return c, nil
}

func (c completed) Validate() bool {
	return c.Location() == time.UTC
}

func (c completed) Data() propertyData {
	return propertyData{
		Name:  completedp,
		Value: c.String(),
	}
}

type dateTimeEnd struct {
	dateTime
}

func (p *parser) readDateTimeOrTime() (t dateTime, err error) {
	as, err := p.readAttributes(tzidparam, valuetypeparam)
	if err != nil {
		return t, err
	}
	var (
		l        *time.Location
		justDate bool
	)
	if tzid, ok := as[tzidparam]; ok {
		l, err = time.LoadLocation(tzid.String())
		if err != nil {
			return t, err
		}
	}
	if v, ok := as[valuetypeparam]; ok {
		val := v.(value)
		switch val {
		case valueDate:
			justDate = true
		case valueDateTime:
			justDate = false
		default:
			return t, ErrUnsupportedValue
		}
	}
	v, err := p.readValue()
	if err != nil {
		return t, err
	}
	if justDate {
		t, err = parseDate(v)
	} else {
		t, err = parseDateTime(v, l)
	}
	return t, err
}

func dateTimeOrTimeData(name string, d dateTime) propertyData {
	params := make(map[string]attribute)
	if d.justDate {
		params[valuetypeparam] = valueDate
	}
	if d.Location() != time.UTC {
		params[tzidparam] = d.Location().String()
	}
	return propertyData{
		Name:   name,
		Params: params,
		Value:  d.String(),
	}
}

func (p *parser) readDateTimeEndProperty() (property, error) {
	t, err := p.readDateTimeOrTime()
	if err != nil {
		return nil, err
	}
	return dateTimeEnd{t}, nil
}

func (d dateTimeEnd) Validate() bool {
	return true
}

func (d dateTimeEnd) Date() propertyData {
	return dateTimeOrTimeData(dtendp, d.dateTime)
}

type dateTimeDue struct {
	dateTime dateTime
}

func (p *parser) readDateTimeDueProperty() (property, error) {
	t, err := p.readDateTimeOrTime()
	if err != nil {
		return nil, err
	}
	return dateTimeDue{t}, nil
}

func (d dateTimeDue) Validate() bool {
	return true
}

func (d dateTimeDue) Data() propertyData {
	return dateTimeOrTimeData(duep, d.dateTime)
}

type dateTimeStart struct {
	dateTime dateTime
}

func (p *parser) readDateTimeStartProperty() (property, error) {
	t, err := p.readDateTimeOrTime()
	if err != nil {
		return nil, err
	}
	return dateTimeStart{t}, nil
}

func (d dateTimeStart) Validate() bool {
	return true
}

func (d dateTimeStart) Data() propertyData {
	return dateTimeOrTimeData(dtstartp, d.dateTime)
}

type duration struct {
	time.Duration
}

func (p *parser) readDurationProperty() (property, error) {
	v, err := p.readValue()
	if err != nil {
		return nil, err
	}
	var d duration
	d.Duration, err = parseDuration(v)
	if err != nil {
		return nil, err
	}
	return d, nil
}

func (d duration) Validate() bool {
	return true
}

func (d duration) Data() propertyData {
	return propertyData{
		Name:  durationp,
		Value: durationString(d.Duration),
	}
}

type freeBusyTime struct {
	Typ     freeBusy
	Periods []period
}

type period struct {
	FixedDuration bool
	Start, End    dateTime
}

func (p period) Bytes() []byte {
	val := make([]byte, 0, 64)
	val = append(val, p.Start.String()...)
	val = append(val, '/')
	if p.FixedDuration {
		val = append(val, durationString(p.End.Sub(p.Start.Time))...)
	} else {
		val = append(val, p.End.String()...)
	}
	return val
}

func parsePeriods(v string, l *time.Location) ([]period, error) {
	periods := make([]period, 0, 1)

	for _, pd := range textSplit(v, ',') {
		parts := strings.Split(pd, "/")
		if len(parts) != 2 {
			return nil, ErrUnsupportedValue
		}
		if parts[0][len(parts[0])-1] != 'Z' {
			return nil, ErrUnsupportedValue
		}
		start, err := parseDateTime(parts[0], l)
		if err != nil {
			return nil, err
		}
		var (
			end           dateTime
			fixedDuration bool
		)
		if parts[1][len(parts[1])-1] == 'Z' {
			end, err = parseDateTime(parts[1], l)
			if err != nil {
				return nil, err
			}
		} else {
			d, err := parseDuration(parts[1])
			if err != nil {
				return nil, err
			}
			if d < 0 {
				return nil, ErrUnsupportedValue
			}
			end = start.Add(d)
			fixedDuration = true
		}
		periods = append(periods, period{fixedDuration, start, end})
	}
	return periods, nil
}

func (p *parser) readFreeBusyTimeProperty() (property, error) {
	as, err := p.readAttributes(fbtypeparam)
	if err != nil {
		return nil, err
	}
	var fb freeBusy
	if f, ok := as[fbtypeparam]; ok {
		fb = f.(freeBusy)
	}
	v, err := p.readValue()
	if err != nil {
		return nil, err
	}
	periods, err := parsePeriods(v, nil)
	if err != nil {
		return nil, err
	}
	return freeBusyTime{
		Typ:     fb,
		Periods: periods,
	}, nil
}

func (f freeBusyTime) Validate() bool {
	return f.Typ >= fbBusy && f.Typ <= fbBusyTentative
}

func (f freeBusyTime) Data() propertyData {
	params := make(map[string]attribute)
	params[fbtypeparam] = f.Typ
	val := make([]byte, 0, len(f.Periods)*64)
	for _, period := range f.Periods {
		val = append(val, ',')
		val = append(val, period.Bytes()...)
	}
	return propertyData{
		Name:   freebusyp,
		Params: params,
		Value:  string(val),
	}
}

const (
	TTOpaque timeTransparency = iota
	TTTransparent
)

type timeTransparency int

func (p *parser) readTimeTransparencyProperty() (property, error) {
	v, err := p.readValue()
	if err != nil {
		return nil, err
	}
	switch v {
	case "OPAQUE":
		return TTOpaque, nil
	case "TRANSPARENT":
		return TTTransparent, nil
	default:
		return nil, ErrUnsupportedValue
	}
}

func (t timeTransparency) String() string {
	switch t {
	case TTOpaque:
		return "OPAQUE"
	case TTTransparent:
		return "TRANSPARENT"
	default:
		return "UNKNOWN"
	}
}

func (t timeTransparency) Validate() bool {
	switch t {
	case TTOpaque, TTTransparent:
		return true
	default:
		return false
	}
}

func (t timeTransparency) Data() propertyData {
	return propertyData{
		Name:  transpp,
		Value: t.String(),
	}
}
<|endoftext|>


<commit_before>package command

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/dnaeon/gru/graph"
	"github.com/dnaeon/gru/module"
	"github.com/dnaeon/gru/resource"
)

// NewGraphCommand creates a new sub-command for
// generating the resource DAG graph
func NewGraphCommand() cli.Command {
	cmd := cli.Command{
		Name:   "graph",
		Usage:  "create DOT representation for modules and resources",
		Action: execGraphCommand,
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:   "siterepo",
				Value:  "",
				Usage:  "path/url to the site repo",
				EnvVar: "GRU_SITEREPO",
			},
		},
	}

	return cmd
}

// Executes the "graph" command
func execGraphCommand(c *cli.Context) {
	if len(c.Args()) < 1 {
		displayError(errNoModuleName, 64)
	}

	main := c.Args()[0]
	config := &module.Config{
		Path:           filepath.Join(c.String("siterepo"), "modules"),
		ResourceConfig: &resource.Config{},
	}

	// Discover and load all modules
	discovered, err := module.DiscoverAndLoad(config)
	if err != nil {
		displayError(err, 1)
	}

	// Generate DOT representation of the module imports
	modulesGraph, err := module.ImportGraph(main, config.Path)
	if err != nil {
		displayError(err, 1)
	}
	modulesGraph.AsDot(fmt.Sprintf("%s_imports", main), os.Stdout)

	// Attempt a topological sort of the module imports graph.
	// In case of circular dependencies in the graph
	// generate a DOT for the remaining nodes, which would
	// give us the nodes causing the circular dependencies
	modulesSorted, err := modulesGraph.Sort()
	if err == graph.ErrCircularDependency {
		circular := graph.New()
		circular.AddNode(modulesSorted...)
		circular.AsDot(fmt.Sprintf("%s_imports_circular", main), os.Stdout)
		displayError(graph.ErrCircularDependency, 1)
	}

	// Get the sorted modules and create a DOT for their resources
	var modules []*module.Module
	for _, node := range modulesSorted {
		modules = append(modules, discovered[node.Name])
	}

	collection, err := module.ResourceCollection(modules)
	if err != nil {
		displayError(err, 1)
	}

	collectionGraph, err := collection.DependencyGraph()
	if err != nil {
		displayError(err, 1)
	}

	// Create resource nodes in their respective module
	fmt.Fprintf(os.Stdout, "digraph resources {\n")
	fmt.Fprintf(os.Stdout, "\tlabel = \"resources\";\n")
	fmt.Fprintf(os.Stdout, "\tnodesep=1.0;\n")
	fmt.Fprintf(os.Stdout, "\tnode [shape=box];\n")
	fmt.Fprintf(os.Stdout, "\tedge [style=filled];\n")
	for _, m := range modules {
		fmt.Fprintf(os.Stdout, fmt.Sprintf("\tsubgraph cluster_%s {\n", m.Name))
		fmt.Fprintf(os.Stdout, fmt.Sprintf("\t\tlabel = %q;\n", m.Name))
		fmt.Fprintf(os.Stdout, "\t\tcolor = black;\n")
		for _, r := range m.Resources {
			fmt.Fprintf(os.Stdout, fmt.Sprintf("\t\t%q;\n", r.ResourceID()))
		}
		fmt.Fprintf(os.Stdout, "\t}\n")
	}

	// Finally, connect the nodes
	for _, node := range collectionGraph.Nodes {
		if len(node.Edges) == 0 {
			continue
		}

		var edges []string
		for _, edge := range node.Edges {
			edges = append(edges, fmt.Sprintf("%q", edge.Name))
		}

		fmt.Fprintf(os.Stdout, fmt.Sprintf("\t%q -> {%s};\n", node.Name, strings.Join(edges, " ")))
	}
	fmt.Fprintf(os.Stdout, "}\n")

	// Attempt a topological sort of the resource dependencies.
	// In case of circular dependencies in the graph
	// generate a DOT for the remaining nodes, which would
	// give us the resources causing the circular dependencies
	collectionSorted, err := collectionGraph.Sort()
	if err == graph.ErrCircularDependency {
		circular := graph.New()
		circular.AddNode(collectionSorted...)
		circular.AsDot("resources_circular", os.Stdout)
		displayError(graph.ErrCircularDependency, 1)
	}
}
<commit_msg>gructl: update 'graph' command after recent refactoring<commit_after>package command

import (
	"os"
	"path/filepath"

	"github.com/codegangsta/cli"
	"github.com/dnaeon/gru/graph"
	"github.com/dnaeon/gru/resource"
	"github.com/layeh/gopher-luar"
	"github.com/yuin/gopher-lua"
)

// NewGraphCommand creates a new sub-command for
// generating the resource DAG graph
func NewGraphCommand() cli.Command {
	cmd := cli.Command{
		Name:   "graph",
		Usage:  "create DOT representation of resources",
		Action: execGraphCommand,
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:   "siterepo",
				Value:  "",
				Usage:  "path/url to the site repo",
				EnvVar: "GRU_SITEREPO",
			},
		},
	}

	return cmd
}

// Executes the "graph" command
func execGraphCommand(c *cli.Context) {
	if len(c.Args()) < 1 {
		displayError(errNoModuleName, 64)
	}

	module := filepath.Join(c.String("siterepo"), c.Args()[0])

	// A fake catalog used to load resources from Lua
	var sorted []resource.Resource

	L := lua.NewState()
	defer L.Close()

	resource.LuaRegisterBuiltin(L)
	L.SetGlobal("catalog", luar.New(L, unsorted))
	if err := L.DoFile(module); err != nil {
		displayError(err, 1)
	}

	collection, err := resource.CreateCollection(unsorted)
	if err != nil {
		displayError(err, 1)
	}

	collectionGraph, err := collection.DependencyGraph()
	if err != nil {
		displayError(err, 1)
	}
	collectionGraph.AsDot("resources", os.Stdout)

	collectionSorted, err := collectionGraph.Sort()
	if err == graph.ErrCircularDependency {
		circular := graph.New()
		circular.AddNode(collectionSorted...)
		circular.AsDot("resources_circular", os.Stdout)
		displayError(graph.ErrCircularDependency, 1)
	}
}
<|endoftext|>