<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"time"

	"database/sql"

	"github.com/Jeffail/gabs"
	"github.com/lib/pq"

	"github.com/Shopify/sarama"
	"github.com/urfave/cli"
)

const (
	reportInterval = 30 * time.Second
)

var consumerTblName = pq.QuoteIdentifier("kafka_consumer")

func main() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)
	myApp := cli.NewApp()
	myApp.Name = "kafka2psql"
	myApp.Usage = `Store Kafka Topic To PostgreSQL Table`
	myApp.Version = "0.1"
	myApp.Flags = []cli.Flag{
		cli.StringSliceFlag{
			Name:  "brokers, b",
			Value: &cli.StringSlice{"localhost:9092"},
			Usage: "kafka brokers address",
		},
		cli.StringFlag{
			Name:  "topic, t",
			Value: "commitlog",
			Usage: "topic name for consuming commit log",
		},
		cli.StringFlag{
			Name:  "pq",
			Value: "postgres://127.0.0.1:5432/pipeline?sslmode=disable",
			Usage: "psql url",
		},
		cli.StringFlag{
			Name:  "tblname",
			Value: "log_20060102",
			Usage: "psql table name, aware of timeformat in golang",
		},
		cli.StringFlag{
			Name:  "consumer",
			Value: "log",
			Usage: "consumer name to differs offsets in psql table:" + consumerTblName,
		},
		cli.StringFlag{
			Name:  "primarykey,PK",
			Value: "",
			Usage: "primary key path in json, if empty, message key will treated as key, format: https://github.com/Jeffail/gabs",
		},
		cli.BoolFlag{
			Name:  "appendonly",
			Usage: "append message only, will omit --primarykey, and use offset as the primary key",
		},
	}
	myApp.Action = processor
	myApp.Run(os.Args)
}

func processor(c *cli.Context) error {
	db, err := sql.Open("postgres", c.String("pq"))
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()
	log.Println("brokers:", c.StringSlice("brokers"))
	log.Println("topic:", c.String("topic"))
	log.Println("pq:", c.String("pq"))
	log.Println("tblname:", c.String("tblname"))
	log.Println("consumer:", c.String("consumer"))
	log.Println("primarykey:", c.String("primarykey"))
	log.Println("appendonly:", c.String("appendonly"))

	consumer, err := sarama.NewConsumer(c.StringSlice("brokers"), nil)
	if err != nil {
		log.Fatalln(err)
	}

	defer func() {
		if err := consumer.Close(); err != nil {
			log.Fatalln(err)
		}
	}()

	// table creation
	db.Exec(fmt.Sprintf("CREATE TABLE %s (id TEXT PRIMARY KEY, value BIGINT)", consumerTblName))
	lastTblName := pq.QuoteIdentifier(time.Now().Format(c.String("tblname")))
	db.Exec("CREATE TABLE " + lastTblName + "(id TEXT PRIMARY KEY, data JSON)")

	// read offset
	offset := sarama.OffsetOldest
	err = db.QueryRow("SELECT value FROM kafka_consumer WHERE id = $1 LIMIT 1", c.String("consumer")).Scan(&offset)
	if err != nil {
		log.Println(err)
	}
	log.Println("consuming from offset:", offset)

	partitionConsumer, err := consumer.ConsumePartition(c.String("topic"), 0, offset)
	if err != nil {
		log.Fatalln(err)
	}

	defer func() {
		if err := partitionConsumer.Close(); err != nil {
			log.Fatalln(err)
		}
	}()

	log.Println("started")
	var count int64
	primKey := c.String("primarykey")
	appendOnly := c.Bool("appendonly")
	reportTicker := time.NewTicker(reportInterval)

	for {
		select {
		case msg := <-partitionConsumer.Messages():
			// create new table if necessary
			tblName := pq.QuoteIdentifier(time.Now().Format(c.String("tblname")))
			if tblName != lastTblName {
				// CREATE TABLE
				db.Exec("CREATE TABLE " + tblName + "(id TEXT PRIMARY KEY, data JSON)")
				lastTblName = tblName
			}

			commit(tblName, primKey, appendOnly, db, msg, c)
			count++
		case <-reportTicker.C:
			log.Println("written:", count)
			count = 0
		}
	}
}

func commit(tblname, primkey string, appendonly bool, db *sql.DB, msg *sarama.ConsumerMessage, c *cli.Context) {
	// compute key
	var key string
	if appendonly {
		key = fmt.Sprint(msg.Offset)
	} else if primkey != "" {
		if jsonParsed, err := gabs.ParseJSON(msg.Value); err == nil {
			key = fmt.Sprint(jsonParsed.Path(primkey).Data())
		} else {
			log.Println(err)
			return
		}
	} else {
		key = string(msg.Key)
	}

	if r, err := db.Exec(fmt.Sprintf("INSERT INTO %s (id, data) VALUES ($1,$2) ON CONFLICT(id) DO UPDATE SET data = EXCLUDED.data",
		tblname), key, string(msg.Value)); err == nil {
		// write offset
		if r, err := db.Exec(fmt.Sprintf("INSERT INTO %s (id, value) VALUES ($1,$2) ON CONFLICT(id) DO UPDATE SET value=EXCLUDED.value",
			consumerTblName), c.String("consumer"), msg.Offset); err != nil {
			log.Println(r, err)
		}
	} else {
		log.Println(r, err)
	}
}
<commit_msg>json -> jsonb<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"time"

	"database/sql"

	"github.com/Jeffail/gabs"
	"github.com/lib/pq"

	"github.com/Shopify/sarama"
	"github.com/urfave/cli"
)

const (
	reportInterval = 30 * time.Second
)

var consumerTblName = pq.QuoteIdentifier("kafka_consumer")

func main() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)
	myApp := cli.NewApp()
	myApp.Name = "kafka2psql"
	myApp.Usage = `Store Kafka Topic To PostgreSQL Table`
	myApp.Version = "0.1"
	myApp.Flags = []cli.Flag{
		cli.StringSliceFlag{
			Name:  "brokers, b",
			Value: &cli.StringSlice{"localhost:9092"},
			Usage: "kafka brokers address",
		},
		cli.StringFlag{
			Name:  "topic, t",
			Value: "commitlog",
			Usage: "topic name for consuming commit log",
		},
		cli.StringFlag{
			Name:  "pq",
			Value: "postgres://127.0.0.1:5432/pipeline?sslmode=disable",
			Usage: "psql url",
		},
		cli.StringFlag{
			Name:  "tblname",
			Value: "log_20060102",
			Usage: "psql table name, aware of timeformat in golang",
		},
		cli.StringFlag{
			Name:  "consumer",
			Value: "log",
			Usage: "consumer name to differs offsets in psql table:" + consumerTblName,
		},
		cli.StringFlag{
			Name:  "primarykey,PK",
			Value: "",
			Usage: "primary key path in json, if empty, message key will treated as key, format: https://github.com/Jeffail/gabs",
		},
		cli.BoolFlag{
			Name:  "appendonly",
			Usage: "append message only, will omit --primarykey, and use offset as the primary key",
		},
	}
	myApp.Action = processor
	myApp.Run(os.Args)
}

func processor(c *cli.Context) error {
	db, err := sql.Open("postgres", c.String("pq"))
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()
	log.Println("brokers:", c.StringSlice("brokers"))
	log.Println("topic:", c.String("topic"))
	log.Println("pq:", c.String("pq"))
	log.Println("tblname:", c.String("tblname"))
	log.Println("consumer:", c.String("consumer"))
	log.Println("primarykey:", c.String("primarykey"))
	log.Println("appendonly:", c.String("appendonly"))

	consumer, err := sarama.NewConsumer(c.StringSlice("brokers"), nil)
	if err != nil {
		log.Fatalln(err)
	}

	defer func() {
		if err := consumer.Close(); err != nil {
			log.Fatalln(err)
		}
	}()

	// table creation
	db.Exec(fmt.Sprintf("CREATE TABLE %s (id TEXT PRIMARY KEY, value BIGINT)", consumerTblName))
	lastTblName := pq.QuoteIdentifier(time.Now().Format(c.String("tblname")))
	db.Exec("CREATE TABLE " + lastTblName + "(id TEXT PRIMARY KEY, data JSONB)")

	// read offset
	offset := sarama.OffsetOldest
	err = db.QueryRow("SELECT value FROM kafka_consumer WHERE id = $1 LIMIT 1", c.String("consumer")).Scan(&offset)
	if err != nil {
		log.Println(err)
	}
	log.Println("consuming from offset:", offset)

	partitionConsumer, err := consumer.ConsumePartition(c.String("topic"), 0, offset)
	if err != nil {
		log.Fatalln(err)
	}

	defer func() {
		if err := partitionConsumer.Close(); err != nil {
			log.Fatalln(err)
		}
	}()

	log.Println("started")
	var count int64
	primKey := c.String("primarykey")
	appendOnly := c.Bool("appendonly")
	reportTicker := time.NewTicker(reportInterval)

	for {
		select {
		case msg := <-partitionConsumer.Messages():
			// create new table if necessary
			tblName := pq.QuoteIdentifier(time.Now().Format(c.String("tblname")))
			if tblName != lastTblName {
				// CREATE TABLE
				db.Exec("CREATE TABLE " + tblName + "(id TEXT PRIMARY KEY, data JSONB)")
				lastTblName = tblName
			}

			commit(tblName, primKey, appendOnly, db, msg, c)
			count++
		case <-reportTicker.C:
			log.Println("written:", count)
			count = 0
		}
	}
}

func commit(tblname, primkey string, appendonly bool, db *sql.DB, msg *sarama.ConsumerMessage, c *cli.Context) {
	// compute key
	var key string
	if appendonly {
		key = fmt.Sprint(msg.Offset)
	} else if primkey != "" {
		if jsonParsed, err := gabs.ParseJSON(msg.Value); err == nil {
			key = fmt.Sprint(jsonParsed.Path(primkey).Data())
		} else {
			log.Println(err)
			return
		}
	} else {
		key = string(msg.Key)
	}

	if r, err := db.Exec(fmt.Sprintf("INSERT INTO %s (id, data) VALUES ($1,$2) ON CONFLICT(id) DO UPDATE SET data = EXCLUDED.data",
		tblname), key, string(msg.Value)); err == nil {
		// write offset
		if r, err := db.Exec(fmt.Sprintf("INSERT INTO %s (id, value) VALUES ($1,$2) ON CONFLICT(id) DO UPDATE SET value=EXCLUDED.value",
			consumerTblName), c.String("consumer"), msg.Offset); err != nil {
			log.Println(r, err)
		}
	} else {
		log.Println(r, err)
	}
}
<|endoftext|>


<commit_before>package clingon

import (
	"⚛sdl"
	"⚛sdl/ttf"
	"unsafe"
)

const (
	MAX_INTERNAL_SIZE_FACTOR     = 3
)

const (
	SCROLL_UP = iota
	SCROLL_DOWN
	SCROLL_UP_ANIMATION
	SCROLL_DOWN_ANIMATION
)

type SDLRenderer struct {
	// Activate/deactivate blended text rendering. By default use
	// solid text rendering
	Blended bool
	// Set the foreground color of the font (white by default)
	Color sdl.Color
	// Set the font family
	Font *ttf.Font

	// Map of built-in animations
	Animations map[int]*Animation

	internalSurface           *sdl.Surface
	visibleSurface            *sdl.Surface
	cursorOn                  bool
	eventCh                   chan interface{}
	scrollCh                  chan int
	updatedRectsCh            chan []sdl.Rect
	pauseCh                   chan bool
	updatedRects              []sdl.Rect
	viewportY                 int16
	commandLineRect           *sdl.Rect
	cursorY                   int16
	lastVisibleLine           int
	internalSurfaceMaxHeight  uint16
	width, height             uint16 // visible surface width, height
	cursorWidth, cursorHeight uint16 // cursor width, height
	fontWidth, fontHeight     int    // font width, height
	paused                    bool
}

func NewSDLRenderer(surface *sdl.Surface, font *ttf.Font) *SDLRenderer {
	rect := new(sdl.Rect)
	surface.GetClipRect(rect)

	renderer := &SDLRenderer{
		Color:          sdl.Color{255, 255, 255, 0},
		Animations:     make(map[int]*Animation),
		visibleSurface: surface,
		Font:           font,
		eventCh:        make(chan interface{}),
		scrollCh:       make(chan int),
		pauseCh:        make(chan bool),
		updatedRectsCh: make(chan []sdl.Rect),
		updatedRects:   make([]sdl.Rect, 0),
		width:          rect.W,
		height:         rect.H,
	}

	fontWidth, fontHeight, _ := font.SizeText("A")

	renderer.fontWidth = fontWidth
	renderer.fontHeight = fontHeight

	renderer.cursorWidth = uint16(fontWidth)
	renderer.cursorHeight = uint16(fontHeight)

	renderer.lastVisibleLine = int(float(renderer.height)/float(renderer.fontHeight)) * MAX_INTERNAL_SIZE_FACTOR
	renderer.internalSurfaceMaxHeight = renderer.height * MAX_INTERNAL_SIZE_FACTOR

	renderer.internalSurface = sdl.CreateRGBSurface(sdl.SWSURFACE, int(renderer.width), int(renderer.fontHeight), 32, 0, 0, 0, 0)
	renderer.calcCommandLineRect()

	renderer.Animations[SCROLL_UP_ANIMATION] = NewSlideUpAnimation(1e9, 10.0)
	renderer.Animations[SCROLL_DOWN_ANIMATION] = NewSlideUpAnimation(1e9, 10.0)

	renderer.updatedRects = append(renderer.updatedRects, sdl.Rect{0, 0, renderer.width, renderer.height})

	go renderer.loop()

	return renderer
}

// Receive the events triggered by the console.
func (renderer *SDLRenderer) EventCh() chan<- interface{} {
	return renderer.eventCh
}

// Tell the renderer to scroll up/down the console.
func (renderer *SDLRenderer) ScrollCh() chan<- int {
	return renderer.scrollCh
}

// Tell the renderer to pause its operations.
func (renderer *SDLRenderer) PauseCh() chan<- bool {
	return renderer.pauseCh
}

// From this channel, the client receives the rects representing the
// updated surface regions.
func (renderer *SDLRenderer) UpdatedRectsCh() <-chan []sdl.Rect {
	return renderer.updatedRectsCh
}

// Return the visible SDL surface
func (renderer *SDLRenderer) GetSurface() *sdl.Surface {
	return renderer.visibleSurface
}

func (renderer *SDLRenderer) calcCommandLineRect() {
	renderer.commandLineRect = &sdl.Rect{
		0,
		int16(renderer.internalSurface.H) - int16(renderer.fontHeight),
		uint16(renderer.internalSurface.W),
		uint16(renderer.fontHeight),
	}
}

func (renderer *SDLRenderer) resizeInternalSurface(console *Console) {
	if renderer.internalSurface != nil {
		renderer.internalSurface.Free()
	}

	h := uint16((console.lines.Len() + 1) * renderer.fontHeight)

	if h > renderer.internalSurfaceMaxHeight {
		h = renderer.internalSurfaceMaxHeight
	}

	renderer.internalSurface = sdl.CreateRGBSurface(sdl.SWSURFACE, int(renderer.width), int(h), 32, 0, 0, 0, 0)
	renderer.calcCommandLineRect()
	renderer.cursorY = renderer.commandLineRect.Y
	renderer.viewportY = int16(renderer.internalSurface.H - renderer.visibleSurface.H)
	renderer.internalSurface.SetClipRect(&sdl.Rect{0, 0, renderer.width, h})
}

func (renderer *SDLRenderer) renderCommandLine(commandLine *commandLine) {
	renderer.clearPrompt()
	renderer.renderLine(0, commandLine.toString())
	renderer.addUpdatedRect(renderer.commandLineRect)
	renderer.renderCursor(commandLine)
}

func (renderer *SDLRenderer) renderConsole(console *Console) {
	renderer.resizeInternalSurface(console)
	for i := console.lines.Len(); i > 0; i-- {
		if i < renderer.lastVisibleLine {
			renderer.renderLine(i, console.lines.At(console.lines.Len()-i))
		} else {
			continue
		}
	}
	renderer.addUpdatedRect(&sdl.Rect{0, 0, uint16(renderer.internalSurface.W), uint16(renderer.internalSurface.H)})
	renderer.renderLine(0, console.commandLine.toString())
}

func (renderer *SDLRenderer) enableCursor(enable bool) {
	renderer.cursorOn = enable
}

func (renderer *SDLRenderer) clear() {
	renderer.internalSurface.FillRect(nil, 0)
}

func (renderer *SDLRenderer) clearPrompt() {
	renderer.internalSurface.FillRect(renderer.commandLineRect, 0)
}

func (renderer *SDLRenderer) renderLine(pos int, line string) {
	var textSurface *sdl.Surface

	if renderer.Blended {
		textSurface = ttf.RenderUTF8_Blended(renderer.Font, line, renderer.Color)
	} else {
		textSurface = ttf.RenderUTF8_Solid(renderer.Font, line, renderer.Color)
	}

	x := renderer.commandLineRect.X
	y := int16(renderer.commandLineRect.Y) - int16(renderer.commandLineRect.H*uint16(pos))
	w := renderer.commandLineRect.W
	h := renderer.commandLineRect.H

	if textSurface != nil {
		renderer.internalSurface.Blit(&sdl.Rect{x, y, w, h}, textSurface, nil)
		textSurface.Free()
	}
}

func (renderer *SDLRenderer) addUpdatedRect(rect *sdl.Rect) {
	visibleX, visibleY := renderer.transformToVisibleXY(rect.X, rect.Y)
	visibleRect := sdl.Rect{visibleX, visibleY, rect.W, rect.H}

	if visibleRect.Y > int16(renderer.height) {
		visibleRect.Y = int16(renderer.height)
	}
	if visibleRect.Y < 0 {
		visibleRect.Y = 0
	}
	if visibleRect.H > uint16(renderer.height) {
		visibleRect.H = uint16(renderer.height)
	}

	renderer.updatedRects = append(renderer.updatedRects, visibleRect)
}

// Return the address of pixel at (x,y)
func (renderer *SDLRenderer) getSurfaceAddr(x, y uint) uintptr {
	pixels := uintptr(unsafe.Pointer(renderer.internalSurface.Pixels))
	offset := uintptr(y*uint(renderer.internalSurface.Pitch) + x*uint(renderer.internalSurface.Format.BytesPerPixel))
	return uintptr(unsafe.Pointer(pixels + offset))
}

func (renderer *SDLRenderer) renderXORRect(x0, y0 int16, w, h uint16, color uint32) {
	x1 := x0 + int16(w)
	y1 := y0 + int16(h)
	for y := y0; y < y1; y++ {
		for x := x0; x < x1; x++ {
			addr := renderer.getSurfaceAddr(uint(x), uint(y))
			currColor := *(*uint32)(unsafe.Pointer(addr))
			*(*uint32)(unsafe.Pointer(addr)) = ^(currColor ^ color)
		}
	}
}

func (renderer *SDLRenderer) renderCursorRect(x int16) {
	var cursorColor uint32
	if renderer.cursorOn {
		cursorColor = 0
	} else {
		cursorColor = 0xffffffff
	}
	renderer.renderXORRect(x, renderer.cursorY, renderer.cursorWidth, renderer.cursorHeight, cursorColor)
	renderer.addUpdatedRect(&sdl.Rect{x, renderer.cursorY, renderer.cursorWidth, renderer.cursorHeight})
}

func (renderer *SDLRenderer) cursorX(commandLine *commandLine) int16 {
	var (
		cursorX  int = int(renderer.commandLineRect.X)
		finalPos = commandLine.cursorPosition + len(commandLine.prompt)
	)

	for pos, c := range commandLine.toString() {
		_, _, _, _, advance, _ := renderer.Font.GlyphMetrics(uint16(c))
		if pos < finalPos {
			cursorX += advance
		} else {
			break
		}
	}
	return int16(cursorX)
}

func (renderer *SDLRenderer) renderCursor(commandLine *commandLine) {
	renderer.renderCursorRect(renderer.cursorX(commandLine))
}

func (renderer *SDLRenderer) hasScrolled() bool {
	if renderer.viewportY != int16(renderer.internalSurface.H-renderer.visibleSurface.H) {
		return true
	}
	return false
}

func (renderer *SDLRenderer) scroll(direction float64) {
	if direction > 0 {
		renderer.viewportY += int16(direction)
	} else {
		renderer.viewportY += int16(direction)
	}
	if renderer.viewportY < 0 {
		renderer.viewportY = 0
	}
	if renderer.viewportY > int16(renderer.internalSurface.H-renderer.visibleSurface.H) {
		renderer.viewportY = int16(renderer.internalSurface.H - renderer.visibleSurface.H)
	}
}

func (renderer *SDLRenderer) transformToVisibleXY(internalX, internalY int16) (int16, int16) {
	visibleX := internalX
	visibleY := internalY - renderer.viewportY
	return visibleX, visibleY
}

func (renderer *SDLRenderer) transformToInternalXY(visibleX, visibleY int16) (int16, int16) {
	internalX := visibleX
	internalY := visibleY + renderer.viewportY
	return internalX, internalY
}

func (renderer *SDLRenderer) render(rects []sdl.Rect) {
	if rects != nil {
		for _, r := range rects {
			internalX, internalY := renderer.transformToInternalXY(r.X, r.Y)
			renderer.visibleSurface.Blit(&r, renderer.internalSurface, &sdl.Rect{internalX, internalY, r.W, r.H})
		}
	} else {
		h := uint16(int16(renderer.internalSurface.H) - renderer.viewportY)
		renderer.visibleSurface.Blit(nil, renderer.internalSurface, &sdl.Rect{0, renderer.viewportY, renderer.width, h})
		renderer.updatedRects = append(renderer.updatedRects, sdl.Rect{0, 0, renderer.width, renderer.height})
	}

	renderer.updatedRectsCh <- renderer.updatedRects
	renderer.updatedRects = make([]sdl.Rect, 0)
}

func (renderer *SDLRenderer) loop() {
	// Control goroutine
	go func() {
		for {
			renderer.paused = <-renderer.pauseCh
		}
	}()
	for {
		if !renderer.paused {
			select {
			case untyped_event := <-renderer.eventCh:
				switch event := untyped_event.(type) {
				case UpdateCommandLineEvent:
					if renderer.hasScrolled() {
						renderer.renderConsole(event.console)
					}
					renderer.enableCursor(true)
					renderer.renderCommandLine(event.commandLine)
				case UpdateConsoleEvent:
					renderer.renderConsole(event.console)
				case UpdateCursorEvent:
					renderer.enableCursor(event.enabled)
					renderer.renderCursor(event.commandLine)
				case PauseEvent:
					renderer.paused = event.paused
				}
				renderer.render(renderer.updatedRects)
			case dir := <-renderer.scrollCh:
				if dir == SCROLL_DOWN {
					renderer.Animations[SCROLL_DOWN_ANIMATION].Start()
				} else {
					renderer.Animations[SCROLL_UP_ANIMATION].Start()
				}
			case value := <-renderer.Animations[SCROLL_UP_ANIMATION].ValueCh():
				renderer.scroll(-value)
				renderer.render(nil)
			case <-renderer.Animations[SCROLL_UP_ANIMATION].FinishedCh():
			case value := <-renderer.Animations[SCROLL_DOWN_ANIMATION].ValueCh():
				renderer.scroll(value)
				renderer.render(nil)
			case <-renderer.Animations[SCROLL_DOWN_ANIMATION].FinishedCh():
			}
		} else {
			select {
			case untyped_event := <-renderer.eventCh:
				switch event := untyped_event.(type) {
				case PauseEvent:
					renderer.paused = event.paused
					if !event.paused { // if unpaused re-render the console
						renderer.renderConsole(event.console)
					}
				}
			case <-renderer.scrollCh:
			case <-renderer.Animations[SCROLL_UP_ANIMATION].ValueCh():
			case <-renderer.Animations[SCROLL_UP_ANIMATION].FinishedCh():
			case <-renderer.Animations[SCROLL_DOWN_ANIMATION].ValueCh():
			case <-renderer.Animations[SCROLL_DOWN_ANIMATION].FinishedCh():
			}
		}
	}

}
<commit_msg>Update in response to Go release.2011-01-20 (removal of the 'float' type)<commit_after>package clingon

import (
	"⚛sdl"
	"⚛sdl/ttf"
	"unsafe"
)

const (
	MAX_INTERNAL_SIZE_FACTOR     = 3
)

const (
	SCROLL_UP = iota
	SCROLL_DOWN
	SCROLL_UP_ANIMATION
	SCROLL_DOWN_ANIMATION
)

type SDLRenderer struct {
	// Activate/deactivate blended text rendering. By default use
	// solid text rendering
	Blended bool
	// Set the foreground color of the font (white by default)
	Color sdl.Color
	// Set the font family
	Font *ttf.Font

	// Map of built-in animations
	Animations map[int]*Animation

	internalSurface           *sdl.Surface
	visibleSurface            *sdl.Surface
	cursorOn                  bool
	eventCh                   chan interface{}
	scrollCh                  chan int
	updatedRectsCh            chan []sdl.Rect
	pauseCh                   chan bool
	updatedRects              []sdl.Rect
	viewportY                 int16
	commandLineRect           *sdl.Rect
	cursorY                   int16
	lastVisibleLine           int
	internalSurfaceMaxHeight  uint16
	width, height             uint16 // visible surface width, height
	cursorWidth, cursorHeight uint16 // cursor width, height
	fontWidth, fontHeight     int    // font width, height
	paused                    bool
}

func NewSDLRenderer(surface *sdl.Surface, font *ttf.Font) *SDLRenderer {
	rect := new(sdl.Rect)
	surface.GetClipRect(rect)

	renderer := &SDLRenderer{
		Color:          sdl.Color{255, 255, 255, 0},
		Animations:     make(map[int]*Animation),
		visibleSurface: surface,
		Font:           font,
		eventCh:        make(chan interface{}),
		scrollCh:       make(chan int),
		pauseCh:        make(chan bool),
		updatedRectsCh: make(chan []sdl.Rect),
		updatedRects:   make([]sdl.Rect, 0),
		width:          rect.W,
		height:         rect.H,
	}

	fontWidth, fontHeight, _ := font.SizeText("A")

	renderer.fontWidth = fontWidth
	renderer.fontHeight = fontHeight

	renderer.cursorWidth = uint16(fontWidth)
	renderer.cursorHeight = uint16(fontHeight)

	renderer.lastVisibleLine = (int(renderer.height)/renderer.fontHeight) * MAX_INTERNAL_SIZE_FACTOR
	renderer.internalSurfaceMaxHeight = renderer.height * MAX_INTERNAL_SIZE_FACTOR

	renderer.internalSurface = sdl.CreateRGBSurface(sdl.SWSURFACE, int(renderer.width), int(renderer.fontHeight), 32, 0, 0, 0, 0)
	renderer.calcCommandLineRect()

	renderer.Animations[SCROLL_UP_ANIMATION] = NewSlideUpAnimation(1e9, 10.0)
	renderer.Animations[SCROLL_DOWN_ANIMATION] = NewSlideUpAnimation(1e9, 10.0)

	renderer.updatedRects = append(renderer.updatedRects, sdl.Rect{0, 0, renderer.width, renderer.height})

	go renderer.loop()

	return renderer
}

// Receive the events triggered by the console.
func (renderer *SDLRenderer) EventCh() chan<- interface{} {
	return renderer.eventCh
}

// Tell the renderer to scroll up/down the console.
func (renderer *SDLRenderer) ScrollCh() chan<- int {
	return renderer.scrollCh
}

// Tell the renderer to pause its operations.
func (renderer *SDLRenderer) PauseCh() chan<- bool {
	return renderer.pauseCh
}

// From this channel, the client receives the rects representing the
// updated surface regions.
func (renderer *SDLRenderer) UpdatedRectsCh() <-chan []sdl.Rect {
	return renderer.updatedRectsCh
}

// Return the visible SDL surface
func (renderer *SDLRenderer) GetSurface() *sdl.Surface {
	return renderer.visibleSurface
}

func (renderer *SDLRenderer) calcCommandLineRect() {
	renderer.commandLineRect = &sdl.Rect{
		0,
		int16(renderer.internalSurface.H) - int16(renderer.fontHeight),
		uint16(renderer.internalSurface.W),
		uint16(renderer.fontHeight),
	}
}

func (renderer *SDLRenderer) resizeInternalSurface(console *Console) {
	if renderer.internalSurface != nil {
		renderer.internalSurface.Free()
	}

	h := uint16((console.lines.Len() + 1) * renderer.fontHeight)

	if h > renderer.internalSurfaceMaxHeight {
		h = renderer.internalSurfaceMaxHeight
	}

	renderer.internalSurface = sdl.CreateRGBSurface(sdl.SWSURFACE, int(renderer.width), int(h), 32, 0, 0, 0, 0)
	renderer.calcCommandLineRect()
	renderer.cursorY = renderer.commandLineRect.Y
	renderer.viewportY = int16(renderer.internalSurface.H - renderer.visibleSurface.H)
	renderer.internalSurface.SetClipRect(&sdl.Rect{0, 0, renderer.width, h})
}

func (renderer *SDLRenderer) renderCommandLine(commandLine *commandLine) {
	renderer.clearPrompt()
	renderer.renderLine(0, commandLine.toString())
	renderer.addUpdatedRect(renderer.commandLineRect)
	renderer.renderCursor(commandLine)
}

func (renderer *SDLRenderer) renderConsole(console *Console) {
	renderer.resizeInternalSurface(console)
	for i := console.lines.Len(); i > 0; i-- {
		if i < renderer.lastVisibleLine {
			renderer.renderLine(i, console.lines.At(console.lines.Len()-i))
		} else {
			continue
		}
	}
	renderer.addUpdatedRect(&sdl.Rect{0, 0, uint16(renderer.internalSurface.W), uint16(renderer.internalSurface.H)})
	renderer.renderLine(0, console.commandLine.toString())
}

func (renderer *SDLRenderer) enableCursor(enable bool) {
	renderer.cursorOn = enable
}

func (renderer *SDLRenderer) clear() {
	renderer.internalSurface.FillRect(nil, 0)
}

func (renderer *SDLRenderer) clearPrompt() {
	renderer.internalSurface.FillRect(renderer.commandLineRect, 0)
}

func (renderer *SDLRenderer) renderLine(pos int, line string) {
	var textSurface *sdl.Surface

	if renderer.Blended {
		textSurface = ttf.RenderUTF8_Blended(renderer.Font, line, renderer.Color)
	} else {
		textSurface = ttf.RenderUTF8_Solid(renderer.Font, line, renderer.Color)
	}

	x := renderer.commandLineRect.X
	y := int16(renderer.commandLineRect.Y) - int16(renderer.commandLineRect.H*uint16(pos))
	w := renderer.commandLineRect.W
	h := renderer.commandLineRect.H

	if textSurface != nil {
		renderer.internalSurface.Blit(&sdl.Rect{x, y, w, h}, textSurface, nil)
		textSurface.Free()
	}
}

func (renderer *SDLRenderer) addUpdatedRect(rect *sdl.Rect) {
	visibleX, visibleY := renderer.transformToVisibleXY(rect.X, rect.Y)
	visibleRect := sdl.Rect{visibleX, visibleY, rect.W, rect.H}

	if visibleRect.Y > int16(renderer.height) {
		visibleRect.Y = int16(renderer.height)
	}
	if visibleRect.Y < 0 {
		visibleRect.Y = 0
	}
	if visibleRect.H > uint16(renderer.height) {
		visibleRect.H = uint16(renderer.height)
	}

	renderer.updatedRects = append(renderer.updatedRects, visibleRect)
}

// Return the address of pixel at (x,y)
func (renderer *SDLRenderer) getSurfaceAddr(x, y uint) uintptr {
	pixels := uintptr(unsafe.Pointer(renderer.internalSurface.Pixels))
	offset := uintptr(y*uint(renderer.internalSurface.Pitch) + x*uint(renderer.internalSurface.Format.BytesPerPixel))
	return uintptr(unsafe.Pointer(pixels + offset))
}

func (renderer *SDLRenderer) renderXORRect(x0, y0 int16, w, h uint16, color uint32) {
	x1 := x0 + int16(w)
	y1 := y0 + int16(h)
	for y := y0; y < y1; y++ {
		for x := x0; x < x1; x++ {
			addr := renderer.getSurfaceAddr(uint(x), uint(y))
			currColor := *(*uint32)(unsafe.Pointer(addr))
			*(*uint32)(unsafe.Pointer(addr)) = ^(currColor ^ color)
		}
	}
}

func (renderer *SDLRenderer) renderCursorRect(x int16) {
	var cursorColor uint32
	if renderer.cursorOn {
		cursorColor = 0
	} else {
		cursorColor = 0xffffffff
	}
	renderer.renderXORRect(x, renderer.cursorY, renderer.cursorWidth, renderer.cursorHeight, cursorColor)
	renderer.addUpdatedRect(&sdl.Rect{x, renderer.cursorY, renderer.cursorWidth, renderer.cursorHeight})
}

func (renderer *SDLRenderer) cursorX(commandLine *commandLine) int16 {
	var (
		cursorX  int = int(renderer.commandLineRect.X)
		finalPos = commandLine.cursorPosition + len(commandLine.prompt)
	)

	for pos, c := range commandLine.toString() {
		_, _, _, _, advance, _ := renderer.Font.GlyphMetrics(uint16(c))
		if pos < finalPos {
			cursorX += advance
		} else {
			break
		}
	}
	return int16(cursorX)
}

func (renderer *SDLRenderer) renderCursor(commandLine *commandLine) {
	renderer.renderCursorRect(renderer.cursorX(commandLine))
}

func (renderer *SDLRenderer) hasScrolled() bool {
	if renderer.viewportY != int16(renderer.internalSurface.H-renderer.visibleSurface.H) {
		return true
	}
	return false
}

func (renderer *SDLRenderer) scroll(direction float64) {
	if direction > 0 {
		renderer.viewportY += int16(direction)
	} else {
		renderer.viewportY += int16(direction)
	}
	if renderer.viewportY < 0 {
		renderer.viewportY = 0
	}
	if renderer.viewportY > int16(renderer.internalSurface.H-renderer.visibleSurface.H) {
		renderer.viewportY = int16(renderer.internalSurface.H - renderer.visibleSurface.H)
	}
}

func (renderer *SDLRenderer) transformToVisibleXY(internalX, internalY int16) (int16, int16) {
	visibleX := internalX
	visibleY := internalY - renderer.viewportY
	return visibleX, visibleY
}

func (renderer *SDLRenderer) transformToInternalXY(visibleX, visibleY int16) (int16, int16) {
	internalX := visibleX
	internalY := visibleY + renderer.viewportY
	return internalX, internalY
}

func (renderer *SDLRenderer) render(rects []sdl.Rect) {
	if rects != nil {
		for _, r := range rects {
			internalX, internalY := renderer.transformToInternalXY(r.X, r.Y)
			renderer.visibleSurface.Blit(&r, renderer.internalSurface, &sdl.Rect{internalX, internalY, r.W, r.H})
		}
	} else {
		h := uint16(int16(renderer.internalSurface.H) - renderer.viewportY)
		renderer.visibleSurface.Blit(nil, renderer.internalSurface, &sdl.Rect{0, renderer.viewportY, renderer.width, h})
		renderer.updatedRects = append(renderer.updatedRects, sdl.Rect{0, 0, renderer.width, renderer.height})
	}

	renderer.updatedRectsCh <- renderer.updatedRects
	renderer.updatedRects = make([]sdl.Rect, 0)
}

func (renderer *SDLRenderer) loop() {
	// Control goroutine
	go func() {
		for {
			renderer.paused = <-renderer.pauseCh
		}
	}()
	for {
		if !renderer.paused {
			select {
			case untyped_event := <-renderer.eventCh:
				switch event := untyped_event.(type) {
				case UpdateCommandLineEvent:
					if renderer.hasScrolled() {
						renderer.renderConsole(event.console)
					}
					renderer.enableCursor(true)
					renderer.renderCommandLine(event.commandLine)
				case UpdateConsoleEvent:
					renderer.renderConsole(event.console)
				case UpdateCursorEvent:
					renderer.enableCursor(event.enabled)
					renderer.renderCursor(event.commandLine)
				case PauseEvent:
					renderer.paused = event.paused
				}
				renderer.render(renderer.updatedRects)
			case dir := <-renderer.scrollCh:
				if dir == SCROLL_DOWN {
					renderer.Animations[SCROLL_DOWN_ANIMATION].Start()
				} else {
					renderer.Animations[SCROLL_UP_ANIMATION].Start()
				}
			case value := <-renderer.Animations[SCROLL_UP_ANIMATION].ValueCh():
				renderer.scroll(-value)
				renderer.render(nil)
			case <-renderer.Animations[SCROLL_UP_ANIMATION].FinishedCh():
			case value := <-renderer.Animations[SCROLL_DOWN_ANIMATION].ValueCh():
				renderer.scroll(value)
				renderer.render(nil)
			case <-renderer.Animations[SCROLL_DOWN_ANIMATION].FinishedCh():
			}
		} else {
			select {
			case untyped_event := <-renderer.eventCh:
				switch event := untyped_event.(type) {
				case PauseEvent:
					renderer.paused = event.paused
					if !event.paused { // if unpaused re-render the console
						renderer.renderConsole(event.console)
					}
				}
			case <-renderer.scrollCh:
			case <-renderer.Animations[SCROLL_UP_ANIMATION].ValueCh():
			case <-renderer.Animations[SCROLL_UP_ANIMATION].FinishedCh():
			case <-renderer.Animations[SCROLL_DOWN_ANIMATION].ValueCh():
			case <-renderer.Animations[SCROLL_DOWN_ANIMATION].FinishedCh():
			}
		}
	}

}
<|endoftext|>


<commit_before>package ipreach

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"strings"
)

type Checker interface {
	CheckTCP(ip string) error
}

type HostCheckNet struct {
}

const HOST_CHECK_NET = "https://check-host.net/"
const WHATS_MY_IP = "http://whatsmyip.org/"

var ErrUnreachable = errors.New("Unreachable IP:PORT by the service")
var ErrUnknownResponse = errors.New("Unable to make sense of the response from the service")

// WhatsMyIp will only be able to check for your own IP address
type WhatsMyIp struct{}

func (w *WhatsMyIp) CheckTCP(ip string) error {
	_, port, err := net.SplitHostPort(ip)
	if err != nil {
		return err
	}
	data := bytes.NewBufferString("port=" + port + "&timeout=default")
	// ask the check
	url := WHATS_MY_IP + "port-scanner/scan.php"
	req, err := http.NewRequest("POST", url, data)
	if err != nil {
		return err
	}
	req.Header.Set("Host", "www.whatsmyip.org")
	req.Header.Set("Referer", "http://www.whatsmyip.org/port-scanner/")

	client := &http.Client{}
	fmt.Println("Created request for ", ip)
	resp, err := client.Do(req)
	if err != nil {
		return err
	}

	buffer, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	fmt.Println("Response:", buffer)
	fmt.Println("req.URL:", req.URL)
	if !bytes.Contains(buffer, []byte("1")) {
		return ErrUnreachable
	}
	return nil
}

// CheckTCP will check if the given IP address is reachable from the internet
// for TCP connection using https://check-host.net/
func (h *HostCheckNet) CheckTCP(ip string) error {
	// ask the check
	url := HOST_CHECK_NET + "check-tcp?host=" + ip + "&max_nodes=1"
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return err
	}
	req.Header.Set("Accept", "application/json")
	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	fmt.Println("Created request for ", ip)
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	fmt.Println("Requested for", ip, " sent to checker.")

	// get the request id and ask for the results
	buffResponse, _ := ioutil.ReadAll(resp.Body)
	fmt.Println("Response: ", string(buffResponse))
	reader := bytes.NewBuffer(buffResponse)
	dec := json.NewDecoder(reader)
	checkResp := &checkResponse{}
	if err := dec.Decode(checkResp); err != nil {
		return err
	}
	resp.Body.Close()

	fmt.Println("Decoded PermanentLink", checkResp.PermanentLink)
	req, err = http.NewRequest("GET", checkResp.PermanentLink, nil)
	if err != nil {
		return err
	}
	req.Header.Set("Accept", "application/json")
	req.Header.Set("Content-Type", "application/json")

	// get the response
	resp, err = client.Do(req)
	if err != nil {
		return err
	}
	fmt.Println("Got response from the check")
	buff, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	if strings.Contains(string(buff), "error") {
		return ErrUnreachable
	}
	return nil
}

type checkResponse struct {
	Ok            int    `json:"ok"`
	PermanentLink string `json:"permanent_link"`
	RequestId     string `json:"request_id"`
}
<commit_msg>print request<commit_after>package ipreach

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"strings"
)

type Checker interface {
	CheckTCP(ip string) error
}

type HostCheckNet struct {
}

const HOST_CHECK_NET = "https://check-host.net/"
const WHATS_MY_IP = "http://whatsmyip.org/"

var ErrUnreachable = errors.New("Unreachable IP:PORT by the service")
var ErrUnknownResponse = errors.New("Unable to make sense of the response from the service")

// WhatsMyIp will only be able to check for your own IP address
type WhatsMyIp struct{}

func (w *WhatsMyIp) CheckTCP(ip string) error {
	_, port, err := net.SplitHostPort(ip)
	if err != nil {
		return err
	}
	data := bytes.NewBufferString("port=" + port + "&timeout=default")
	// ask the check
	url := WHATS_MY_IP + "port-scanner/scan.php"
	req, err := http.NewRequest("POST", url, data)
	if err != nil {
		return err
	}
	req.Header.Set("Host", "www.whatsmyip.org")
	req.Header.Set("Referer", "http://www.whatsmyip.org/port-scanner/")

	client := &http.Client{}
	fmt.Println("Created request for ", ip)
	resp, err := client.Do(req)
	if err != nil {
		return err
	}

	buffer, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	fmt.Println("Response:", buffer)
	requestBuff := &bytes.Buffer{}
	err := req.Write(requestBuff)
	fmt.Println(requestBuff, err)
	if !bytes.Contains(buffer, []byte("1")) {
		return ErrUnreachable
	}
	return nil
}

// CheckTCP will check if the given IP address is reachable from the internet
// for TCP connection using https://check-host.net/
func (h *HostCheckNet) CheckTCP(ip string) error {
	// ask the check
	url := HOST_CHECK_NET + "check-tcp?host=" + ip + "&max_nodes=1"
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return err
	}
	req.Header.Set("Accept", "application/json")
	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	fmt.Println("Created request for ", ip)
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	fmt.Println("Requested for", ip, " sent to checker.")

	// get the request id and ask for the results
	buffResponse, _ := ioutil.ReadAll(resp.Body)
	fmt.Println("Response: ", string(buffResponse))
	reader := bytes.NewBuffer(buffResponse)
	dec := json.NewDecoder(reader)
	checkResp := &checkResponse{}
	if err := dec.Decode(checkResp); err != nil {
		return err
	}
	resp.Body.Close()

	fmt.Println("Decoded PermanentLink", checkResp.PermanentLink)
	req, err = http.NewRequest("GET", checkResp.PermanentLink, nil)
	if err != nil {
		return err
	}
	req.Header.Set("Accept", "application/json")
	req.Header.Set("Content-Type", "application/json")

	// get the response
	resp, err = client.Do(req)
	if err != nil {
		return err
	}
	fmt.Println("Got response from the check")
	buff, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	if strings.Contains(string(buff), "error") {
		return ErrUnreachable
	}
	return nil
}

type checkResponse struct {
	Ok            int    `json:"ok"`
	PermanentLink string `json:"permanent_link"`
	RequestId     string `json:"request_id"`
}
<|endoftext|>


<commit_before>// Streaming relation (overlap, distance, KNN) testing of (any number of) sorted files of intervals.
package irelate

import (
	"container/heap"
	"fmt"
	"log"
	"os"

	. "github.com/brentp/irelate/interfaces"

	"vbom.ml/util/sortorder"
)

// RelatableChannel
type RelatableChannel chan Relatable

func relate(a Relatable, b Relatable, relativeTo int) {
	if a.Source() != b.Source() {
		if relativeTo == -1 {
			a.AddRelated(b)
			b.AddRelated(a)
		} else {
			if uint32(relativeTo) == a.Source() {
				a.AddRelated(b)
			}
			if uint32(relativeTo) == b.Source() {
				b.AddRelated(a)
			}
		}
	}
}

func Less(a Relatable, b Relatable) bool {
	if a.Chrom() != b.Chrom() {
		return a.Chrom() < b.Chrom()
	}
	return a.Start() < b.Start() // || (a.Start() == b.Start() && a.End() < b.End())
}

// 1, 2, 3 ... 9, 10, 11...
func NaturalLessPrefix(a Relatable, b Relatable) bool {
	if !SameChrom(a.Chrom(), b.Chrom()) {
		return sortorder.NaturalLess(StripChr(a.Chrom()), StripChr(b.Chrom()))
	}
	return a.Start() < b.Start() || (a.Start() == b.Start() && a.End() < b.End())

}

// 1, 10, 11... 19, 2, 20, 21 ...
func LessPrefix(a Relatable, b Relatable) bool {
	if !SameChrom(a.Chrom(), b.Chrom()) {
		return StripChr(a.Chrom()) < StripChr(b.Chrom())
	}
	return a.Start() < b.Start() || (a.Start() == b.Start() && a.End() < b.End())
}

// CheckRelatedByOverlap returns true if Relatables overlap.
func CheckRelatedByOverlap(a Relatable, b Relatable) bool {
	return (b.Start() < a.End()) && (b.Chrom() == a.Chrom())
	// note with distance == 0 this just overlap.
	//distance := uint32(0)
	//return (b.Start()-distance < a.End()) && (b.Chrom() == a.Chrom())
}

// handles chromomomes like 'chr1' from one org and '1' from another.
func CheckOverlapPrefix(a Relatable, b Relatable) bool {
	if b.Start() < a.End() {
		return SameChrom(a.Chrom(), b.Chrom())
	}
	return false
}

// CheckKNN relates an interval to its k-nearest neighbors.
// The reporting function will have to do some filtering since this is only
// guaranteed to associate *at least* k neighbors, but it could be returning extra.
func CheckKNN(a Relatable, b Relatable) bool {
	// the first n checked would be the n_closest, but need to consider ties
	// the report function can decide what to do with them.
	k := 4
	r := a.Related()
	if len(r) >= k {
		// TODO: double-check this.
		return r[len(r)-1].Start()-a.End() < b.Start()-a.End()
	}
	return true
}

// filter rewrites the input-slice to remove nils.
func filter(s []Relatable, nils int) []Relatable {
	j := 0
	if len(s) != nils {

		for _, v := range s {
			if v != nil {
				s[j] = v
				j++
			}
		}
	}
	for k := j; k < len(s); k++ {
		s[k] = nil
	}
	return s[:j]
}

// Send the relatables to the channel in sorted order.
// Check that we couldn't later get an item with a lower start from the current cache.
func sendSortedRelatables(sendQ *relatableQueue, cache []Relatable, out chan Relatable, less func(a, b Relatable) bool) {
	var j int
	for j = 0; j < len((*sendQ).rels) && (len(cache) == 0 || less((*sendQ).rels[j].(Relatable), cache[0])); j++ {
	}
	for i := 0; i < j; i++ {
		out <- heap.Pop(sendQ).(Relatable)
	}
}

// IRelate provides the basis for flexible overlap/proximity/k-nearest neighbor
// testing. IRelate receives merged, ordered Relatables via stream and takes
// function that checks if they are related (see CheckRelatedByOverlap).
// It is guaranteed that !Less(b, a) is true (we can't guarantee that Less(a, b)
// is true since they may have the same start). Once checkRelated returns false,
// it is assumed that no other `b` Relatables could possibly be related to `a`
// and so `a` is sent to the returnQ.
// streams are a variable number of channels that send intervals.
func IRelate(checkRelated func(a, b Relatable) bool,
	relativeTo int,
	less func(a, b Relatable) bool,
	streams ...RelatableChannel) chan Relatable {

	// we infer the chromosome order by the order that we see from source 0.
	stream := Merge(less, relativeTo, streams...)
	out := make(chan Relatable, 64)
	go func() {

		// use the cache to keep relatables to test against.
		cache := make([]Relatable, 1, 1024)
		cache[0] = <-stream

		// Use sendQ to make sure we output in sorted order.
		// We know we can print something when sendQ.minStart < cache.minStart
		sendQ := relatableQueue{make([]Relatable, 0, 1024), less}
		nils := 0

		// TODO:if we know the ends are sorted (in addition to start) then we have some additional
		// optimizations. As soon as checkRelated is false, then all others in the cache before that
		// should be true... binary search if endSorted and len(cache) > 20?
		//endSorted := true
		for interval := range stream {

			for i, c := range cache {
				// tried using futures for checkRelated to parallelize... got slower
				if c == nil {
					continue
				}
				if checkRelated(c, interval) {
					relate(c, interval, relativeTo)
				} else {
					if relativeTo == -1 || c.Source() == uint32(relativeTo) {
						heap.Push(&sendQ, c)
					}
					cache[i] = nil
					nils++
				}
			}

			// only do this when we have a lot of nils as it's expensive to create a new slice.
			if nils > 1 {
				// remove nils from the cache (must do this before sending)
				cache, nils = filter(cache, nils), 0
				// send the elements from cache in order.
				// use heuristic to minimize the sending.
				if len(sendQ.rels) > 8 {
					sendSortedRelatables(&sendQ, cache, out, less)
				}
			}
			cache = append(cache, interval)

		}
		for _, c := range filter(cache, nils) {
			if c.Source() == uint32(relativeTo) || relativeTo == -1 {
				heap.Push(&sendQ, c)
			}
		}
		for i := 0; i < len(sendQ.rels); i++ {
			out <- sendQ.rels[i]
		}
		close(out)
	}()
	return out
}

// Merge accepts channels of Relatables and merges them in order.
// Streams of Relatable's from different source must be merged to send
// to IRelate.
// This uses a priority queue and acts like python's heapq.merge.
func Merge(less func(a, b Relatable) bool, relativeTo int, streams ...RelatableChannel) RelatableChannel {
	verbose := os.Getenv("IRELATE_VERBOSE") == "TRUE"
	q := relatableQueue{make([]Relatable, 0, len(streams)), less}
	seen := make(map[string]struct{})
	for i, stream := range streams {
		interval := <-stream
		if interval != nil {
			interval.SetSource(uint32(i))
			heap.Push(&q, interval)
		}
	}

	ch := make(chan Relatable, 8)
	go func() {
		var interval Relatable
		sentinel := struct{}{}
		lastChrom := ""
		// heuristic to use this to stop when end of query records is reached.
		j := -1000
		for len(q.rels) > 0 {
			interval = heap.Pop(&q).(Relatable)
			source := interval.Source()
			ch <- interval
			if SameChrom(interval.Chrom(), lastChrom) {
				lastChrom = StripChr(interval.Chrom())
				if _, ok := seen[lastChrom]; ok {
					log.Println("warning: chromosomes must be in different order between files or the chromosome sort order is not as expected.")
					log.Printf("warning: overlaps will likely be missed after this chrom: %s from source: %d\n", lastChrom, interval.Source())
				}
				seen[lastChrom] = sentinel
				if verbose {
					log.Printf("on chromosome: %s\n", lastChrom)
				}
			}
			// pull the next interval from the same source.
			next_interval, ok := <-streams[source]
			if ok {
				if next_interval.Start() < interval.Start() {
					if SameChrom(next_interval.Chrom(), interval.Chrom()) {
						panic(fmt.Sprintf("intervals out of order within file: starts at: %d and %d from source: %d", interval.Start(), next_interval.Start(), source))
					}
				}
				next_interval.SetSource(source)
				heap.Push(&q, next_interval)
				j--
				if j == 0 {
					break
				}
			} else {
				if int(source) == relativeTo {
					// we pull in 200K more records and then stop. to make sure we get anything that might
					// relate to last query
					j = 200000
				}
			}
		}
		close(ch)
	}()
	return ch
}
<commit_msg>fix test for same Chrom. now we get warning messages if files are not in same order.<commit_after>// Streaming relation (overlap, distance, KNN) testing of (any number of) sorted files of intervals.
package irelate

import (
	"container/heap"
	"fmt"
	"log"
	"os"

	. "github.com/brentp/irelate/interfaces"

	"vbom.ml/util/sortorder"
)

// RelatableChannel
type RelatableChannel chan Relatable

func relate(a Relatable, b Relatable, relativeTo int) {
	if a.Source() != b.Source() {
		if relativeTo == -1 {
			a.AddRelated(b)
			b.AddRelated(a)
		} else {
			if uint32(relativeTo) == a.Source() {
				a.AddRelated(b)
			}
			if uint32(relativeTo) == b.Source() {
				b.AddRelated(a)
			}
		}
	}
}

func Less(a Relatable, b Relatable) bool {
	if a.Chrom() != b.Chrom() {
		return a.Chrom() < b.Chrom()
	}
	return a.Start() < b.Start() // || (a.Start() == b.Start() && a.End() < b.End())
}

// 1, 2, 3 ... 9, 10, 11...
func NaturalLessPrefix(a Relatable, b Relatable) bool {
	if !SameChrom(a.Chrom(), b.Chrom()) {
		return sortorder.NaturalLess(StripChr(a.Chrom()), StripChr(b.Chrom()))
	}
	return a.Start() < b.Start() || (a.Start() == b.Start() && a.End() < b.End())

}

// 1, 10, 11... 19, 2, 20, 21 ...
func LessPrefix(a Relatable, b Relatable) bool {
	if !SameChrom(a.Chrom(), b.Chrom()) {
		return StripChr(a.Chrom()) < StripChr(b.Chrom())
	}
	return a.Start() < b.Start() || (a.Start() == b.Start() && a.End() < b.End())
}

// CheckRelatedByOverlap returns true if Relatables overlap.
func CheckRelatedByOverlap(a Relatable, b Relatable) bool {
	return (b.Start() < a.End()) && (b.Chrom() == a.Chrom())
	// note with distance == 0 this just overlap.
	//distance := uint32(0)
	//return (b.Start()-distance < a.End()) && (b.Chrom() == a.Chrom())
}

// handles chromomomes like 'chr1' from one org and '1' from another.
func CheckOverlapPrefix(a Relatable, b Relatable) bool {
	if b.Start() < a.End() {
		return SameChrom(a.Chrom(), b.Chrom())
	}
	return false
}

// CheckKNN relates an interval to its k-nearest neighbors.
// The reporting function will have to do some filtering since this is only
// guaranteed to associate *at least* k neighbors, but it could be returning extra.
func CheckKNN(a Relatable, b Relatable) bool {
	// the first n checked would be the n_closest, but need to consider ties
	// the report function can decide what to do with them.
	k := 4
	r := a.Related()
	if len(r) >= k {
		// TODO: double-check this.
		return r[len(r)-1].Start()-a.End() < b.Start()-a.End()
	}
	return true
}

// filter rewrites the input-slice to remove nils.
func filter(s []Relatable, nils int) []Relatable {
	j := 0
	if len(s) != nils {

		for _, v := range s {
			if v != nil {
				s[j] = v
				j++
			}
		}
	}
	for k := j; k < len(s); k++ {
		s[k] = nil
	}
	return s[:j]
}

// Send the relatables to the channel in sorted order.
// Check that we couldn't later get an item with a lower start from the current cache.
func sendSortedRelatables(sendQ *relatableQueue, cache []Relatable, out chan Relatable, less func(a, b Relatable) bool) {
	var j int
	for j = 0; j < len((*sendQ).rels) && (len(cache) == 0 || less((*sendQ).rels[j].(Relatable), cache[0])); j++ {
	}
	for i := 0; i < j; i++ {
		out <- heap.Pop(sendQ).(Relatable)
	}
}

// IRelate provides the basis for flexible overlap/proximity/k-nearest neighbor
// testing. IRelate receives merged, ordered Relatables via stream and takes
// function that checks if they are related (see CheckRelatedByOverlap).
// It is guaranteed that !Less(b, a) is true (we can't guarantee that Less(a, b)
// is true since they may have the same start). Once checkRelated returns false,
// it is assumed that no other `b` Relatables could possibly be related to `a`
// and so `a` is sent to the returnQ.
// streams are a variable number of channels that send intervals.
func IRelate(checkRelated func(a, b Relatable) bool,
	relativeTo int,
	less func(a, b Relatable) bool,
	streams ...RelatableChannel) chan Relatable {

	// we infer the chromosome order by the order that we see from source 0.
	stream := Merge(less, relativeTo, streams...)
	out := make(chan Relatable, 64)
	go func() {

		// use the cache to keep relatables to test against.
		cache := make([]Relatable, 1, 1024)
		cache[0] = <-stream

		// Use sendQ to make sure we output in sorted order.
		// We know we can print something when sendQ.minStart < cache.minStart
		sendQ := relatableQueue{make([]Relatable, 0, 1024), less}
		nils := 0

		// TODO:if we know the ends are sorted (in addition to start) then we have some additional
		// optimizations. As soon as checkRelated is false, then all others in the cache before that
		// should be true... binary search if endSorted and len(cache) > 20?
		//endSorted := true
		for interval := range stream {

			for i, c := range cache {
				// tried using futures for checkRelated to parallelize... got slower
				if c == nil {
					continue
				}
				if checkRelated(c, interval) {
					relate(c, interval, relativeTo)
				} else {
					if relativeTo == -1 || c.Source() == uint32(relativeTo) {
						heap.Push(&sendQ, c)
					}
					cache[i] = nil
					nils++
				}
			}

			// only do this when we have a lot of nils as it's expensive to create a new slice.
			if nils > 1 {
				// remove nils from the cache (must do this before sending)
				cache, nils = filter(cache, nils), 0
				// send the elements from cache in order.
				// use heuristic to minimize the sending.
				if len(sendQ.rels) > 8 {
					sendSortedRelatables(&sendQ, cache, out, less)
				}
			}
			cache = append(cache, interval)

		}
		for _, c := range filter(cache, nils) {
			if c.Source() == uint32(relativeTo) || relativeTo == -1 {
				heap.Push(&sendQ, c)
			}
		}
		for i := 0; i < len(sendQ.rels); i++ {
			out <- sendQ.rels[i]
		}
		close(out)
	}()
	return out
}

// Merge accepts channels of Relatables and merges them in order.
// Streams of Relatable's from different source must be merged to send
// to IRelate.
// This uses a priority queue and acts like python's heapq.merge.
func Merge(less func(a, b Relatable) bool, relativeTo int, streams ...RelatableChannel) RelatableChannel {
	verbose := os.Getenv("IRELATE_VERBOSE") == "TRUE"
	q := relatableQueue{make([]Relatable, 0, len(streams)), less}
	seen := make(map[string]struct{})
	for i, stream := range streams {
		interval := <-stream
		if interval != nil {
			interval.SetSource(uint32(i))
			heap.Push(&q, interval)
		}
	}

	ch := make(chan Relatable, 8)
	go func() {
		var interval Relatable
		sentinel := struct{}{}
		lastChrom := ""
		// heuristic to use this to stop when end of query records is reached.
		j := -1000
		for len(q.rels) > 0 {
			interval = heap.Pop(&q).(Relatable)
			source := interval.Source()
			ch <- interval
			if !SameChrom(interval.Chrom(), lastChrom) {
				lastChrom = StripChr(interval.Chrom())
				if _, ok := seen[lastChrom]; ok {
					log.Println("warning: chromosomes must be in different order between files or the chromosome sort order is not as expected.")
					log.Printf("warning: overlaps will likely be missed after this chrom: %s from source: %d\n", lastChrom, interval.Source())
				}
				seen[lastChrom] = sentinel
				if verbose {
					log.Printf("on chromosome: %s\n", lastChrom)
				}
			}
			// pull the next interval from the same source.
			next_interval, ok := <-streams[source]
			if ok {
				if next_interval.Start() < interval.Start() {
					if SameChrom(next_interval.Chrom(), interval.Chrom()) {
						panic(fmt.Sprintf("intervals out of order within file: starts at: %d and %d from source: %d", interval.Start(), next_interval.Start(), source))
					}
				}
				next_interval.SetSource(source)
				heap.Push(&q, next_interval)
				j--
				if j == 0 {
					break
				}
			} else {
				if int(source) == relativeTo {
					// we pull in 200K more records and then stop. to make sure we get anything that might
					// relate to last query
					j = 200000
				}
			}
		}
		close(ch)
	}()
	return ch
}
<|endoftext|>


<commit_before>package influx

import (
	"strings"
	"time"

	"github.com/influxdata/chronograf"
	"github.com/influxdata/influxdb/influxql"
)

// Convert changes an InfluxQL query to a QueryConfig
func Convert(influxQL string) (chronograf.QueryConfig, error) {
	query, err := influxql.ParseQuery(influxQL)
	if err != nil {
		return chronograf.QueryConfig{}, err
	}

	raw := chronograf.QueryConfig{
		RawText: &influxQL,
		Fields:  []chronograf.Field{},
		GroupBy: chronograf.GroupBy{
			Tags: []string{},
		},
		Tags: make(map[string][]string, 0),
	}
	qc := chronograf.QueryConfig{
		GroupBy: chronograf.GroupBy{
			Tags: []string{},
		},
		Tags: make(map[string][]string, 0),
	}

	if len(query.Statements) != 1 {
		return raw, nil
	}

	stmt, ok := query.Statements[0].(*influxql.SelectStatement)
	if !ok {
		return raw, nil
	}

	// Query config doesn't support limits
	if stmt.Limit != 0 || stmt.Offset != 0 || stmt.SLimit != 0 || stmt.SOffset != 0 {
		return raw, nil
	}

	// Query config doesn't support sorting
	if len(stmt.SortFields) > 0 {
		return raw, nil
	}

	// Query config doesn't support fill
	if stmt.Fill != influxql.NullFill {
		return raw, nil
	}

	// Query config doesn't allow SELECT INTO
	if stmt.Target != nil {
		return raw, nil
	}

	// Query config only allows selecting from one source at a time.
	if len(stmt.Sources) != 1 {
		return raw, nil
	}

	src := stmt.Sources[0]
	measurement, ok := src.(*influxql.Measurement)
	if !ok {
		return raw, nil
	}

	if measurement.Regex != nil {
		return raw, nil
	}
	qc.Database = measurement.Database
	qc.RetentionPolicy = measurement.RetentionPolicy
	qc.Measurement = measurement.Name

	for _, dim := range stmt.Dimensions {
		switch v := dim.Expr.(type) {
		default:
			return raw, nil
		case *influxql.Call:
			if v.Name != "time" {
				return raw, nil
			}
			// Make sure there is exactly one argument.
			if len(v.Args) != 1 {
				return raw, nil
			}
			// Ensure the argument is a duration.
			lit, ok := v.Args[0].(*influxql.DurationLiteral)
			if !ok {
				return raw, nil
			}
			qc.GroupBy.Time = lit.String()
		case *influxql.VarRef:
			qc.GroupBy.Tags = append(qc.GroupBy.Tags, v.Val)
		}
	}

	fields := map[string][]string{}
	for _, fld := range stmt.Fields {
		switch f := fld.Expr.(type) {
		default:
			return raw, nil
		case *influxql.Call:
			// only support certain query config functions
			if _, ok := supportedFuncs[f.Name]; !ok {
				return raw, nil
			}
			// Query configs only support single argument functions
			if len(f.Args) != 1 {
				return raw, nil
			}
			ref, ok := f.Args[0].(*influxql.VarRef)
			// query config only support fields in the function
			if !ok {
				return raw, nil
			}
			// We only support field strings
			if ref.Type != influxql.Unknown {
				return raw, nil
			}
			if call, ok := fields[ref.Val]; !ok {
				fields[ref.Val] = []string{f.Name}
			} else {
				fields[ref.Val] = append(call, f.Name)
			}
		case *influxql.VarRef:
			if f.Type != influxql.Unknown {
				return raw, nil
			}
			if _, ok := fields[f.Val]; !ok {
				fields[f.Val] = []string{}
			}
		}
	}

	for fld, funcs := range fields {
		qc.Fields = append(qc.Fields, chronograf.Field{
			Field: fld,
			Funcs: funcs,
		})
	}

	if stmt.Condition == nil {
		return qc, nil
	}

	reduced := influxql.Reduce(stmt.Condition, nil)
	logic, ok := isTagLogic(reduced)
	if !ok {
		return raw, nil
	}

	ops := map[string]bool{}
	for _, l := range logic {
		values, ok := qc.Tags[l.Tag]
		if !ok {
			values = []string{}
		}
		ops[l.Op] = true
		values = append(values, l.Value)
		qc.Tags[l.Tag] = values
	}

	if len(logic) > 0 {
		if len(ops) != 1 {
			return raw, nil
		}
		if _, ok := ops["=="]; ok {
			qc.AreTagsAccepted = true
		}
	}

	// If the condition has a time range we report back its duration
	if dur, ok := hasTimeRange(stmt.Condition); ok {
		qc.Range = &chronograf.DurationRange{
			Lower: "now() - " + shortDur(dur),
		}
	}

	return qc, nil
}

// tagFilter represents a single tag that is filtered by some condition
type tagFilter struct {
	Op    string
	Tag   string
	Value string
}

func isTime(exp influxql.Expr) bool {
	if p, ok := exp.(*influxql.ParenExpr); ok {
		return isTime(p.Expr)
	} else if ref, ok := exp.(*influxql.VarRef); ok && strings.ToLower(ref.Val) == "time" {
		return true
	}
	return false
}

func isNow(exp influxql.Expr) bool {
	if p, ok := exp.(*influxql.ParenExpr); ok {
		return isNow(p.Expr)
	} else if call, ok := exp.(*influxql.Call); ok && strings.ToLower(call.Name) == "now" && len(call.Args) == 0 {
		return true
	}
	return false
}

func isDuration(exp influxql.Expr) (time.Duration, bool) {
	switch e := exp.(type) {
	case *influxql.ParenExpr:
		return isDuration(e.Expr)
	case *influxql.DurationLiteral:
		return e.Val, true
	case *influxql.NumberLiteral, *influxql.IntegerLiteral, *influxql.TimeLiteral:
		return 0, false
	}
	return 0, false
}

func isPreviousTime(exp influxql.Expr) (time.Duration, bool) {
	if p, ok := exp.(*influxql.ParenExpr); ok {
		return isPreviousTime(p.Expr)
	} else if bin, ok := exp.(*influxql.BinaryExpr); ok {
		now := isNow(bin.LHS) || isNow(bin.RHS) // either side can be now
		op := bin.Op == influxql.SUB
		dur, hasDur := isDuration(bin.LHS)
		if !hasDur {
			dur, hasDur = isDuration(bin.RHS)
		}
		return dur, now && op && hasDur
	} else if isNow(exp) { // just comparing to now
		return 0, true
	}
	return 0, false
}

func isTimeRange(exp influxql.Expr) (time.Duration, bool) {
	if p, ok := exp.(*influxql.ParenExpr); ok {
		return isTimeRange(p.Expr)
	} else if bin, ok := exp.(*influxql.BinaryExpr); ok {
		tm := isTime(bin.LHS) || isTime(bin.RHS) // Either side could be time
		op := false
		switch bin.Op {
		case influxql.LT, influxql.LTE, influxql.GT, influxql.GTE:
			op = true
		}
		dur, prev := isPreviousTime(bin.LHS)
		if !prev {
			dur, prev = isPreviousTime(bin.RHS)
		}
		return dur, tm && op && prev
	}
	return 0, false
}

func hasTimeRange(exp influxql.Expr) (time.Duration, bool) {
	if p, ok := exp.(*influxql.ParenExpr); ok {
		return hasTimeRange(p.Expr)
	} else if dur, ok := isTimeRange(exp); ok {
		return dur, true
	} else if bin, ok := exp.(*influxql.BinaryExpr); ok {
		dur, ok := isTimeRange(bin.LHS)
		if !ok {
			dur, ok = isTimeRange(bin.RHS)
		}
		return dur, ok
	}
	return 0, false
}

func isTagLogic(exp influxql.Expr) ([]tagFilter, bool) {
	if p, ok := exp.(*influxql.ParenExpr); ok {
		return isTagLogic(p.Expr)
	}

	if _, ok := isTimeRange(exp); ok {
		return nil, true
	} else if tf, ok := isTagFilter(exp); ok {
		return []tagFilter{tf}, true
	}

	bin, ok := exp.(*influxql.BinaryExpr)
	if !ok {
		return nil, false
	}

	lhs, lhsOK := isTagFilter(bin.LHS)
	rhs, rhsOK := isTagFilter(bin.RHS)

	if lhsOK && rhsOK && lhs.Tag == rhs.Tag && lhs.Op == rhs.Op && bin.Op == influxql.OR {
		return []tagFilter{lhs, rhs}, true
	}

	if bin.Op != influxql.AND {
		return nil, false
	}

	_, tm := isTimeRange(bin.LHS)
	if !tm {
		_, tm = isTimeRange(bin.RHS)
	}
	tf := lhsOK || rhsOK
	if tm && tf {
		if lhsOK {
			return []tagFilter{lhs}, true
		}
		return []tagFilter{rhs}, true
	}

	tlLHS, lhsOK := isTagLogic(bin.LHS)
	tlRHS, rhsOK := isTagLogic(bin.RHS)
	if lhsOK && rhsOK {
		ops := map[string]bool{} // there must only be one kind of ops
		for _, tf := range tlLHS {
			ops[tf.Op] = true
		}
		for _, tf := range tlRHS {
			ops[tf.Op] = true
		}
		if len(ops) > 1 {
			return nil, false
		}
		return append(tlLHS, tlRHS...), true
	}
	return nil, false
}

func isVarRef(exp influxql.Expr) bool {
	if p, ok := exp.(*influxql.ParenExpr); ok {
		return isVarRef(p.Expr)
	} else if _, ok := exp.(*influxql.VarRef); ok {
		return true
	}
	return false
}

func isString(exp influxql.Expr) bool {
	if p, ok := exp.(*influxql.ParenExpr); ok {
		return isString(p.Expr)
	} else if _, ok := exp.(*influxql.StringLiteral); ok {
		return true
	}
	return false
}

func isTagFilter(exp influxql.Expr) (tagFilter, bool) {
	switch expr := exp.(type) {
	default:
		return tagFilter{}, false
	case *influxql.ParenExpr:
		return isTagFilter(expr.Expr)
	case *influxql.BinaryExpr:
		var Op string
		if expr.Op == influxql.EQ {
			Op = "=="
		} else if expr.Op == influxql.NEQ {
			Op = "!="
		} else {
			return tagFilter{}, false
		}

		hasValue := isString(expr.LHS) || isString(expr.RHS)
		hasTag := isVarRef(expr.LHS) || isVarRef(expr.RHS)
		if !(hasValue && hasTag) {
			return tagFilter{}, false
		}

		value := ""
		tag := ""
		// Either tag op value or value op tag
		if isVarRef(expr.LHS) {
			t, _ := expr.LHS.(*influxql.VarRef)
			tag = t.Val
			v, _ := expr.RHS.(*influxql.StringLiteral)
			value = v.Val
		} else {
			t, _ := expr.RHS.(*influxql.VarRef)
			tag = t.Val
			v, _ := expr.LHS.(*influxql.StringLiteral)
			value = v.Val
		}

		return tagFilter{
			Op:    Op,
			Tag:   tag,
			Value: value,
		}, true
	}
}

var supportedFuncs = map[string]bool{
	"mean":   true,
	"median": true,
	"count":  true,
	"min":    true,
	"max":    true,
	"sum":    true,
	"first":  true,
	"last":   true,
	"spread": true,
	"stddev": true,
}

// shortDur converts duration into the queryConfig duration format
func shortDur(d time.Duration) string {
	s := d.String()
	if strings.HasSuffix(s, "m0s") {
		s = s[:len(s)-2]
	}
	if strings.HasSuffix(s, "h0m") {
		s = s[:len(s)-2]
	}
	return s
}
<commit_msg>Cowabunga, it's dashboard 🕑<commit_after>package influx

import (
	"strings"
	"time"

	"github.com/influxdata/chronograf"
	"github.com/influxdata/influxdb/influxql"
)

// Convert changes an InfluxQL query to a QueryConfig
func Convert(influxQL string) (chronograf.QueryConfig, error) {
	itsDashboardTime := false
	if strings.Contains(influxQL, ":dashboardTime:") {
		influxQL = strings.Replace(influxQL, ":dashboardTime:", "now() - 15m", 1)
		itsDashboardTime = true
	}

	query, err := influxql.ParseQuery(influxQL)
	if err != nil {
		return chronograf.QueryConfig{}, err
	}

	raw := chronograf.QueryConfig{
		RawText: &influxQL,
		Fields:  []chronograf.Field{},
		GroupBy: chronograf.GroupBy{
			Tags: []string{},
		},
		Tags: make(map[string][]string, 0),
	}
	qc := chronograf.QueryConfig{
		GroupBy: chronograf.GroupBy{
			Tags: []string{},
		},
		Tags: make(map[string][]string, 0),
	}

	if len(query.Statements) != 1 {
		return raw, nil
	}

	stmt, ok := query.Statements[0].(*influxql.SelectStatement)
	if !ok {
		return raw, nil
	}

	// Query config doesn't support limits
	if stmt.Limit != 0 || stmt.Offset != 0 || stmt.SLimit != 0 || stmt.SOffset != 0 {
		return raw, nil
	}

	// Query config doesn't support sorting
	if len(stmt.SortFields) > 0 {
		return raw, nil
	}

	// Query config doesn't support fill
	if stmt.Fill != influxql.NullFill {
		return raw, nil
	}

	// Query config doesn't allow SELECT INTO
	if stmt.Target != nil {
		return raw, nil
	}

	// Query config only allows selecting from one source at a time.
	if len(stmt.Sources) != 1 {
		return raw, nil
	}

	src := stmt.Sources[0]
	measurement, ok := src.(*influxql.Measurement)
	if !ok {
		return raw, nil
	}

	if measurement.Regex != nil {
		return raw, nil
	}
	qc.Database = measurement.Database
	qc.RetentionPolicy = measurement.RetentionPolicy
	qc.Measurement = measurement.Name

	for _, dim := range stmt.Dimensions {
		switch v := dim.Expr.(type) {
		default:
			return raw, nil
		case *influxql.Call:
			if v.Name != "time" {
				return raw, nil
			}
			// Make sure there is exactly one argument.
			if len(v.Args) != 1 {
				return raw, nil
			}
			// Ensure the argument is a duration.
			lit, ok := v.Args[0].(*influxql.DurationLiteral)
			if !ok {
				return raw, nil
			}
			qc.GroupBy.Time = lit.String()
		case *influxql.VarRef:
			qc.GroupBy.Tags = append(qc.GroupBy.Tags, v.Val)
		}
	}

	fields := map[string][]string{}
	for _, fld := range stmt.Fields {
		switch f := fld.Expr.(type) {
		default:
			return raw, nil
		case *influxql.Call:
			// only support certain query config functions
			if _, ok := supportedFuncs[f.Name]; !ok {
				return raw, nil
			}
			// Query configs only support single argument functions
			if len(f.Args) != 1 {
				return raw, nil
			}
			ref, ok := f.Args[0].(*influxql.VarRef)
			// query config only support fields in the function
			if !ok {
				return raw, nil
			}
			// We only support field strings
			if ref.Type != influxql.Unknown {
				return raw, nil
			}
			if call, ok := fields[ref.Val]; !ok {
				fields[ref.Val] = []string{f.Name}
			} else {
				fields[ref.Val] = append(call, f.Name)
			}
		case *influxql.VarRef:
			if f.Type != influxql.Unknown {
				return raw, nil
			}
			if _, ok := fields[f.Val]; !ok {
				fields[f.Val] = []string{}
			}
		}
	}

	for fld, funcs := range fields {
		qc.Fields = append(qc.Fields, chronograf.Field{
			Field: fld,
			Funcs: funcs,
		})
	}

	if stmt.Condition == nil {
		return qc, nil
	}

	reduced := influxql.Reduce(stmt.Condition, nil)
	logic, ok := isTagLogic(reduced)
	if !ok {
		return raw, nil
	}

	ops := map[string]bool{}
	for _, l := range logic {
		values, ok := qc.Tags[l.Tag]
		if !ok {
			values = []string{}
		}
		ops[l.Op] = true
		values = append(values, l.Value)
		qc.Tags[l.Tag] = values
	}

	if len(logic) > 0 {
		if len(ops) != 1 {
			return raw, nil
		}
		if _, ok := ops["=="]; ok {
			qc.AreTagsAccepted = true
		}
	}

	// If the condition has a time range we report back its duration
	if dur, ok := hasTimeRange(stmt.Condition); ok {
		if !itsDashboardTime {
			qc.Range = &chronograf.DurationRange{
				Lower: "now() - " + shortDur(dur),
			}
		} else {
			strings.Replace(influxQL, "now() - 15m", ":dashboardTime:", 1)
		}
	}

	return qc, nil
}

// tagFilter represents a single tag that is filtered by some condition
type tagFilter struct {
	Op    string
	Tag   string
	Value string
}

func isTime(exp influxql.Expr) bool {
	if p, ok := exp.(*influxql.ParenExpr); ok {
		return isTime(p.Expr)
	} else if ref, ok := exp.(*influxql.VarRef); ok && strings.ToLower(ref.Val) == "time" {
		return true
	}
	return false
}

func isNow(exp influxql.Expr) bool {
	if p, ok := exp.(*influxql.ParenExpr); ok {
		return isNow(p.Expr)
	} else if call, ok := exp.(*influxql.Call); ok && strings.ToLower(call.Name) == "now" && len(call.Args) == 0 {
		return true
	}
	return false
}

func isDuration(exp influxql.Expr) (time.Duration, bool) {
	switch e := exp.(type) {
	case *influxql.ParenExpr:
		return isDuration(e.Expr)
	case *influxql.DurationLiteral:
		return e.Val, true
	case *influxql.NumberLiteral, *influxql.IntegerLiteral, *influxql.TimeLiteral:
		return 0, false
	}
	return 0, false
}

func isPreviousTime(exp influxql.Expr) (time.Duration, bool) {
	if p, ok := exp.(*influxql.ParenExpr); ok {
		return isPreviousTime(p.Expr)
	} else if bin, ok := exp.(*influxql.BinaryExpr); ok {
		now := isNow(bin.LHS) || isNow(bin.RHS) // either side can be now
		op := bin.Op == influxql.SUB
		dur, hasDur := isDuration(bin.LHS)
		if !hasDur {
			dur, hasDur = isDuration(bin.RHS)
		}
		return dur, now && op && hasDur
	} else if isNow(exp) { // just comparing to now
		return 0, true
	}
	return 0, false
}

func isTimeRange(exp influxql.Expr) (time.Duration, bool) {
	if p, ok := exp.(*influxql.ParenExpr); ok {
		return isTimeRange(p.Expr)
	} else if bin, ok := exp.(*influxql.BinaryExpr); ok {
		tm := isTime(bin.LHS) || isTime(bin.RHS) // Either side could be time
		op := false
		switch bin.Op {
		case influxql.LT, influxql.LTE, influxql.GT, influxql.GTE:
			op = true
		}
		dur, prev := isPreviousTime(bin.LHS)
		if !prev {
			dur, prev = isPreviousTime(bin.RHS)
		}
		return dur, tm && op && prev
	}
	return 0, false
}

func hasTimeRange(exp influxql.Expr) (time.Duration, bool) {
	if p, ok := exp.(*influxql.ParenExpr); ok {
		return hasTimeRange(p.Expr)
	} else if dur, ok := isTimeRange(exp); ok {
		return dur, true
	} else if bin, ok := exp.(*influxql.BinaryExpr); ok {
		dur, ok := isTimeRange(bin.LHS)
		if !ok {
			dur, ok = isTimeRange(bin.RHS)
		}
		return dur, ok
	}
	return 0, false
}

func isTagLogic(exp influxql.Expr) ([]tagFilter, bool) {
	if p, ok := exp.(*influxql.ParenExpr); ok {
		return isTagLogic(p.Expr)
	}

	if _, ok := isTimeRange(exp); ok {
		return nil, true
	} else if tf, ok := isTagFilter(exp); ok {
		return []tagFilter{tf}, true
	}

	bin, ok := exp.(*influxql.BinaryExpr)
	if !ok {
		return nil, false
	}

	lhs, lhsOK := isTagFilter(bin.LHS)
	rhs, rhsOK := isTagFilter(bin.RHS)

	if lhsOK && rhsOK && lhs.Tag == rhs.Tag && lhs.Op == rhs.Op && bin.Op == influxql.OR {
		return []tagFilter{lhs, rhs}, true
	}

	if bin.Op != influxql.AND {
		return nil, false
	}

	_, tm := isTimeRange(bin.LHS)
	if !tm {
		_, tm = isTimeRange(bin.RHS)
	}
	tf := lhsOK || rhsOK
	if tm && tf {
		if lhsOK {
			return []tagFilter{lhs}, true
		}
		return []tagFilter{rhs}, true
	}

	tlLHS, lhsOK := isTagLogic(bin.LHS)
	tlRHS, rhsOK := isTagLogic(bin.RHS)
	if lhsOK && rhsOK {
		ops := map[string]bool{} // there must only be one kind of ops
		for _, tf := range tlLHS {
			ops[tf.Op] = true
		}
		for _, tf := range tlRHS {
			ops[tf.Op] = true
		}
		if len(ops) > 1 {
			return nil, false
		}
		return append(tlLHS, tlRHS...), true
	}
	return nil, false
}

func isVarRef(exp influxql.Expr) bool {
	if p, ok := exp.(*influxql.ParenExpr); ok {
		return isVarRef(p.Expr)
	} else if _, ok := exp.(*influxql.VarRef); ok {
		return true
	}
	return false
}

func isString(exp influxql.Expr) bool {
	if p, ok := exp.(*influxql.ParenExpr); ok {
		return isString(p.Expr)
	} else if _, ok := exp.(*influxql.StringLiteral); ok {
		return true
	}
	return false
}

func isTagFilter(exp influxql.Expr) (tagFilter, bool) {
	switch expr := exp.(type) {
	default:
		return tagFilter{}, false
	case *influxql.ParenExpr:
		return isTagFilter(expr.Expr)
	case *influxql.BinaryExpr:
		var Op string
		if expr.Op == influxql.EQ {
			Op = "=="
		} else if expr.Op == influxql.NEQ {
			Op = "!="
		} else {
			return tagFilter{}, false
		}

		hasValue := isString(expr.LHS) || isString(expr.RHS)
		hasTag := isVarRef(expr.LHS) || isVarRef(expr.RHS)
		if !(hasValue && hasTag) {
			return tagFilter{}, false
		}

		value := ""
		tag := ""
		// Either tag op value or value op tag
		if isVarRef(expr.LHS) {
			t, _ := expr.LHS.(*influxql.VarRef)
			tag = t.Val
			v, _ := expr.RHS.(*influxql.StringLiteral)
			value = v.Val
		} else {
			t, _ := expr.RHS.(*influxql.VarRef)
			tag = t.Val
			v, _ := expr.LHS.(*influxql.StringLiteral)
			value = v.Val
		}

		return tagFilter{
			Op:    Op,
			Tag:   tag,
			Value: value,
		}, true
	}
}

var supportedFuncs = map[string]bool{
	"mean":   true,
	"median": true,
	"count":  true,
	"min":    true,
	"max":    true,
	"sum":    true,
	"first":  true,
	"last":   true,
	"spread": true,
	"stddev": true,
}

// shortDur converts duration into the queryConfig duration format
func shortDur(d time.Duration) string {
	s := d.String()
	if strings.HasSuffix(s, "m0s") {
		s = s[:len(s)-2]
	}
	if strings.HasSuffix(s, "h0m") {
		s = s[:len(s)-2]
	}
	return s
}
<|endoftext|>


<commit_before>/*
 * Copyright 2016 Robin Engel
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package context

import (
	"net/url"
	"net/http"
	"log"
	"bufio"
)

type Node struct {
	Name string
	URL  url.URL
}

type nodeCache struct {
	nodes map[string]*Node
}

func NewNodeCache() (*nodeCache, error) {
	c := &nodeCache{}
	c.nodes = make(map[string]*Node)
	return c, nil
}

func (cache *nodeCache) Add(node *Node) {
	cache.nodes[node.Name] = node

	go func() {
		Connection: for {
			resp, err := http.Get(node.URL.String())

			if err != nil {
				log.Fatal(err)
			}

			reader := bufio.NewReader(resp.Body)

			for {
				line, err := reader.ReadBytes('\n')

				if err != nil {
					//check weather we are dealing with a non-stream resource
					if err.Error() != "EOF" {
						log.Println(err)
					}

					resp.Body.Close()
					continue Connection
				}

				log.Print(string(line))
			}

		}
	}()
}

func (cache *nodeCache) Get(name string) *Node {
	return cache.nodes[name]
}<commit_msg>fix nullpointer and make logging better<commit_after>/*
 * Copyright 2016 Robin Engel
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package context

import (
	"net/url"
	"net/http"
	"log"
	"bufio"
	"time"
)

type Node struct {
	Name string
	URL  url.URL
}

type nodeCache struct {
	nodes map[string]*Node
}

func NewNodeCache() (*nodeCache, error) {
	c := &nodeCache{}
	c.nodes = make(map[string]*Node)
	return c, nil
}

func (cache *nodeCache) Add(node *Node) {
	cache.nodes[node.Name] = node

	go func() {
		Connection: for {
			resp, err := http.Get(node.URL.String())

			if err != nil {
				log.Println(err)

				//todo make reconnection timeout configurable by moseld.conf
				time.Sleep(10 * time.Second)
				continue Connection
			}

			reader := bufio.NewReader(resp.Body)

			for {
				line, err := reader.ReadBytes('\n')

				if err != nil {
					//check weather we are dealing with a non-stream resource
					if err.Error() != "EOF" {
						log.Println(err)
					}

					resp.Body.Close()
					continue Connection
				}

				log.Print(string(line))
			}

		}
	}()
}

func (cache *nodeCache) Get(name string) *Node {
	return cache.nodes[name]
}<|endoftext|>


<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package minunitsworker_test

import (
	stdtesting "testing"
	"time"

	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/juju/testing"
	statetesting "launchpad.net/juju-core/state/testing"
	coretesting "launchpad.net/juju-core/testing"
	"launchpad.net/juju-core/worker"
	"launchpad.net/juju-core/worker/minunitsworker"
)

func TestPackage(t *stdtesting.T) {
	coretesting.MgoTestPackage(t)
}

type minUnitsWorkerSuite struct {
	testing.JujuConnSuite
}

var _ = gc.Suite(&minUnitsWorkerSuite{})

var _ worker.Worker = (*minunitsworker.MinUnitsWorker)(nil)

func (s *minUnitsWorkerSuite) TestMinUnitsWorker(c *gc.C) {
	mu := minunitsworker.NewMinUnitsWorker(s.State)
	defer statetesting.AssertStop(c, mu)

	// Set up services and units for later use.
	wordpress, err := s.State.AddService("wordpress", s.AddTestingCharm(c, "wordpress"))
	c.Assert(err, gc.IsNil)
	mysql, err := s.State.AddService("mysql", s.AddTestingCharm(c, "mysql"))
	c.Assert(err, gc.IsNil)
	unit, err := wordpress.AddUnit()
	c.Assert(err, gc.IsNil)
	_, err = wordpress.AddUnit()
	c.Assert(err, gc.IsNil)

	// Observe minimum units with a watcher.
	w := s.State.WatchMinUnits()
	defer statetesting.AssertStop(c, w)

	// Set up minimum units for services.
	err = wordpress.SetMinUnits(3)
	c.Assert(err, gc.IsNil)
	err = mysql.SetMinUnits(2)
	c.Assert(err, gc.IsNil)

	// Remove a unit for a service.
	err = unit.Destroy()
	c.Assert(err, gc.IsNil)

	timeout := time.After(coretesting.LongWait)
	for {
		s.State.StartSync()
		select {
		case <-time.After(coretesting.ShortWait):
			wordpressUnits, err := wordpress.AllUnits()
			c.Assert(err, gc.IsNil)
			mysqlUnits, err := mysql.AllUnits()
			c.Assert(err, gc.IsNil)
			if len(wordpressUnits) == 3 && len(mysqlUnits) == 2 {
				return
			}
		case <-timeout:
			c.Fatalf("timed out waiting for minunits events")
		}
	}
}
<commit_msg>Changes as per review.<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package minunitsworker_test

import (
	stdtesting "testing"
	"time"

	gc "launchpad.net/gocheck"
	"launchpad.net/loggo"

	"launchpad.net/juju-core/juju/testing"
	statetesting "launchpad.net/juju-core/state/testing"
	coretesting "launchpad.net/juju-core/testing"
	"launchpad.net/juju-core/worker"
	"launchpad.net/juju-core/worker/minunitsworker"
)

var logger = loggo.GetLogger("juju.worker.minunitsworker_test")

func TestPackage(t *stdtesting.T) {
	coretesting.MgoTestPackage(t)
}

type minUnitsWorkerSuite struct {
	testing.JujuConnSuite
}

var _ = gc.Suite(&minUnitsWorkerSuite{})

var _ worker.Worker = (*minunitsworker.MinUnitsWorker)(nil)

func (s *minUnitsWorkerSuite) TestMinUnitsWorker(c *gc.C) {
	mu := minunitsworker.NewMinUnitsWorker(s.State)
	defer statetesting.AssertStop(c, mu)

	// Set up services and units for later use.
	wordpress, err := s.State.AddService("wordpress", s.AddTestingCharm(c, "wordpress"))
	c.Assert(err, gc.IsNil)
	mysql, err := s.State.AddService("mysql", s.AddTestingCharm(c, "mysql"))
	c.Assert(err, gc.IsNil)
	unit, err := wordpress.AddUnit()
	c.Assert(err, gc.IsNil)
	_, err = wordpress.AddUnit()
	c.Assert(err, gc.IsNil)

	// Set up minimum units for services.
	err = wordpress.SetMinUnits(3)
	c.Assert(err, gc.IsNil)
	err = mysql.SetMinUnits(2)
	c.Assert(err, gc.IsNil)

	// Remove a unit for a service.
	err = unit.Destroy()
	c.Assert(err, gc.IsNil)

	timeout := time.After(coretesting.LongWait)
	for {
		s.State.StartSync()
		select {
		case <-time.After(coretesting.ShortWait):
			wordpressUnits, err := wordpress.AllUnits()
			c.Assert(err, gc.IsNil)
			mysqlUnits, err := mysql.AllUnits()
			c.Assert(err, gc.IsNil)
			wordpressCount := len(wordpressUnits)
			mysqlCount := len(mysqlUnits)
			if wordpressCount == 3 && mysqlCount == 2 {
				return
			}
			logger.Infof("wordpress units: %d; mysql units: %d", wordpressCount, mysqlCount)
		case <-timeout:
			c.Fatalf("timed out waiting for minunits events")
		}
	}
}
<|endoftext|>