<commit_before>package main

import (
	"strings"
	"testing"

	cv "github.com/smartystreets/goconvey/convey"
)

// testing helper that panics on bad out
func toScheme(line string) string {
	anew := NewAccum()
	s, err := TranslateToScheme(line, anew)
	if err != nil {
		panic(err)
	}
	return strings.Join(s, "")
}

/*
func TestCanPrint(t *testing.T) {

	output := ""
	cv.Convey("Given a birdbrain repl", t, func() {

		cv.Convey("When we see a request to print 'hello bb'", func() {

			cv.Convey("then 'hello bb' should be printed", func() {
				cv.So(output, cv.ShouldEqual, "hello bb")
			})
		})
	})
}
*/

func TestConstantExpressions(t *testing.T) {

	cv.Convey("Given a birdbrain repl", t, func() {
		cv.Convey("When we evaluate numeric constants", func() {
			cv.Convey("they should stay the same", func() {
				cv.So(toScheme("1"), cv.ShouldEqual, "1")
				cv.So(toScheme("23456"), cv.ShouldEqual, "23456")
				cv.So(toScheme("1e40"), cv.ShouldEqual, "1e40")
				cv.So(toScheme("98764.321e12"), cv.ShouldEqual, "98764.321e12")
			})
		})
		cv.Convey("When we evaluate string constants", func() {
			cv.Convey("they should stay the same", func() {
				cv.So(toScheme("`abc`"), cv.ShouldEqual, `"abc"`)
				cv.So(toScheme(`"hello"`), cv.ShouldEqual, `"hello"`)
				cv.So(toScheme(`"I have spaces"`), cv.ShouldEqual, `"I have spaces"`)
				cv.So(toScheme(`"I have \"double\" quotes"`), cv.ShouldEqual, `"I have \"double\" quotes"`)
				cv.So(toScheme("`I have \nnewline`"), cv.ShouldEqual, `"I have \nnewline"`)
			})
			cv.Convey("raw strings should not have their back-ticks sent to scheme (since ` is macro defintion in scheme)", func() {
				cv.So(toScheme("a := `abc he\nya\"`"), cv.ShouldEqual, `(define a "abc he\nya\"")`)
			})

		})

		cv.Convey("When we evaluate //scm: comments", func() {
			cv.Convey("they should turn into pass-through scheme code", func() {
				cv.So(toScheme(`//scm:(write "hello")`), cv.ShouldEqual, `(write "hello")`)
				cv.So(toScheme(`// just a comment`), cv.ShouldEqual, `;;// just a comment`)

			})
		})

		cv.Convey("When we evaluate boolean literals", func() {
			cv.Convey("true in golang should become #t in scheme.", func() {
				cv.So(toScheme("true"), cv.ShouldEqual, "#t")
			})
			cv.Convey("false in golang should become #f in scheme.", func() {
				cv.So(toScheme("false"), cv.ShouldEqual, "#f")
			})
		})
	})
}

func TestIntegerVariables(t *testing.T) {

	cv.Convey("Given a birdbrain repl", t, func() {
		cv.Convey("When we declare and assign an integer variable", func() {
			cv.Convey("then we should get a scheme define expression in return. ", func() {

				cv.So(toScheme("a := 23"), cv.ShouldEqual, "(define a 23)")
				cv.So(toScheme(`str := "twentythree"`), cv.ShouldEqual, `(define str "twentythree")`)
			})
		})
		cv.Convey("When we just ask for the name of a variable", func() {
			cv.Convey("then we should get a request for the value of that variable. ", func() {
				cv.So(toScheme("a"), cv.ShouldEqual, "a")
				cv.So(toScheme("myVar"), cv.ShouldEqual, "myVar")
			})
		})

		cv.Convey("When we assign more than one variable in parallel in the same := stmt, all should be assigned.", func() {
			cv.So(toScheme("a, b := 10, 12"), cv.ShouldEqual, "(define a 10)(define b 12)")
		})
	})
}

func TestBinop(t *testing.T) {

	cv.Convey("Given a birdbrain repl", t, func() {
		cv.Convey("When we use addition as in: a + b", func() {
			cv.Convey("then we should get the prefix notation (+ a b)", func() {
				cv.So(toScheme("2 + 5"), cv.ShouldEqual, "(+ 2 5)")
			})
		})
		cv.Convey("When we use binary-operations in general", func() {
			cv.Convey("then we should get the prefix notation.", func() {
				cv.So(toScheme("2 * 5"), cv.ShouldEqual, "(* 2 5)")
				cv.So(toScheme("2 / 5"), cv.ShouldEqual, "(quotient 2 5)")
				cv.So(toScheme("2 - 5"), cv.ShouldEqual, "(- 2 5)")
				cv.So(toScheme("5 % 2"), cv.ShouldEqual, "(remainder 5 2)")

				cv.So(toScheme("5 / 2"), cv.ShouldEqual, "(quotient 5 2)") // integer division
				cv.So(toScheme("1 << 3"), cv.ShouldEqual, "(arithmetic-shift 1 3)")
				cv.So(toScheme("32 >> 3"), cv.ShouldEqual, "(arithmetic-shift 32 (- 3))")
				cv.So(toScheme("32 == 3"), cv.ShouldEqual, "(equal? 32 3)")

				cv.So(toScheme("5 ^ 1"), cv.ShouldEqual, "(bitwise-xor 5 1)") // == 4
				cv.So(toScheme("4 | 1"), cv.ShouldEqual, "(bitwise-ior 4 1)") // == 5
				cv.So(toScheme("5 & 1"), cv.ShouldEqual, "(bitwise-and 5 1)") // == 1

				cv.So(toScheme("true && false"), cv.ShouldEqual, "(and #t #f)")
				cv.So(toScheme("true || false"), cv.ShouldEqual, "(or #t #f)")

				cv.So(toScheme("5 &^ 1"), cv.ShouldEqual, "(bitwise-and 5 (bitwise-not 1))") // == 4
				cv.So(toScheme("5 != 1"), cv.ShouldEqual, "(not (equal? 5 1))")

			})
		})

		cv.Convey("When we use unary-operations", func() {
			cv.Convey("then we should get the prefix notation.", func() {
				cv.So(toScheme("!false"), cv.ShouldEqual, "(not #f)")
				cv.So(toScheme("!true"), cv.ShouldEqual, "(not #t)")
				cv.So(toScheme("b := -a"), cv.ShouldEqual, "(define b (- a))")

				// ~5 isn't a legal golang expression, but ^5 means bitwise compliment:
				cv.So(toScheme("^5"), cv.ShouldEqual, "(bitwise-not-is-likely-wrong! 5)") // 4611686018427387898
				// -6 in goland, signed.
				// 18446744073709551610 in golang; this is the unsigned, full 64-bits minus 5 version
				// i.e. 2^64 == 18446744073709551616
				// but note that (bitwise-not 5) in scheme is:
				// 4611686018427387898 in scheme, 2 bits (4x) less, indicating scheme is using 62-bit fixnums.
			})
		})

		cv.Convey("When we use floating-point division, that is: a / b, for floating-point a or b", func() {
			cv.Convey("then we should get the prefix notation (/ a b).", func() {
				cv.So(toScheme("5.0 / 2.0"), cv.ShouldEqual, "(/ 5.0 2.0)") // floating-point
				cv.So(toScheme("a / b"), cv.ShouldEqual, "(/ a b)")         // floating-point
			})
		})

	})
}
<commit_msg>add a test for function definition<commit_after>package main

import (
	"strings"
	"testing"

	cv "github.com/smartystreets/goconvey/convey"
)

// testing helper that panics on bad out
func toScheme(line string) string {
	anew := NewAccum()
	s, err := TranslateToScheme(line, anew)
	if err != nil {
		panic(err)
	}
	return strings.Join(s, "")
}

/*
func TestCanPrint(t *testing.T) {

	output := ""
	cv.Convey("Given a birdbrain repl", t, func() {

		cv.Convey("When we see a request to print 'hello bb'", func() {

			cv.Convey("then 'hello bb' should be printed", func() {
				cv.So(output, cv.ShouldEqual, "hello bb")
			})
		})
	})
}
*/

func TestConstantExpressions(t *testing.T) {

	cv.Convey("Given a birdbrain repl", t, func() {
		cv.Convey("When we evaluate numeric constants", func() {
			cv.Convey("they should stay the same", func() {
				cv.So(toScheme("1"), cv.ShouldEqual, "1")
				cv.So(toScheme("23456"), cv.ShouldEqual, "23456")
				cv.So(toScheme("1e40"), cv.ShouldEqual, "1e40")
				cv.So(toScheme("98764.321e12"), cv.ShouldEqual, "98764.321e12")
			})
		})
		cv.Convey("When we evaluate string constants", func() {
			cv.Convey("they should stay the same", func() {
				cv.So(toScheme("`abc`"), cv.ShouldEqual, `"abc"`)
				cv.So(toScheme(`"hello"`), cv.ShouldEqual, `"hello"`)
				cv.So(toScheme(`"I have spaces"`), cv.ShouldEqual, `"I have spaces"`)
				cv.So(toScheme(`"I have \"double\" quotes"`), cv.ShouldEqual, `"I have \"double\" quotes"`)
				cv.So(toScheme("`I have \nnewline`"), cv.ShouldEqual, `"I have \nnewline"`)
			})
			cv.Convey("raw strings should not have their back-ticks sent to scheme (since ` is macro defintion in scheme)", func() {
				cv.So(toScheme("a := `abc he\nya\"`"), cv.ShouldEqual, `(define a "abc he\nya\"")`)
			})

		})

		cv.Convey("When we evaluate //scm: comments", func() {
			cv.Convey("they should turn into pass-through scheme code", func() {
				cv.So(toScheme(`//scm:(write "hello")`), cv.ShouldEqual, `(write "hello")`)
				cv.So(toScheme(`// just a comment`), cv.ShouldEqual, `;;// just a comment`)

			})
		})

		cv.Convey("When we evaluate boolean literals", func() {
			cv.Convey("true in golang should become #t in scheme.", func() {
				cv.So(toScheme("true"), cv.ShouldEqual, "#t")
			})
			cv.Convey("false in golang should become #f in scheme.", func() {
				cv.So(toScheme("false"), cv.ShouldEqual, "#f")
			})
		})
	})
}

func TestIntegerVariables(t *testing.T) {

	cv.Convey("Given a birdbrain repl", t, func() {
		cv.Convey("When we declare and assign an integer variable", func() {
			cv.Convey("then we should get a scheme define expression in return. ", func() {

				cv.So(toScheme("a := 23"), cv.ShouldEqual, "(define a 23)")
				cv.So(toScheme(`str := "twentythree"`), cv.ShouldEqual, `(define str "twentythree")`)
			})
		})
		cv.Convey("When we just ask for the name of a variable", func() {
			cv.Convey("then we should get a request for the value of that variable. ", func() {
				cv.So(toScheme("a"), cv.ShouldEqual, "a")
				cv.So(toScheme("myVar"), cv.ShouldEqual, "myVar")
			})
		})

		cv.Convey("When we assign more than one variable in parallel in the same := stmt, all should be assigned.", func() {
			cv.So(toScheme("a, b := 10, 12"), cv.ShouldEqual, "(define a 10)(define b 12)")
		})
	})
}

func TestBinop(t *testing.T) {

	cv.Convey("Given a birdbrain repl", t, func() {
		cv.Convey("When we use addition as in: a + b", func() {
			cv.Convey("then we should get the prefix notation (+ a b)", func() {
				cv.So(toScheme("2 + 5"), cv.ShouldEqual, "(+ 2 5)")
			})
		})
		cv.Convey("When we use binary-operations in general", func() {
			cv.Convey("then we should get the prefix notation.", func() {
				cv.So(toScheme("2 * 5"), cv.ShouldEqual, "(* 2 5)")
				cv.So(toScheme("2 / 5"), cv.ShouldEqual, "(quotient 2 5)")
				cv.So(toScheme("2 - 5"), cv.ShouldEqual, "(- 2 5)")
				cv.So(toScheme("5 % 2"), cv.ShouldEqual, "(remainder 5 2)")

				cv.So(toScheme("5 / 2"), cv.ShouldEqual, "(quotient 5 2)") // integer division
				cv.So(toScheme("1 << 3"), cv.ShouldEqual, "(arithmetic-shift 1 3)")
				cv.So(toScheme("32 >> 3"), cv.ShouldEqual, "(arithmetic-shift 32 (- 3))")
				cv.So(toScheme("32 == 3"), cv.ShouldEqual, "(equal? 32 3)")

				cv.So(toScheme("5 ^ 1"), cv.ShouldEqual, "(bitwise-xor 5 1)") // == 4
				cv.So(toScheme("4 | 1"), cv.ShouldEqual, "(bitwise-ior 4 1)") // == 5
				cv.So(toScheme("5 & 1"), cv.ShouldEqual, "(bitwise-and 5 1)") // == 1

				cv.So(toScheme("true && false"), cv.ShouldEqual, "(and #t #f)")
				cv.So(toScheme("true || false"), cv.ShouldEqual, "(or #t #f)")

				cv.So(toScheme("5 &^ 1"), cv.ShouldEqual, "(bitwise-and 5 (bitwise-not 1))") // == 4
				cv.So(toScheme("5 != 1"), cv.ShouldEqual, "(not (equal? 5 1))")

			})
		})

		cv.Convey("When we use unary-operations", func() {
			cv.Convey("then we should get the prefix notation.", func() {
				cv.So(toScheme("!false"), cv.ShouldEqual, "(not #f)")
				cv.So(toScheme("!true"), cv.ShouldEqual, "(not #t)")
				cv.So(toScheme("b := -a"), cv.ShouldEqual, "(define b (- a))")

				// ~5 isn't a legal golang expression, but ^5 means bitwise compliment:
				cv.So(toScheme("^5"), cv.ShouldEqual, "(bitwise-not-is-likely-wrong! 5)") // 4611686018427387898
				// -6 in goland, signed.
				// 18446744073709551610 in golang; this is the unsigned, full 64-bits minus 5 version
				// i.e. 2^64 == 18446744073709551616
				// but note that (bitwise-not 5) in scheme is:
				// 4611686018427387898 in scheme, 2 bits (4x) less, indicating scheme is using 62-bit fixnums.
			})
		})

		cv.Convey("When we use floating-point division, that is: a / b, for floating-point a or b", func() {
			cv.Convey("then we should get the prefix notation (/ a b).", func() {
				cv.So(toScheme("5.0 / 2.0"), cv.ShouldEqual, "(/ 5.0 2.0)") // floating-point
				cv.So(toScheme("a / b"), cv.ShouldEqual, "(/ a b)")         // floating-point
			})
		})

		cv.Convey("When we define an increment function; func incr(x int) int { return x + 1 }", func() {
			cv.Convey("then we should get (define (incr x) (+ x 1)) in scheme.", func() {
				cv.So(toScheme("func incr(x int) int { return x + 1 }"), cv.ShouldEqual, "(define (incr x) (+ x 1))")
			})
		})

	})
}
<|endoftext|>


<commit_before>/*
 *    Copyright (C) 2014 Christian Muehlhaeuser
 *                  2014 Michael Wendland
 *
 *    This program is free software: you can redistribute it and/or modify
 *    it under the terms of the GNU Affero General Public License as published
 *    by the Free Software Foundation, either version 3 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *    Authors:
 *      Christian Muehlhaeuser <muesli@gmail.com>
 *      Michael Wendland <michiwend@michiwend.com>
 */

package main

import (
	"encoding/json"
	"io/ioutil"
	"log"

	"github.com/muesli/beehive/app"
	_ "github.com/muesli/beehive/filters"
	_ "github.com/muesli/beehive/filters/contains"
	_ "github.com/muesli/beehive/filters/endswith"
	_ "github.com/muesli/beehive/filters/startswith"

	"github.com/muesli/beehive/modules"
	//_ "github.com/muesli/beehive/modules/hellobee"
	_ "github.com/muesli/beehive/modules/anelpowerctrlbee"
	_ "github.com/muesli/beehive/modules/ircbee"
	_ "github.com/muesli/beehive/modules/jabberbee"
	_ "github.com/muesli/beehive/modules/jenkinsbee"
	_ "github.com/muesli/beehive/modules/nagiosbee"
	_ "github.com/muesli/beehive/modules/notificationbee"
	_ "github.com/muesli/beehive/modules/rssbee"
	_ "github.com/muesli/beehive/modules/webbee"
	_ "github.com/muesli/beehive/modules/timebee"
	_ "github.com/muesli/beehive/modules/serialbee"
)

var (
	configFile = "./beehive.conf"
)

type Config struct {
	Bees   []modules.Bee
	Chains []modules.Chain
}

// Loads chains from config
func loadConfig() Config {
	config := Config{}

	j, err := ioutil.ReadFile(configFile)
	if err == nil {
		err = json.Unmarshal(j, &config)
	}

	if err != nil {
		log.Fatal(err)
	}

	return config
}

// Saves chains to config
func saveConfig(c Config) {
	j, err := json.MarshalIndent(c, "", "  ")
	if err == nil {
		err = ioutil.WriteFile(configFile, j, 0644)
	}

	if err != nil {
		log.Fatal(err)
	}
}

func main() {
	// Parse command-line args for all registered modules
	app.Run()

	log.Println()
	log.Println("Beehive is buzzing...")

	config := loadConfig()

	// Initialize modules
	modules.StartModules(config.Bees)
	// Load chains from config
	modules.SetChains(config.Chains)

	// Keep app alive
	ch := make(chan bool)
	<-ch

	// Save chains to config
	config.Chains = modules.Chains()
	saveConfig(config)
}
<commit_msg>* Enable spaceapibee.<commit_after>/*
 *    Copyright (C) 2014 Christian Muehlhaeuser
 *                  2014 Michael Wendland
 *
 *    This program is free software: you can redistribute it and/or modify
 *    it under the terms of the GNU Affero General Public License as published
 *    by the Free Software Foundation, either version 3 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *    Authors:
 *      Christian Muehlhaeuser <muesli@gmail.com>
 *      Michael Wendland <michiwend@michiwend.com>
 */

package main

import (
	"encoding/json"
	"io/ioutil"
	"log"

	"github.com/muesli/beehive/app"
	_ "github.com/muesli/beehive/filters"
	_ "github.com/muesli/beehive/filters/contains"
	_ "github.com/muesli/beehive/filters/endswith"
	_ "github.com/muesli/beehive/filters/startswith"

	"github.com/muesli/beehive/modules"
	//_ "github.com/muesli/beehive/modules/hellobee"
	_ "github.com/muesli/beehive/modules/anelpowerctrlbee"
	_ "github.com/muesli/beehive/modules/ircbee"
	_ "github.com/muesli/beehive/modules/jabberbee"
	_ "github.com/muesli/beehive/modules/jenkinsbee"
	_ "github.com/muesli/beehive/modules/nagiosbee"
	_ "github.com/muesli/beehive/modules/notificationbee"
	_ "github.com/muesli/beehive/modules/rssbee"
	_ "github.com/muesli/beehive/modules/webbee"
	_ "github.com/muesli/beehive/modules/timebee"
	_ "github.com/muesli/beehive/modules/serialbee"
	_ "github.com/muesli/beehive/modules/spaceapibee"
)

var (
	configFile = "./beehive.conf"
)

type Config struct {
	Bees   []modules.Bee
	Chains []modules.Chain
}

// Loads chains from config
func loadConfig() Config {
	config := Config{}

	j, err := ioutil.ReadFile(configFile)
	if err == nil {
		err = json.Unmarshal(j, &config)
	}

	if err != nil {
		log.Fatal(err)
	}

	return config
}

// Saves chains to config
func saveConfig(c Config) {
	j, err := json.MarshalIndent(c, "", "  ")
	if err == nil {
		err = ioutil.WriteFile(configFile, j, 0644)
	}

	if err != nil {
		log.Fatal(err)
	}
}

func main() {
	// Parse command-line args for all registered modules
	app.Run()

	log.Println()
	log.Println("Beehive is buzzing...")

	config := loadConfig()

	// Initialize modules
	modules.StartModules(config.Bees)
	// Load chains from config
	modules.SetChains(config.Chains)

	// Keep app alive
	ch := make(chan bool)
	<-ch

	// Save chains to config
	config.Chains = modules.Chains()
	saveConfig(config)
}
<|endoftext|>


<commit_before>package robotgo

/*
//#if defined(IS_MACOSX)
	#cgo darwin CFLAGS: -x objective-c  -Wno-deprecated-declarations -I/usr/local/opt/libpng/include -I/usr/local/opt/zlib/include
	#cgo darwin LDFLAGS: -framework Cocoa -framework OpenGL -framework IOKit -framework Carbon -framework CoreFoundation -L/usr/local/opt/libpng/lib -lpng -L/usr/local/opt/zlib/lib -lz
//#elif defined(USE_X11)
	#cgo linux CFLAGS:-I/usr/src
	#cgo linux LDFLAGS:-L/usr/src -lpng -lz -lX11 -lXtst -lm
//#endif
	#cgo windows LDFLAGS: -lgdi32 -luser32 -lpng -lz
//#include <AppKit/NSEvent.h>
#include "screen/goScreen.h"
#include "mouse/goMouse.h"
#include "key/goKey.h"
#include "bitmap/goBitmap.h"
//#include "event/goEvent.h"
//#include "window/goWindow.h"
*/
import "C"

import (
	. "fmt"
	"unsafe"
	// "runtime"
	// "syscall"
)

/*
      _______.  ______ .______       _______  _______ .__   __.
    /       | /      ||   _  \     |   ____||   ____||  \ |  |
   |   (----`|  ,----'|  |_)  |    |  |__   |  |__   |   \|  |
    \   \    |  |     |      /     |   __|  |   __|  |  . `  |
.----)   |   |  `----.|  |\  \----.|  |____ |  |____ |  |\   |
|_______/     \______|| _| `._____||_______||_______||__| \__|
*/

type Bit_map struct {
	ImageBuffer   *C.uint8_t
	Width         C.size_t
	Height        C.size_t
	Bytewidth     C.size_t
	BitsPerPixel  C.uint8_t
	BytesPerPixel C.uint8_t
}

func GetPixelColor(x, y int) string {
	cx := C.size_t(x)
	cy := C.size_t(y)
	color := C.aGetPixelColor(cx, cy)
	// color := C.aGetPixelColor(x, y)
	gcolor := C.GoString(color)
	defer C.free(unsafe.Pointer(color))
	return gcolor
}

func GetScreenSize() (C.size_t, C.size_t) {
	size := C.aGetScreenSize()
	// Println("...", size, size.width)
	return size.width, size.height
}

func GetXDisplayName() string {
	name := C.aGetXDisplayName()
	gname := C.GoString(name)
	defer C.free(unsafe.Pointer(name))
	return gname
}

func SetXDisplayName(name string) string {
	cname := C.CString(name)
	str := C.aSetXDisplayName(cname)
	gstr := C.GoString(str)
	return gstr
}

func CaptureScreen(args ...int) C.MMBitmapRef {
	var x C.size_t
	var y C.size_t
	var w C.size_t
	var h C.size_t
	Try(func() {
		x = C.size_t(args[0])
		y = C.size_t(args[1])
		w = C.size_t(args[2])
		h = C.size_t(args[3])
	}, func(e interface{}) {
		// Println("err:::", e)
		x = 0
		y = 0
		//Get screen size.
		var displaySize C.MMSize
		displaySize = C.getMainDisplaySize()
		w = displaySize.width
		h = displaySize.height
	})

	bit := C.aCaptureScreen(x, y, w, h)
	// Println("...", bit.width)
	return bit
}

func Capture_Screen(x, y, w, h C.size_t) Bit_map {
	bit := C.aCaptureScreen(x, y, w, h)
	// Println("...", bit)
	bit_map := Bit_map{
		ImageBuffer:   bit.imageBuffer,
		Width:         bit.width,
		Height:        bit.height,
		Bytewidth:     bit.bytewidth,
		BitsPerPixel:  bit.bitsPerPixel,
		BytesPerPixel: bit.bytesPerPixel,
	}

	return bit_map
}

/*
 __  __
|  \/  | ___  _   _ ___  ___
| |\/| |/ _ \| | | / __|/ _ \
| |  | | (_) | |_| \__ \  __/
|_|  |_|\___/ \__,_|___/\___|

*/

type MPoint struct {
	x int
	y int
}

//C.size_t  int
func MoveMouse(x, y int) {
	cx := C.size_t(x)
	cy := C.size_t(y)
	C.aMoveMouse(cx, cy)
}

func DragMouse(x, y int) {
	cx := C.size_t(x)
	cy := C.size_t(y)
	C.aDragMouse(cx, cy)
}

func MoveMouseSmooth(x, y int) {
	cx := C.size_t(x)
	cy := C.size_t(y)
	C.aMoveMouseSmooth(cx, cy)
}

func GetMousePos() (int, int) {
	pos := C.aGetMousePos()
	// Println("pos:###", pos, pos.x, pos.y)
	x := int(pos.x)
	y := int(pos.y)
	// return pos.x, pos.y
	return x, y
}

func MouseClick() {
	C.aMouseClick()
}

func MouseToggle(args ...interface{}) {
	var button C.MMMouseButton
	Try(func() {
		button = args[1].(C.MMMouseButton)
	}, func(e interface{}) {
		// Println("err:::", e)
		button = C.LEFT_BUTTON
	})
	down := C.CString(args[0].(string))
	C.aMouseToggle(down, button)
}

func SetMouseDelay(x int) {
	cx := C.size_t(x)
	C.aSetMouseDelay(cx)
}

func ScrollMouse(x int, y string) {
	cx := C.size_t(x)
	z := C.CString(y)
	C.aScrollMouse(cx, z)
	defer C.free(unsafe.Pointer(z))
}

/*
 _  __          _                         _
| |/ /___ _   _| |__   ___   __ _ _ __ __| |
| ' // _ \ | | | '_ \ / _ \ / _` | '__/ _` |
| . \  __/ |_| | |_) | (_) | (_| | | | (_| |
|_|\_\___|\__, |_.__/ \___/ \__,_|_|  \__,_|
		  |___/
*/
func Try(fun func(), handler func(interface{})) {
	defer func() {
		if err := recover(); err != nil {
			handler(err)
		}
	}()
	fun()
}

func KeyTap(args ...string) {
	var apara string
	Try(func() {
		apara = args[1]
	}, func(e interface{}) {
		// Println("err:::", e)
		apara = "null"
	})

	zkey := C.CString(args[0])
	amod := C.CString(apara)
	// defer func() {
	C.aKeyTap(zkey, amod)
	// }()

	defer C.free(unsafe.Pointer(zkey))
	defer C.free(unsafe.Pointer(amod))
}

func KeyToggle(args ...string) {
	var apara string
	Try(func() {
		apara = args[1]
	}, func(e interface{}) {
		// Println("err:::", e)
		apara = "null"
	})

	zkey := C.CString(args[0])
	amod := C.CString(apara)
	// defer func() {
	str := C.aKeyToggle(zkey, amod)
	Println(str)
	// }()
	defer C.free(unsafe.Pointer(zkey))
	defer C.free(unsafe.Pointer(amod))
}

func TypeString(x string) {
	cx := C.CString(x)
	C.aTypeString(cx)
	defer C.free(unsafe.Pointer(cx))
}

func TypeStringDelayed(x string, y C.size_t) {
	cx := C.CString(x)
	C.aTypeStringDelayed(cx, y)
	defer C.free(unsafe.Pointer(cx))
}

func SetKeyboardDelay(x C.size_t) {
	C.aSetKeyboardDelay(x)
}

/*
.______    __  .___________..___  ___.      ___      .______
|   _  \  |  | |           ||   \/   |     /   \     |   _  \
|  |_)  | |  | `---|  |----`|  \  /  |    /  ^  \    |  |_)  |
|   _  <  |  |     |  |     |  |\/|  |   /  /_\  \   |   ___/
|  |_)  | |  |     |  |     |  |  |  |  /  _____  \  |  |
|______/  |__|     |__|     |__|  |__| /__/     \__\ | _|
*/
func FindBitmap(args ...interface{}) (C.size_t, C.size_t) {
	var bit C.MMBitmapRef
	bit = args[0].(C.MMBitmapRef)

	var rect C.MMRect
	Try(func() {
		rect.origin.x = C.size_t(args[1].(int))
		rect.origin.y = C.size_t(args[2].(int))
		rect.size.width = C.size_t(args[3].(int))
		rect.size.height = C.size_t(args[4].(int))
	}, func(e interface{}) {
		Println("err:::", e)
		// rect.origin.x = x
		// rect.origin.y = y
		// rect.size.width = w
		// rect.size.height = h
	})

	pos := C.aFindBitmap(bit, rect)
	// Println("pos----", pos)
	return pos.x, pos.y
}

func OpenBitmap(gpath string) C.MMBitmapRef {
	path := C.CString(gpath)
	bit := C.aOpenBitmap(path)
	// Println("opening...", bit)
	return bit
	// defer C.free(unsafe.Pointer(path))
}

func SaveBitmap(args ...interface{}) {
	var mtype C.uint16_t
	Try(func() {
		mtype = C.uint16_t(args[2].(int))
	}, func(e interface{}) {
		// Println("err:::", e)
		mtype = 1
	})

	path := C.CString(args[1].(string))
	savebit := C.aSaveBitmap(args[0].(C.MMBitmapRef), path, mtype)
	Println("saved...", savebit)
	// return bit
	// defer C.free(unsafe.Pointer(path))
}

// func SaveBitmap(bit C.MMBitmapRef, gpath string, mtype C.MMImageType) {
// 	path := C.CString(gpath)
// 	savebit := C.aSaveBitmap(bit, path, mtype)
// 	Println("opening...", savebit)
// 	// return bit
// 	// defer C.free(unsafe.Pointer(path))
// }

func TostringBitmap(bit C.MMBitmapRef) *C.char {
	str_bit := C.aTostringBitmap(bit)
	// Println("...", str_bit)
	return str_bit
}

func GetPortion(bit C.MMBitmapRef, x, y, w, h C.size_t) C.MMBitmapRef {
	var rect C.MMRect
	rect.origin.x = x
	rect.origin.y = y
	rect.size.width = w
	rect.size.height = h

	pos := C.aGetPortion(bit, rect)
	return pos
}

/*
------------ ---    ---  ------------ ----    ---- ------------
************ ***    ***  ************ *****   **** ************
----         ---    ---  ----         ------  ---- ------------
************ ***    ***  ************ ************     ****
------------ ---    ---  ------------ ------------     ----
****          ********   ****         ****  ******     ****
------------   ------    ------------ ----   -----     ----
************    ****     ************ ****    ****     ****

*/

/*
____    __    ____  __  .__   __.  _______   ______   ____    __    ____
\   \  /  \  /   / |  | |  \ |  | |       \ /  __  \  \   \  /  \  /   /
 \   \/    \/   /  |  | |   \|  | |  .--.  |  |  |  |  \   \/    \/   /
  \            /   |  | |  . `  | |  |  |  |  |  |  |   \            /
   \    /\    /    |  | |  |\   | |  '--'  |  `--'  |    \    /\    /
    \__/  \__/     |__| |__| \__| |_______/ \______/      \__/  \__/

*/
<commit_msg>Update ascii<commit_after>package robotgo

/*
//#if defined(IS_MACOSX)
	#cgo darwin CFLAGS: -x objective-c  -Wno-deprecated-declarations -I/usr/local/opt/libpng/include -I/usr/local/opt/zlib/include
	#cgo darwin LDFLAGS: -framework Cocoa -framework OpenGL -framework IOKit -framework Carbon -framework CoreFoundation -L/usr/local/opt/libpng/lib -lpng -L/usr/local/opt/zlib/lib -lz
//#elif defined(USE_X11)
	#cgo linux CFLAGS:-I/usr/src
	#cgo linux LDFLAGS:-L/usr/src -lpng -lz -lX11 -lXtst -lm
//#endif
	#cgo windows LDFLAGS: -lgdi32 -luser32 -lpng -lz
//#include <AppKit/NSEvent.h>
#include "screen/goScreen.h"
#include "mouse/goMouse.h"
#include "key/goKey.h"
#include "bitmap/goBitmap.h"
//#include "event/goEvent.h"
//#include "window/goWindow.h"
*/
import "C"

import (
	. "fmt"
	"unsafe"
	// "runtime"
	// "syscall"
)

/*
      _______.  ______ .______       _______  _______ .__   __.
    /       | /      ||   _  \     |   ____||   ____||  \ |  |
   |   (----`|  ,----'|  |_)  |    |  |__   |  |__   |   \|  |
    \   \    |  |     |      /     |   __|  |   __|  |  . `  |
.----)   |   |  `----.|  |\  \----.|  |____ |  |____ |  |\   |
|_______/     \______|| _| `._____||_______||_______||__| \__|
*/

type Bit_map struct {
	ImageBuffer   *C.uint8_t
	Width         C.size_t
	Height        C.size_t
	Bytewidth     C.size_t
	BitsPerPixel  C.uint8_t
	BytesPerPixel C.uint8_t
}

func GetPixelColor(x, y int) string {
	cx := C.size_t(x)
	cy := C.size_t(y)
	color := C.aGetPixelColor(cx, cy)
	// color := C.aGetPixelColor(x, y)
	gcolor := C.GoString(color)
	defer C.free(unsafe.Pointer(color))
	return gcolor
}

func GetScreenSize() (C.size_t, C.size_t) {
	size := C.aGetScreenSize()
	// Println("...", size, size.width)
	return size.width, size.height
}

func GetXDisplayName() string {
	name := C.aGetXDisplayName()
	gname := C.GoString(name)
	defer C.free(unsafe.Pointer(name))
	return gname
}

func SetXDisplayName(name string) string {
	cname := C.CString(name)
	str := C.aSetXDisplayName(cname)
	gstr := C.GoString(str)
	return gstr
}

func CaptureScreen(args ...int) C.MMBitmapRef {
	var x C.size_t
	var y C.size_t
	var w C.size_t
	var h C.size_t
	Try(func() {
		x = C.size_t(args[0])
		y = C.size_t(args[1])
		w = C.size_t(args[2])
		h = C.size_t(args[3])
	}, func(e interface{}) {
		// Println("err:::", e)
		x = 0
		y = 0
		//Get screen size.
		var displaySize C.MMSize
		displaySize = C.getMainDisplaySize()
		w = displaySize.width
		h = displaySize.height
	})

	bit := C.aCaptureScreen(x, y, w, h)
	// Println("...", bit.width)
	return bit
}

func Capture_Screen(x, y, w, h C.size_t) Bit_map {
	bit := C.aCaptureScreen(x, y, w, h)
	// Println("...", bit)
	bit_map := Bit_map{
		ImageBuffer:   bit.imageBuffer,
		Width:         bit.width,
		Height:        bit.height,
		Bytewidth:     bit.bytewidth,
		BitsPerPixel:  bit.bitsPerPixel,
		BytesPerPixel: bit.bytesPerPixel,
	}

	return bit_map
}

/*
.___  ___.   ______    __    __       _______. _______
|   \/   |  /  __  \  |  |  |  |     /       ||   ____|
|  \  /  | |  |  |  | |  |  |  |    |   (----`|  |__
|  |\/|  | |  |  |  | |  |  |  |     \   \    |   __|
|  |  |  | |  `--'  | |  `--'  | .----)   |   |  |____
|__|  |__|  \______/   \______/  |_______/    |_______|

*/

type MPoint struct {
	x int
	y int
}

//C.size_t  int
func MoveMouse(x, y int) {
	cx := C.size_t(x)
	cy := C.size_t(y)
	C.aMoveMouse(cx, cy)
}

func DragMouse(x, y int) {
	cx := C.size_t(x)
	cy := C.size_t(y)
	C.aDragMouse(cx, cy)
}

func MoveMouseSmooth(x, y int) {
	cx := C.size_t(x)
	cy := C.size_t(y)
	C.aMoveMouseSmooth(cx, cy)
}

func GetMousePos() (int, int) {
	pos := C.aGetMousePos()
	// Println("pos:###", pos, pos.x, pos.y)
	x := int(pos.x)
	y := int(pos.y)
	// return pos.x, pos.y
	return x, y
}

func MouseClick() {
	C.aMouseClick()
}

func MouseToggle(args ...interface{}) {
	var button C.MMMouseButton
	Try(func() {
		button = args[1].(C.MMMouseButton)
	}, func(e interface{}) {
		// Println("err:::", e)
		button = C.LEFT_BUTTON
	})
	down := C.CString(args[0].(string))
	C.aMouseToggle(down, button)
}

func SetMouseDelay(x int) {
	cx := C.size_t(x)
	C.aSetMouseDelay(cx)
}

func ScrollMouse(x int, y string) {
	cx := C.size_t(x)
	z := C.CString(y)
	C.aScrollMouse(cx, z)
	defer C.free(unsafe.Pointer(z))
}

/*
 __  ___  ___________    ____ .______     ______        ___      .______       _______
|  |/  / |   ____\   \  /   / |   _  \   /  __  \      /   \     |   _  \     |       \
|  '  /  |  |__   \   \/   /  |  |_)  | |  |  |  |    /  ^  \    |  |_)  |    |  .--.  |
|    <   |   __|   \_    _/   |   _  <  |  |  |  |   /  /_\  \   |      /     |  |  |  |
|  .  \  |  |____    |  |     |  |_)  | |  `--'  |  /  _____  \  |  |\  \----.|  '--'  |
|__|\__\ |_______|   |__|     |______/   \______/  /__/     \__\ | _| `._____||_______/

*/
func Try(fun func(), handler func(interface{})) {
	defer func() {
		if err := recover(); err != nil {
			handler(err)
		}
	}()
	fun()
}

func KeyTap(args ...string) {
	var apara string
	Try(func() {
		apara = args[1]
	}, func(e interface{}) {
		// Println("err:::", e)
		apara = "null"
	})

	zkey := C.CString(args[0])
	amod := C.CString(apara)
	// defer func() {
	C.aKeyTap(zkey, amod)
	// }()

	defer C.free(unsafe.Pointer(zkey))
	defer C.free(unsafe.Pointer(amod))
}

func KeyToggle(args ...string) {
	var apara string
	Try(func() {
		apara = args[1]
	}, func(e interface{}) {
		// Println("err:::", e)
		apara = "null"
	})

	zkey := C.CString(args[0])
	amod := C.CString(apara)
	// defer func() {
	str := C.aKeyToggle(zkey, amod)
	Println(str)
	// }()
	defer C.free(unsafe.Pointer(zkey))
	defer C.free(unsafe.Pointer(amod))
}

func TypeString(x string) {
	cx := C.CString(x)
	C.aTypeString(cx)
	defer C.free(unsafe.Pointer(cx))
}

func TypeStringDelayed(x string, y C.size_t) {
	cx := C.CString(x)
	C.aTypeStringDelayed(cx, y)
	defer C.free(unsafe.Pointer(cx))
}

func SetKeyboardDelay(x C.size_t) {
	C.aSetKeyboardDelay(x)
}

/*
.______    __  .___________..___  ___.      ___      .______
|   _  \  |  | |           ||   \/   |     /   \     |   _  \
|  |_)  | |  | `---|  |----`|  \  /  |    /  ^  \    |  |_)  |
|   _  <  |  |     |  |     |  |\/|  |   /  /_\  \   |   ___/
|  |_)  | |  |     |  |     |  |  |  |  /  _____  \  |  |
|______/  |__|     |__|     |__|  |__| /__/     \__\ | _|
*/
func FindBitmap(args ...interface{}) (C.size_t, C.size_t) {
	var bit C.MMBitmapRef
	bit = args[0].(C.MMBitmapRef)

	var rect C.MMRect
	Try(func() {
		rect.origin.x = C.size_t(args[1].(int))
		rect.origin.y = C.size_t(args[2].(int))
		rect.size.width = C.size_t(args[3].(int))
		rect.size.height = C.size_t(args[4].(int))
	}, func(e interface{}) {
		Println("err:::", e)
		// rect.origin.x = x
		// rect.origin.y = y
		// rect.size.width = w
		// rect.size.height = h
	})

	pos := C.aFindBitmap(bit, rect)
	// Println("pos----", pos)
	return pos.x, pos.y
}

func OpenBitmap(gpath string) C.MMBitmapRef {
	path := C.CString(gpath)
	bit := C.aOpenBitmap(path)
	// Println("opening...", bit)
	return bit
	// defer C.free(unsafe.Pointer(path))
}

func SaveBitmap(args ...interface{}) {
	var mtype C.uint16_t
	Try(func() {
		mtype = C.uint16_t(args[2].(int))
	}, func(e interface{}) {
		// Println("err:::", e)
		mtype = 1
	})

	path := C.CString(args[1].(string))
	savebit := C.aSaveBitmap(args[0].(C.MMBitmapRef), path, mtype)
	Println("saved...", savebit)
	// return bit
	// defer C.free(unsafe.Pointer(path))
}

// func SaveBitmap(bit C.MMBitmapRef, gpath string, mtype C.MMImageType) {
// 	path := C.CString(gpath)
// 	savebit := C.aSaveBitmap(bit, path, mtype)
// 	Println("opening...", savebit)
// 	// return bit
// 	// defer C.free(unsafe.Pointer(path))
// }

func TostringBitmap(bit C.MMBitmapRef) *C.char {
	str_bit := C.aTostringBitmap(bit)
	// Println("...", str_bit)
	return str_bit
}

func GetPortion(bit C.MMBitmapRef, x, y, w, h C.size_t) C.MMBitmapRef {
	var rect C.MMRect
	rect.origin.x = x
	rect.origin.y = y
	rect.size.width = w
	rect.size.height = h

	pos := C.aGetPortion(bit, rect)
	return pos
}

/*
------------ ---    ---  ------------ ----    ---- ------------
************ ***    ***  ************ *****   **** ************
----         ---    ---  ----         ------  ---- ------------
************ ***    ***  ************ ************     ****
------------ ---    ---  ------------ ------------     ----
****          ********   ****         ****  ******     ****
------------   ------    ------------ ----   -----     ----
************    ****     ************ ****    ****     ****

*/

/*
____    __    ____  __  .__   __.  _______   ______   ____    __    ____
\   \  /  \  /   / |  | |  \ |  | |       \ /  __  \  \   \  /  \  /   /
 \   \/    \/   /  |  | |   \|  | |  .--.  |  |  |  |  \   \/    \/   /
  \            /   |  | |  . `  | |  |  |  |  |  |  |   \            /
   \    /\    /    |  | |  |\   | |  '--'  |  `--'  |    \    /\    /
    \__/  \__/     |__| |__| \__| |_______/ \______/      \__/  \__/

*/
<|endoftext|>


<commit_before>package row

// TODO integrate this functionality into the go code.
// Probably should have Base implement Parser.

import (
	"context"
	"errors"
	"log"
	"sync"
	"time"

	"github.com/prometheus/client_golang/prometheus"

	"github.com/m-lab/annotation-service/api"
	v2as "github.com/m-lab/annotation-service/api/v2"
	"github.com/m-lab/go/logx"

	"github.com/m-lab/etl/metrics"
)

// Errors that may be returned by Buffer functions.
var (
	ErrAnnotationError = errors.New("Annotation error")
	ErrNotAnnotatable  = errors.New("object does not implement Annotatable")
	ErrBufferFull      = errors.New("Buffer full")
)

// Annotatable interface enables integration of annotation into parser.Base.
// The row type should implement the interface, and the annotations will be added
// prior to insertion.
type Annotatable interface {
	GetLogTime() time.Time
	GetClientIPs() []string // This is a slice to support mutliple hops in traceroute data.
	GetServerIP() string
	AnnotateClients(map[string]*api.Annotations) error // Must properly handle missing annotations.
	AnnotateServer(*api.Annotations) error             // Must properly handle nil parameter.
}

// Stats contains stats about buffer history.
type Stats struct {
	Buffered  int // rows buffered but not yet sent.
	Pending   int // pending counts previously buffered rows that are being committed.
	Committed int
	Failed    int
}

// Total returns the total number of rows handled.
func (s Stats) Total() int {
	return s.Buffered + s.Pending + s.Committed + s.Failed
}

// ActiveStats is a stats object that supports updates.
type ActiveStats struct {
	lock sync.RWMutex // Protects all Stats fields.
	Stats
}

// GetStats implements HasStats()
func (as *ActiveStats) GetStats() Stats {
	as.lock.RLock()
	defer as.lock.RUnlock()
	return as.Stats
}

// MoveToPending increments the Pending field.
func (as *ActiveStats) MoveToPending(n int) {
	as.lock.Lock()
	defer as.lock.Unlock()
	as.Buffered -= n
	if as.Buffered < 0 {
		log.Println("BROKEN - negative buffered")
	}
	as.Pending += n
}

// Inc increments the Buffered field
func (as *ActiveStats) Inc() {
	logx.Debug.Println("IncPending")
	as.lock.Lock()
	defer as.lock.Unlock()
	as.Buffered++
}

// Done updates the pending to failed or committed.
func (as *ActiveStats) Done(n int, err error) {
	as.lock.Lock()
	defer as.lock.Unlock()
	as.Pending -= n
	if as.Pending < 0 {
		log.Println("BROKEN: negative Pending")
	}
	if err != nil {
		as.Failed += n
	} else {
		as.Committed += n
	}
	logx.Debug.Printf("Done %d->%d %v\n", as.Pending+n, as.Pending, err)
}

// HasStats can provide stats
type HasStats interface {
	GetStats() Stats
}

// Sink defines the interface for committing rows.
// Returns the number of rows successfully committed, and error.
// Implementations should be threadsafe.
type Sink interface {
	Commit(rows []interface{}, label string) (int, error)
}

// Buffer provides all basic functionality generally needed for buffering, annotating, and inserting
// rows that implement Annotatable.
// Buffer functions are THREAD-SAFE
type Buffer struct {
	lock sync.Mutex
	size int // Number of rows before starting new buffer.
	rows []interface{}
}

// NewBuffer returns a new buffer of the desired size.
func NewBuffer(size int) *Buffer {
	return &Buffer{size: size, rows: make([]interface{}, 0, size)}
}

// Append appends a row to the buffer.
// If buffer is full, this returns the buffered rows, and saves provided row
// in new buffer.  Client MUST handle the returned rows.
func (buf *Buffer) Append(row interface{}) []interface{} {
	buf.lock.Lock()
	defer buf.lock.Unlock()
	if len(buf.rows) < buf.size {
		buf.rows = append(buf.rows, row)
		return nil
	}
	rows := buf.rows
	buf.rows = make([]interface{}, 0, buf.size)
	buf.rows = append(buf.rows, row)

	return rows
}

// Reset clears the buffer, returning all pending rows.
func (buf *Buffer) Reset() []interface{} {
	buf.lock.Lock()
	defer buf.lock.Unlock()
	res := buf.rows
	buf.rows = make([]interface{}, 0, buf.size)
	return res
}

type annotator struct {
	v2 v2as.Annotator
}

// label is used to label metrics and errors in GetAnnotations
func (ann *annotator) annotateServers(rows []interface{}, label string) error {
	serverIPs := make(map[string]struct{})
	logTime := time.Time{}
	for i := range rows {
		r, ok := rows[i].(Annotatable)
		if !ok {
			return ErrNotAnnotatable
		}

		// Only ask for the IP if it is non-empty.
		ip := r.GetServerIP()
		if ip != "" {
			serverIPs[ip] = struct{}{}
		}

		if (logTime == time.Time{}) {
			logTime = r.GetLogTime()
		}
	}

	ipSlice := make([]string, 0, len(rows))
	for ip := range serverIPs {
		ipSlice = append(ipSlice, ip)
	}
	if len(ipSlice) == 0 {
		return nil
	}
	response, err := ann.v2.GetAnnotations(context.Background(), logTime, ipSlice, label)
	if err != nil {
		log.Println("error in server GetAnnotations: ", err)
		metrics.AnnotationErrorCount.With(prometheus.
			Labels{"source": "Server IP: RPC err in GetAnnotations."}).Inc()
		return err
	}
	annMap := response.Annotations
	if annMap == nil {
		log.Println("empty server annotation response")
		metrics.AnnotationErrorCount.With(prometheus.
			Labels{"source": "Server IP: empty response"}).Inc()
		return ErrAnnotationError
	}

	for i := range rows {
		r, ok := rows[i].(Annotatable)
		if !ok {
			err = ErrNotAnnotatable
		} else {
			ip := r.GetServerIP()
			if ip != "" {
				ann, ok := annMap[ip]
				if ok {
					r.AnnotateServer(ann)
				}
			}
		}
	}

	return err
}

// label is used to label metrics and errors in GetAnnotations
func (ann *annotator) annotateClients(rows []interface{}, label string) error {
	ipSlice := make([]string, 0, 2*len(rows)) // This may be inadequate, but its a reasonable start.
	logTime := time.Time{}
	for i := range rows {
		r, ok := rows[i].(Annotatable)
		if !ok {
			return ErrNotAnnotatable
		}
		ipSlice = append(ipSlice, r.GetClientIPs()...)
		if (logTime == time.Time{}) {
			logTime = r.GetLogTime()
		}
	}

	response, err := ann.v2.GetAnnotations(context.Background(), logTime, ipSlice, label)
	if err != nil {
		log.Println("error in client GetAnnotations: ", err)
		metrics.AnnotationErrorCount.With(prometheus.
			Labels{"source": "Client IP: RPC err in GetAnnotations."}).Inc()
		return err
	}
	annMap := response.Annotations
	if annMap == nil {
		log.Println("empty client annotation response")
		metrics.AnnotationErrorCount.With(prometheus.
			Labels{"source": "Client IP: empty response"}).Inc()
		return ErrAnnotationError
	}

	for i := range rows {
		r, ok := rows[i].(Annotatable)
		if !ok {
			err = ErrNotAnnotatable
		} else {
			// Will not error because we check for nil annMap above.
			r.AnnotateClients(annMap)
		}
	}

	return err
}

// Annotate fetches and applies annotations for all rows
func (ann *annotator) Annotate(rows []interface{}, metricLabel string) error {
	metrics.WorkerState.WithLabelValues(metricLabel, "annotate").Inc()
	defer metrics.WorkerState.WithLabelValues(metricLabel, "annotate").Dec()
	if len(rows) == 0 {
		return nil
	}
	// TODO replace this with a histogram.
	defer func(label string, start time.Time) {
		metrics.AnnotationTimeSummary.With(prometheus.Labels{"test_type": label}).Observe(float64(time.Since(start).Nanoseconds()))
	}(metricLabel, time.Now())

	// TODO Consider doing these in parallel?
	clientErr := ann.annotateClients(rows, metricLabel)
	serverErr := ann.annotateServers(rows, metricLabel)

	if clientErr != nil {
		return clientErr
	}

	if serverErr != nil {
		return serverErr
	}

	return nil
}

// Base provides common parser functionality.
// Base is NOT THREAD-SAFE
type Base struct {
	sink  Sink
	ann   annotator
	buf   *Buffer
	label string // Used in metrics and errors.

	stats ActiveStats
}

// NewBase creates a new Base.  This will generally be embedded in a type specific parser.
func NewBase(label string, sink Sink, bufSize int, ann v2as.Annotator) *Base {
	buf := NewBuffer(bufSize)
	return &Base{sink: sink, ann: annotator{ann}, buf: buf, label: label}
}

// GetStats returns the buffer/sink stats.
func (pb *Base) GetStats() Stats {
	return pb.stats.GetStats()
}

// TaskError return the task level error, based on failed rows, or any other criteria.
func (pb *Base) TaskError() error {
	return nil
}

func (pb *Base) commit(rows []interface{}) error {
	// TODO - care about error?
	_ = pb.ann.Annotate(rows, pb.label)
	// TODO do we need these to be done in order.
	// This is synchronous, blocking, and thread safe.
	done, err := pb.sink.Commit(rows, pb.label)
	if done > 0 {
		pb.stats.Done(done, nil)
	}
	if err != nil {
		pb.stats.Done(len(rows)-done, err)
	}
	return err
}

// Flush synchronously flushes any pending rows.
func (pb *Base) Flush() error {
	rows := pb.buf.Reset()
	pb.stats.MoveToPending(len(rows))
	return pb.commit(rows)
}

// Put adds a row to the buffer.
// Iff the buffer is already full the prior buffered rows are
// annotated and committed to the Sink.
// NOTE: There is no guarantee about ordering of writes resulting from
// sequential calls to Put.  However, once a block of rows is submitted
// to pb.commit, it should be written in the same order to the Sink.
// TODO improve Annotatable architecture.
func (pb *Base) Put(row Annotatable) {
	log.Println("append")
	rows := pb.buf.Append(row)
	pb.stats.Inc()

	if rows != nil {
		log.Println("commit")
		pb.stats.MoveToPending(len(rows))
		// TODO consider making this asynchronous.
		err := pb.commit(rows)
		if err != nil {
			log.Println(err)
		}
	}
}
<commit_msg>delete spammy debug logs (#851)<commit_after>package row

// TODO integrate this functionality into the go code.
// Probably should have Base implement Parser.

import (
	"context"
	"errors"
	"log"
	"sync"
	"time"

	"github.com/prometheus/client_golang/prometheus"

	"github.com/m-lab/annotation-service/api"
	v2as "github.com/m-lab/annotation-service/api/v2"
	"github.com/m-lab/go/logx"

	"github.com/m-lab/etl/metrics"
)

// Errors that may be returned by Buffer functions.
var (
	ErrAnnotationError = errors.New("Annotation error")
	ErrNotAnnotatable  = errors.New("object does not implement Annotatable")
	ErrBufferFull      = errors.New("Buffer full")
)

// Annotatable interface enables integration of annotation into parser.Base.
// The row type should implement the interface, and the annotations will be added
// prior to insertion.
type Annotatable interface {
	GetLogTime() time.Time
	GetClientIPs() []string // This is a slice to support mutliple hops in traceroute data.
	GetServerIP() string
	AnnotateClients(map[string]*api.Annotations) error // Must properly handle missing annotations.
	AnnotateServer(*api.Annotations) error             // Must properly handle nil parameter.
}

// Stats contains stats about buffer history.
type Stats struct {
	Buffered  int // rows buffered but not yet sent.
	Pending   int // pending counts previously buffered rows that are being committed.
	Committed int
	Failed    int
}

// Total returns the total number of rows handled.
func (s Stats) Total() int {
	return s.Buffered + s.Pending + s.Committed + s.Failed
}

// ActiveStats is a stats object that supports updates.
type ActiveStats struct {
	lock sync.RWMutex // Protects all Stats fields.
	Stats
}

// GetStats implements HasStats()
func (as *ActiveStats) GetStats() Stats {
	as.lock.RLock()
	defer as.lock.RUnlock()
	return as.Stats
}

// MoveToPending increments the Pending field.
func (as *ActiveStats) MoveToPending(n int) {
	as.lock.Lock()
	defer as.lock.Unlock()
	as.Buffered -= n
	if as.Buffered < 0 {
		log.Println("BROKEN - negative buffered")
	}
	as.Pending += n
}

// Inc increments the Buffered field
func (as *ActiveStats) Inc() {
	logx.Debug.Println("IncPending")
	as.lock.Lock()
	defer as.lock.Unlock()
	as.Buffered++
}

// Done updates the pending to failed or committed.
func (as *ActiveStats) Done(n int, err error) {
	as.lock.Lock()
	defer as.lock.Unlock()
	as.Pending -= n
	if as.Pending < 0 {
		log.Println("BROKEN: negative Pending")
	}
	if err != nil {
		as.Failed += n
	} else {
		as.Committed += n
	}
	logx.Debug.Printf("Done %d->%d %v\n", as.Pending+n, as.Pending, err)
}

// HasStats can provide stats
type HasStats interface {
	GetStats() Stats
}

// Sink defines the interface for committing rows.
// Returns the number of rows successfully committed, and error.
// Implementations should be threadsafe.
type Sink interface {
	Commit(rows []interface{}, label string) (int, error)
}

// Buffer provides all basic functionality generally needed for buffering, annotating, and inserting
// rows that implement Annotatable.
// Buffer functions are THREAD-SAFE
type Buffer struct {
	lock sync.Mutex
	size int // Number of rows before starting new buffer.
	rows []interface{}
}

// NewBuffer returns a new buffer of the desired size.
func NewBuffer(size int) *Buffer {
	return &Buffer{size: size, rows: make([]interface{}, 0, size)}
}

// Append appends a row to the buffer.
// If buffer is full, this returns the buffered rows, and saves provided row
// in new buffer.  Client MUST handle the returned rows.
func (buf *Buffer) Append(row interface{}) []interface{} {
	buf.lock.Lock()
	defer buf.lock.Unlock()
	if len(buf.rows) < buf.size {
		buf.rows = append(buf.rows, row)
		return nil
	}
	rows := buf.rows
	buf.rows = make([]interface{}, 0, buf.size)
	buf.rows = append(buf.rows, row)

	return rows
}

// Reset clears the buffer, returning all pending rows.
func (buf *Buffer) Reset() []interface{} {
	buf.lock.Lock()
	defer buf.lock.Unlock()
	res := buf.rows
	buf.rows = make([]interface{}, 0, buf.size)
	return res
}

type annotator struct {
	v2 v2as.Annotator
}

// label is used to label metrics and errors in GetAnnotations
func (ann *annotator) annotateServers(rows []interface{}, label string) error {
	serverIPs := make(map[string]struct{})
	logTime := time.Time{}
	for i := range rows {
		r, ok := rows[i].(Annotatable)
		if !ok {
			return ErrNotAnnotatable
		}

		// Only ask for the IP if it is non-empty.
		ip := r.GetServerIP()
		if ip != "" {
			serverIPs[ip] = struct{}{}
		}

		if (logTime == time.Time{}) {
			logTime = r.GetLogTime()
		}
	}

	ipSlice := make([]string, 0, len(rows))
	for ip := range serverIPs {
		ipSlice = append(ipSlice, ip)
	}
	if len(ipSlice) == 0 {
		return nil
	}
	response, err := ann.v2.GetAnnotations(context.Background(), logTime, ipSlice, label)
	if err != nil {
		log.Println("error in server GetAnnotations: ", err)
		metrics.AnnotationErrorCount.With(prometheus.
			Labels{"source": "Server IP: RPC err in GetAnnotations."}).Inc()
		return err
	}
	annMap := response.Annotations
	if annMap == nil {
		log.Println("empty server annotation response")
		metrics.AnnotationErrorCount.With(prometheus.
			Labels{"source": "Server IP: empty response"}).Inc()
		return ErrAnnotationError
	}

	for i := range rows {
		r, ok := rows[i].(Annotatable)
		if !ok {
			err = ErrNotAnnotatable
		} else {
			ip := r.GetServerIP()
			if ip != "" {
				ann, ok := annMap[ip]
				if ok {
					r.AnnotateServer(ann)
				}
			}
		}
	}

	return err
}

// label is used to label metrics and errors in GetAnnotations
func (ann *annotator) annotateClients(rows []interface{}, label string) error {
	ipSlice := make([]string, 0, 2*len(rows)) // This may be inadequate, but its a reasonable start.
	logTime := time.Time{}
	for i := range rows {
		r, ok := rows[i].(Annotatable)
		if !ok {
			return ErrNotAnnotatable
		}
		ipSlice = append(ipSlice, r.GetClientIPs()...)
		if (logTime == time.Time{}) {
			logTime = r.GetLogTime()
		}
	}

	response, err := ann.v2.GetAnnotations(context.Background(), logTime, ipSlice, label)
	if err != nil {
		log.Println("error in client GetAnnotations: ", err)
		metrics.AnnotationErrorCount.With(prometheus.
			Labels{"source": "Client IP: RPC err in GetAnnotations."}).Inc()
		return err
	}
	annMap := response.Annotations
	if annMap == nil {
		log.Println("empty client annotation response")
		metrics.AnnotationErrorCount.With(prometheus.
			Labels{"source": "Client IP: empty response"}).Inc()
		return ErrAnnotationError
	}

	for i := range rows {
		r, ok := rows[i].(Annotatable)
		if !ok {
			err = ErrNotAnnotatable
		} else {
			// Will not error because we check for nil annMap above.
			r.AnnotateClients(annMap)
		}
	}

	return err
}

// Annotate fetches and applies annotations for all rows
func (ann *annotator) Annotate(rows []interface{}, metricLabel string) error {
	metrics.WorkerState.WithLabelValues(metricLabel, "annotate").Inc()
	defer metrics.WorkerState.WithLabelValues(metricLabel, "annotate").Dec()
	if len(rows) == 0 {
		return nil
	}
	// TODO replace this with a histogram.
	defer func(label string, start time.Time) {
		metrics.AnnotationTimeSummary.With(prometheus.Labels{"test_type": label}).Observe(float64(time.Since(start).Nanoseconds()))
	}(metricLabel, time.Now())

	// TODO Consider doing these in parallel?
	clientErr := ann.annotateClients(rows, metricLabel)
	serverErr := ann.annotateServers(rows, metricLabel)

	if clientErr != nil {
		return clientErr
	}

	if serverErr != nil {
		return serverErr
	}

	return nil
}

// Base provides common parser functionality.
// Base is NOT THREAD-SAFE
type Base struct {
	sink  Sink
	ann   annotator
	buf   *Buffer
	label string // Used in metrics and errors.

	stats ActiveStats
}

// NewBase creates a new Base.  This will generally be embedded in a type specific parser.
func NewBase(label string, sink Sink, bufSize int, ann v2as.Annotator) *Base {
	buf := NewBuffer(bufSize)
	return &Base{sink: sink, ann: annotator{ann}, buf: buf, label: label}
}

// GetStats returns the buffer/sink stats.
func (pb *Base) GetStats() Stats {
	return pb.stats.GetStats()
}

// TaskError return the task level error, based on failed rows, or any other criteria.
func (pb *Base) TaskError() error {
	return nil
}

func (pb *Base) commit(rows []interface{}) error {
	// TODO - care about error?
	_ = pb.ann.Annotate(rows, pb.label)
	// TODO do we need these to be done in order.
	// This is synchronous, blocking, and thread safe.
	done, err := pb.sink.Commit(rows, pb.label)
	if done > 0 {
		pb.stats.Done(done, nil)
	}
	if err != nil {
		pb.stats.Done(len(rows)-done, err)
	}
	return err
}

// Flush synchronously flushes any pending rows.
func (pb *Base) Flush() error {
	rows := pb.buf.Reset()
	pb.stats.MoveToPending(len(rows))
	return pb.commit(rows)
}

// Put adds a row to the buffer.
// Iff the buffer is already full the prior buffered rows are
// annotated and committed to the Sink.
// NOTE: There is no guarantee about ordering of writes resulting from
// sequential calls to Put.  However, once a block of rows is submitted
// to pb.commit, it should be written in the same order to the Sink.
// TODO improve Annotatable architecture.
func (pb *Base) Put(row Annotatable) {
	rows := pb.buf.Append(row)
	pb.stats.Inc()

	if rows != nil {
		pb.stats.MoveToPending(len(rows))
		// TODO consider making this asynchronous.
		err := pb.commit(rows)
		if err != nil {
			log.Println(err)
		}
	}
}
<|endoftext|>


<commit_before>package monitor

import (
	"net/http"
	"time"
)

// Poller makes HTTP request to some URL to return its availability status.
type Poller struct {
	// Timeout of network request.
	Timeout time.Duration
}

// NewPoller constructs a new Poller with default fields.
func NewPoller() *Poller {
	return &Poller{
		Timeout: 3 * time.Second,
	}
}

// PollService makes HTTP GET request to the URL and returns its availability status.
// If there was an error during request, the returned Status structure will
// contain information about the error.
func (p *Poller) PollService(url string) Status {
	client := &http.Client{}
	client.Timeout = p.Timeout

	req, err := http.NewRequest(http.MethodGet, url, nil)
	if err != nil {
		return newURLParsingErrorStatus(err, 0)
	}

	reqStart := time.Now()
	resp, err := client.Do(req)
	reqEnd := time.Now()
	dur := reqEnd.Sub(reqStart)

	if err != nil {
		return netErrToStatus(err, dur)
	}
	defer resp.Body.Close()

	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		return newHTTPErrorStatus(resp, dur)
	}

	return newSuccessStatus(resp, dur)
}
<commit_msg>:wrench: Make monitor.Poller use http:// as default schema if no http schema was provided<commit_after>package monitor

import (
	"net/http"
	"strings"
	"time"
)

// Poller makes HTTP request to some URL to return its availability status.
type Poller struct {
	// Timeout of network request.
	Timeout time.Duration
}

// NewPoller constructs a new Poller with default fields.
func NewPoller() *Poller {
	return &Poller{
		Timeout: 3 * time.Second,
	}
}

// PollService makes HTTP GET request to the URL and returns its availability status.
// If there was an error during request, the returned Status structure will
// contain information about the error.
func (p *Poller) PollService(url string) Status {
	client := &http.Client{}
	client.Timeout = p.Timeout

	if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") {
		url = "http://" + url
	}

	req, err := http.NewRequest(http.MethodGet, url, nil)
	if err != nil {
		return newURLParsingErrorStatus(err, 0)
	}

	reqStart := time.Now()
	resp, err := client.Do(req)
	reqEnd := time.Now()
	dur := reqEnd.Sub(reqStart)

	if err != nil {
		return netErrToStatus(err, dur)
	}
	defer resp.Body.Close()

	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		return newHTTPErrorStatus(resp, dur)
	}

	return newSuccessStatus(resp, dur)
}
<|endoftext|>


<commit_before>package sudoku

import (
		"testing"
		"strconv"
)

func TestGrid(t *testing.T){
	puzzles := [1][9][9]int{
								{
									{3,0,0,9,6,0,0,0,0},
									{1,4,0,0,0,5,0,9,0},
									{0,0,5,0,0,0,0,0,8},
									{0,0,0,0,5,0,0,2,0},
									{0,0,3,8,0,0,0,1,9},
									{0,0,0,6,4,0,0,3,0},
									{0,0,0,0,0,0,0,0,1},
									{8,0,0,0,2,0,0,0,0},
									{0,0,1,0,0,3,0,0,4},
								},
	}

	solutions := [1][9][9]int{
								{
									{3,7,2,9,6,8,1,4,5},
									{1,4,8,7,3,5,6,9,2},
									{9,6,5,2,1,4,3,7,8},
									{4,1,7,3,5,9,8,2,6},
									{6,5,3,8,7,2,4,1,9},
									{2,8,9,6,4,1,5,3,7},
									{5,3,6,4,9,7,2,8,1},
									{8,9,4,1,2,6,7,5,3},
									{7,2,1,5,8,3,9,6,4},
								},
	}

	for i,puzz := range puzzles{
		var g Grid
		g.Fill(puzz)

		res,err := g.Solve()
		if err != nil{
			t.Errorf(err.Error())
		}else{
			if res.Solved() && res.Grid() != nil{
				if !res.Grid().KnownEquals(solutions[i]){
					t.Errorf("Test " + strconv.Itoa(i) + ": Solution found, but did not match expected solution instead got\n" + res.Grid().String())
				}
			}else{
				t.Errorf("unable to solve puzzle")
			}

		}
	}
}<commit_msg>Add same info to other test fail cases<commit_after>package sudoku

import (
		"testing"
		"strconv"
)

func TestGrid(t *testing.T){
	puzzles := [1][9][9]int{
								{
									{3,0,0,9,6,0,0,0,0},
									{1,4,0,0,0,5,0,9,0},
									{0,0,5,0,0,0,0,0,8},
									{0,0,0,0,5,0,0,2,0},
									{0,0,3,8,0,0,0,1,9},
									{0,0,0,6,4,0,0,3,0},
									{0,0,0,0,0,0,0,0,1},
									{8,0,0,0,2,0,0,0,0},
									{0,0,1,0,0,3,0,0,4},
								},
	}

	solutions := [1][9][9]int{
								{
									{3,7,2,9,6,8,1,4,5},
									{1,4,8,7,3,5,6,9,2},
									{9,6,5,2,1,4,3,7,8},
									{4,1,7,3,5,9,8,2,6},
									{6,5,3,8,7,2,4,1,9},
									{2,8,9,6,4,1,5,3,7},
									{5,3,6,4,9,7,2,8,1},
									{8,9,4,1,2,6,7,5,3},
									{7,2,1,5,8,3,9,6,4},
								},
	}

	for i,puzz := range puzzles{
		var g Grid
		g.Fill(puzz)

		res,err := g.Solve()
		if err != nil{
			t.Errorf("Test " + strconv.Itoa(i) + ": " + err.Error())
		}else{
			if res.Solved() && res.Grid() != nil{
				if !res.Grid().KnownEquals(solutions[i]){
					t.Errorf("Test " + strconv.Itoa(i) + ": Solution found, but did not match expected solution instead got\n" + res.Grid().String())
				}
			}else{
				t.Errorf("Test " + strconv.Itoa(i) + ": unable to solve puzzle")
			}

		}
	}
}<|endoftext|>


<commit_before>// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package commands

import (
	"bytes"
	"errors"
	"sync"

	"golang.org/x/sync/semaphore"

	"github.com/gohugoio/hugo/modules"

	"io/ioutil"

	"github.com/gohugoio/hugo/common/herrors"
	"github.com/gohugoio/hugo/common/hugo"

	jww "github.com/spf13/jwalterweatherman"

	"os"
	"path/filepath"
	"regexp"
	"time"

	"github.com/gohugoio/hugo/common/loggers"
	"github.com/gohugoio/hugo/config"

	"github.com/spf13/cobra"

	"github.com/gohugoio/hugo/hugolib"
	"github.com/spf13/afero"

	"github.com/bep/debounce"
	"github.com/gohugoio/hugo/common/types"
	"github.com/gohugoio/hugo/deps"
	"github.com/gohugoio/hugo/helpers"
	"github.com/gohugoio/hugo/hugofs"
	"github.com/gohugoio/hugo/langs"
)

type commandeerHugoState struct {
	*deps.DepsCfg
	hugoSites *hugolib.HugoSites
	fsCreate  sync.Once
	created   chan struct{}
}

type commandeer struct {
	*commandeerHugoState

	logger *loggers.Logger

	// Currently only set when in "fast render mode". But it seems to
	// be fast enough that we could maybe just add it for all server modes.
	changeDetector *fileChangeDetector

	// We need to reuse this on server rebuilds.
	destinationFs afero.Fs

	h    *hugoBuilderCommon
	ftch flagsToConfigHandler

	visitedURLs *types.EvictingStringQueue

	doWithCommandeer func(c *commandeer) error

	// We watch these for changes.
	configFiles []string

	// Used in cases where we get flooded with events in server mode.
	debounce func(f func())

	serverPorts         []int
	languagesConfigured bool
	languages           langs.Languages
	doLiveReload        bool
	fastRenderMode      bool
	showErrorInBrowser  bool

	configured bool
	paused     bool

	fullRebuildSem *semaphore.Weighted

	// Any error from the last build.
	buildErr error
}

func newCommandeerHugoState() *commandeerHugoState {
	return &commandeerHugoState{
		created: make(chan struct{}),
	}
}

func (c *commandeerHugoState) hugo() *hugolib.HugoSites {
	<-c.created
	return c.hugoSites
}

func (c *commandeer) errCount() int {
	return int(c.logger.ErrorCounter.Count())
}

func (c *commandeer) getErrorWithContext() interface{} {
	errCount := c.errCount()

	if errCount == 0 {
		return nil
	}

	m := make(map[string]interface{})

	m["Error"] = errors.New(removeErrorPrefixFromLog(c.logger.Errors()))
	m["Version"] = hugo.BuildVersionString()

	fe := herrors.UnwrapErrorWithFileContext(c.buildErr)
	if fe != nil {
		m["File"] = fe
	}

	if c.h.verbose {
		var b bytes.Buffer
		herrors.FprintStackTrace(&b, c.buildErr)
		m["StackTrace"] = b.String()
	}

	return m
}

func (c *commandeer) Set(key string, value interface{}) {
	if c.configured {
		panic("commandeer cannot be changed")
	}
	c.Cfg.Set(key, value)
}

func (c *commandeer) initFs(fs *hugofs.Fs) error {
	c.destinationFs = fs.Destination
	c.DepsCfg.Fs = fs

	return nil
}

func newCommandeer(mustHaveConfigFile, running bool, h *hugoBuilderCommon, f flagsToConfigHandler, doWithCommandeer func(c *commandeer) error, subCmdVs ...*cobra.Command) (*commandeer, error) {

	var rebuildDebouncer func(f func())
	if running {
		// The time value used is tested with mass content replacements in a fairly big Hugo site.
		// It is better to wait for some seconds in those cases rather than get flooded
		// with rebuilds.
		rebuildDebouncer = debounce.New(4 * time.Second)
	}

	c := &commandeer{
		h:                   h,
		ftch:                f,
		commandeerHugoState: newCommandeerHugoState(),
		doWithCommandeer:    doWithCommandeer,
		visitedURLs:         types.NewEvictingStringQueue(10),
		debounce:            rebuildDebouncer,
		fullRebuildSem:      semaphore.NewWeighted(1),
		// This will be replaced later, but we need something to log to before the configuration is read.
		logger: loggers.NewLogger(jww.LevelError, jww.LevelError, os.Stdout, ioutil.Discard, running),
	}

	return c, c.loadConfig(mustHaveConfigFile, running)
}

type fileChangeDetector struct {
	sync.Mutex
	current map[string]string
	prev    map[string]string

	irrelevantRe *regexp.Regexp
}

func (f *fileChangeDetector) OnFileClose(name, md5sum string) {
	f.Lock()
	defer f.Unlock()
	f.current[name] = md5sum
}

func (f *fileChangeDetector) changed() []string {
	if f == nil {
		return nil
	}
	f.Lock()
	defer f.Unlock()
	var c []string
	for k, v := range f.current {
		vv, found := f.prev[k]
		if !found || v != vv {
			c = append(c, k)
		}
	}

	return f.filterIrrelevant(c)
}

func (f *fileChangeDetector) filterIrrelevant(in []string) []string {
	var filtered []string
	for _, v := range in {
		if !f.irrelevantRe.MatchString(v) {
			filtered = append(filtered, v)
		}
	}
	return filtered
}

func (f *fileChangeDetector) PrepareNew() {
	if f == nil {
		return
	}

	f.Lock()
	defer f.Unlock()

	if f.current == nil {
		f.current = make(map[string]string)
		f.prev = make(map[string]string)
		return
	}

	f.prev = make(map[string]string)
	for k, v := range f.current {
		f.prev[k] = v
	}
	f.current = make(map[string]string)
}

func (c *commandeer) loadConfig(mustHaveConfigFile, running bool) error {

	if c.DepsCfg == nil {
		c.DepsCfg = &deps.DepsCfg{}
	}

	if c.logger != nil {
		// Truncate the error log if this is a reload.
		c.logger.Reset()
	}

	cfg := c.DepsCfg
	c.configured = false
	cfg.Running = running

	var dir string
	if c.h.source != "" {
		dir, _ = filepath.Abs(c.h.source)
	} else {
		dir, _ = os.Getwd()
	}

	var sourceFs afero.Fs = hugofs.Os
	if c.DepsCfg.Fs != nil {
		sourceFs = c.DepsCfg.Fs.Source
	}

	environment := c.h.getEnvironment(running)

	doWithConfig := func(cfg config.Provider) error {

		if c.ftch != nil {
			c.ftch.flagsToConfig(cfg)
		}

		cfg.Set("workingDir", dir)
		cfg.Set("environment", environment)
		return nil
	}

	doWithCommandeer := func(cfg config.Provider) error {
		c.Cfg = cfg
		if c.doWithCommandeer == nil {
			return nil
		}
		err := c.doWithCommandeer(c)
		return err
	}

	configPath := c.h.source
	if configPath == "" {
		configPath = dir
	}
	config, configFiles, err := hugolib.LoadConfig(
		hugolib.ConfigSourceDescriptor{
			Fs:           sourceFs,
			Path:         configPath,
			WorkingDir:   dir,
			Filename:     c.h.cfgFile,
			AbsConfigDir: c.h.getConfigDir(dir),
			Environ:      os.Environ(),
			Environment:  environment},
		doWithCommandeer,
		doWithConfig)

	if err != nil {
		if mustHaveConfigFile {
			return err
		}
		if err != hugolib.ErrNoConfigFile && !modules.IsNotExist(err) {
			return err
		}

	}

	c.configFiles = configFiles

	if l, ok := c.Cfg.Get("languagesSorted").(langs.Languages); ok {
		c.languagesConfigured = true
		c.languages = l
	}

	// Set some commonly used flags
	c.doLiveReload = running && !c.Cfg.GetBool("disableLiveReload")
	c.fastRenderMode = c.doLiveReload && !c.Cfg.GetBool("disableFastRender")
	c.showErrorInBrowser = c.doLiveReload && !c.Cfg.GetBool("disableBrowserError")

	// This is potentially double work, but we need to do this one more time now
	// that all the languages have been configured.
	if c.doWithCommandeer != nil {
		if err := c.doWithCommandeer(c); err != nil {
			return err
		}
	}

	logger, err := c.createLogger(config, running)
	if err != nil {
		return err
	}

	cfg.Logger = logger
	c.logger = logger

	createMemFs := config.GetBool("renderToMemory")

	if createMemFs {
		// Rendering to memoryFS, publish to Root regardless of publishDir.
		config.Set("publishDir", "/")
	}

	c.fsCreate.Do(func() {
		fs := hugofs.NewFrom(sourceFs, config)

		if c.destinationFs != nil {
			// Need to reuse the destination on server rebuilds.
			fs.Destination = c.destinationFs
		} else if createMemFs {
			// Hugo writes the output to memory instead of the disk.
			fs.Destination = new(afero.MemMapFs)
		}

		if c.fastRenderMode {
			// For now, fast render mode only. It should, however, be fast enough
			// for the full variant, too.
			changeDetector := &fileChangeDetector{
				// We use this detector to decide to do a Hot reload of a single path or not.
				// We need to filter out source maps and possibly some other to be able
				// to make that decision.
				irrelevantRe: regexp.MustCompile(`\.map$`),
			}

			changeDetector.PrepareNew()
			fs.Destination = hugofs.NewHashingFs(fs.Destination, changeDetector)
			c.changeDetector = changeDetector
		}

		if c.Cfg.GetBool("logPathWarnings") {
			fs.Destination = hugofs.NewCreateCountingFs(fs.Destination)
		}

		// To debug hard-to-find path issues.
		//fs.Destination = hugofs.NewStacktracerFs(fs.Destination, `fr/fr`)

		err = c.initFs(fs)
		if err != nil {
			close(c.created)
			return
		}

		var h *hugolib.HugoSites

		h, err = hugolib.NewHugoSites(*c.DepsCfg)
		c.hugoSites = h
		close(c.created)

	})

	if err != nil {
		return err
	}

	cacheDir, err := helpers.GetCacheDir(sourceFs, config)
	if err != nil {
		return err
	}
	config.Set("cacheDir", cacheDir)

	cfg.Logger.INFO.Println("Using config file:", config.ConfigFileUsed())

	return nil

}
<commit_msg>Do not attempt to build if there is no config file<commit_after>// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package commands

import (
	"bytes"
	"errors"
	"sync"

	"golang.org/x/sync/semaphore"

	"github.com/gohugoio/hugo/modules"

	"io/ioutil"

	"github.com/gohugoio/hugo/common/herrors"
	"github.com/gohugoio/hugo/common/hugo"

	jww "github.com/spf13/jwalterweatherman"

	"os"
	"path/filepath"
	"regexp"
	"time"

	"github.com/gohugoio/hugo/common/loggers"
	"github.com/gohugoio/hugo/config"

	"github.com/spf13/cobra"

	"github.com/gohugoio/hugo/hugolib"
	"github.com/spf13/afero"

	"github.com/bep/debounce"
	"github.com/gohugoio/hugo/common/types"
	"github.com/gohugoio/hugo/deps"
	"github.com/gohugoio/hugo/helpers"
	"github.com/gohugoio/hugo/hugofs"
	"github.com/gohugoio/hugo/langs"
)

type commandeerHugoState struct {
	*deps.DepsCfg
	hugoSites *hugolib.HugoSites
	fsCreate  sync.Once
	created   chan struct{}
}

type commandeer struct {
	*commandeerHugoState

	logger *loggers.Logger

	// Currently only set when in "fast render mode". But it seems to
	// be fast enough that we could maybe just add it for all server modes.
	changeDetector *fileChangeDetector

	// We need to reuse this on server rebuilds.
	destinationFs afero.Fs

	h    *hugoBuilderCommon
	ftch flagsToConfigHandler

	visitedURLs *types.EvictingStringQueue

	doWithCommandeer func(c *commandeer) error

	// We watch these for changes.
	configFiles []string

	// Used in cases where we get flooded with events in server mode.
	debounce func(f func())

	serverPorts         []int
	languagesConfigured bool
	languages           langs.Languages
	doLiveReload        bool
	fastRenderMode      bool
	showErrorInBrowser  bool

	configured bool
	paused     bool

	fullRebuildSem *semaphore.Weighted

	// Any error from the last build.
	buildErr error
}

func newCommandeerHugoState() *commandeerHugoState {
	return &commandeerHugoState{
		created: make(chan struct{}),
	}
}

func (c *commandeerHugoState) hugo() *hugolib.HugoSites {
	<-c.created
	return c.hugoSites
}

func (c *commandeer) errCount() int {
	return int(c.logger.ErrorCounter.Count())
}

func (c *commandeer) getErrorWithContext() interface{} {
	errCount := c.errCount()

	if errCount == 0 {
		return nil
	}

	m := make(map[string]interface{})

	m["Error"] = errors.New(removeErrorPrefixFromLog(c.logger.Errors()))
	m["Version"] = hugo.BuildVersionString()

	fe := herrors.UnwrapErrorWithFileContext(c.buildErr)
	if fe != nil {
		m["File"] = fe
	}

	if c.h.verbose {
		var b bytes.Buffer
		herrors.FprintStackTrace(&b, c.buildErr)
		m["StackTrace"] = b.String()
	}

	return m
}

func (c *commandeer) Set(key string, value interface{}) {
	if c.configured {
		panic("commandeer cannot be changed")
	}
	c.Cfg.Set(key, value)
}

func (c *commandeer) initFs(fs *hugofs.Fs) error {
	c.destinationFs = fs.Destination
	c.DepsCfg.Fs = fs

	return nil
}

func newCommandeer(mustHaveConfigFile, running bool, h *hugoBuilderCommon, f flagsToConfigHandler, doWithCommandeer func(c *commandeer) error, subCmdVs ...*cobra.Command) (*commandeer, error) {

	var rebuildDebouncer func(f func())
	if running {
		// The time value used is tested with mass content replacements in a fairly big Hugo site.
		// It is better to wait for some seconds in those cases rather than get flooded
		// with rebuilds.
		rebuildDebouncer = debounce.New(4 * time.Second)
	}

	c := &commandeer{
		h:                   h,
		ftch:                f,
		commandeerHugoState: newCommandeerHugoState(),
		doWithCommandeer:    doWithCommandeer,
		visitedURLs:         types.NewEvictingStringQueue(10),
		debounce:            rebuildDebouncer,
		fullRebuildSem:      semaphore.NewWeighted(1),
		// This will be replaced later, but we need something to log to before the configuration is read.
		logger: loggers.NewLogger(jww.LevelError, jww.LevelError, os.Stdout, ioutil.Discard, running),
	}

	return c, c.loadConfig(mustHaveConfigFile, running)
}

type fileChangeDetector struct {
	sync.Mutex
	current map[string]string
	prev    map[string]string

	irrelevantRe *regexp.Regexp
}

func (f *fileChangeDetector) OnFileClose(name, md5sum string) {
	f.Lock()
	defer f.Unlock()
	f.current[name] = md5sum
}

func (f *fileChangeDetector) changed() []string {
	if f == nil {
		return nil
	}
	f.Lock()
	defer f.Unlock()
	var c []string
	for k, v := range f.current {
		vv, found := f.prev[k]
		if !found || v != vv {
			c = append(c, k)
		}
	}

	return f.filterIrrelevant(c)
}

func (f *fileChangeDetector) filterIrrelevant(in []string) []string {
	var filtered []string
	for _, v := range in {
		if !f.irrelevantRe.MatchString(v) {
			filtered = append(filtered, v)
		}
	}
	return filtered
}

func (f *fileChangeDetector) PrepareNew() {
	if f == nil {
		return
	}

	f.Lock()
	defer f.Unlock()

	if f.current == nil {
		f.current = make(map[string]string)
		f.prev = make(map[string]string)
		return
	}

	f.prev = make(map[string]string)
	for k, v := range f.current {
		f.prev[k] = v
	}
	f.current = make(map[string]string)
}

func (c *commandeer) loadConfig(mustHaveConfigFile, running bool) error {

	if c.DepsCfg == nil {
		c.DepsCfg = &deps.DepsCfg{}
	}

	if c.logger != nil {
		// Truncate the error log if this is a reload.
		c.logger.Reset()
	}

	cfg := c.DepsCfg
	c.configured = false
	cfg.Running = running

	var dir string
	if c.h.source != "" {
		dir, _ = filepath.Abs(c.h.source)
	} else {
		dir, _ = os.Getwd()
	}

	var sourceFs afero.Fs = hugofs.Os
	if c.DepsCfg.Fs != nil {
		sourceFs = c.DepsCfg.Fs.Source
	}

	environment := c.h.getEnvironment(running)

	doWithConfig := func(cfg config.Provider) error {

		if c.ftch != nil {
			c.ftch.flagsToConfig(cfg)
		}

		cfg.Set("workingDir", dir)
		cfg.Set("environment", environment)
		return nil
	}

	doWithCommandeer := func(cfg config.Provider) error {
		c.Cfg = cfg
		if c.doWithCommandeer == nil {
			return nil
		}
		err := c.doWithCommandeer(c)
		return err
	}

	configPath := c.h.source
	if configPath == "" {
		configPath = dir
	}
	config, configFiles, err := hugolib.LoadConfig(
		hugolib.ConfigSourceDescriptor{
			Fs:           sourceFs,
			Path:         configPath,
			WorkingDir:   dir,
			Filename:     c.h.cfgFile,
			AbsConfigDir: c.h.getConfigDir(dir),
			Environ:      os.Environ(),
			Environment:  environment},
		doWithCommandeer,
		doWithConfig)

	if err != nil {
		if mustHaveConfigFile {
			return err
		}
		if err != hugolib.ErrNoConfigFile && !modules.IsNotExist(err) {
			return err
		}

	} else if mustHaveConfigFile && len(configFiles) == 0 {
		return hugolib.ErrNoConfigFile
	}

	c.configFiles = configFiles

	if l, ok := c.Cfg.Get("languagesSorted").(langs.Languages); ok {
		c.languagesConfigured = true
		c.languages = l
	}

	// Set some commonly used flags
	c.doLiveReload = running && !c.Cfg.GetBool("disableLiveReload")
	c.fastRenderMode = c.doLiveReload && !c.Cfg.GetBool("disableFastRender")
	c.showErrorInBrowser = c.doLiveReload && !c.Cfg.GetBool("disableBrowserError")

	// This is potentially double work, but we need to do this one more time now
	// that all the languages have been configured.
	if c.doWithCommandeer != nil {
		if err := c.doWithCommandeer(c); err != nil {
			return err
		}
	}

	logger, err := c.createLogger(config, running)
	if err != nil {
		return err
	}

	cfg.Logger = logger
	c.logger = logger

	createMemFs := config.GetBool("renderToMemory")

	if createMemFs {
		// Rendering to memoryFS, publish to Root regardless of publishDir.
		config.Set("publishDir", "/")
	}

	c.fsCreate.Do(func() {
		fs := hugofs.NewFrom(sourceFs, config)

		if c.destinationFs != nil {
			// Need to reuse the destination on server rebuilds.
			fs.Destination = c.destinationFs
		} else if createMemFs {
			// Hugo writes the output to memory instead of the disk.
			fs.Destination = new(afero.MemMapFs)
		}

		if c.fastRenderMode {
			// For now, fast render mode only. It should, however, be fast enough
			// for the full variant, too.
			changeDetector := &fileChangeDetector{
				// We use this detector to decide to do a Hot reload of a single path or not.
				// We need to filter out source maps and possibly some other to be able
				// to make that decision.
				irrelevantRe: regexp.MustCompile(`\.map$`),
			}

			changeDetector.PrepareNew()
			fs.Destination = hugofs.NewHashingFs(fs.Destination, changeDetector)
			c.changeDetector = changeDetector
		}

		if c.Cfg.GetBool("logPathWarnings") {
			fs.Destination = hugofs.NewCreateCountingFs(fs.Destination)
		}

		// To debug hard-to-find path issues.
		//fs.Destination = hugofs.NewStacktracerFs(fs.Destination, `fr/fr`)

		err = c.initFs(fs)
		if err != nil {
			close(c.created)
			return
		}

		var h *hugolib.HugoSites

		h, err = hugolib.NewHugoSites(*c.DepsCfg)
		c.hugoSites = h
		close(c.created)

	})

	if err != nil {
		return err
	}

	cacheDir, err := helpers.GetCacheDir(sourceFs, config)
	if err != nil {
		return err
	}
	config.Set("cacheDir", cacheDir)

	cfg.Logger.INFO.Println("Using config file:", config.ConfigFileUsed())

	return nil

}
<|endoftext|>


<commit_before>#!/bin/bash
#
# The "build" function below will later build a binary from a preprocessed
# source file. When "build" is called, these variables are set:
#
# - SOURCE:  the full path of the source file.
# - PREPROCESSED_SOURCE: the full path of the preprocessed source file,
#            i.e. without the leading #! line
# - BINARY:  the full path of the binary. The build function must generate
#            this file.
# - JITOS:   the host operating system
# - JITARCH: the host architecture.
#
function build() {
  if [[ $JIT_DIST_BUILD = y ]]; then
    log "Building for distribution"
    local platforms=${JIT_GO_TARGET_PLATFORMS:-darwin.amd64 linux.amd64 linux.arm linux.386}
  else
    local platforms=$JITOS.$JITARCH
  fi

  local binary_base=$(echo $BINARY | sed 's-[.][^.]*[.][^.]*$--')

  local platform
  for platform in $platforms ; do
    local goos=$(cut -d . -f 1 <<<$platform)
    local goarch=$(cut -d . -f 2 <<<$platform)
    env GOARCH=$goarch GOOS=$goos go build -ldflags "-s" -o $binary_base.$goos.$goarch $PREPROCESSED_SOURCE
    log Built $binary_base.$goarch.$goos
  done
}

JIT_EXT=go
jit_inc=$(echo "$0" | sed 's-[.][^.]*$-.inc-')
. "$jit_inc"
<commit_msg>go: read JIT_GO_FLAGS<commit_after>#!/bin/bash
#
# The "build" function below will later build a binary from a preprocessed
# source file. When "build" is called, these variables are set:
#
# - SOURCE:  the full path of the source file.
# - PREPROCESSED_SOURCE: the full path of the preprocessed source file,
#            i.e. without the leading #! line
# - BINARY:  the full path of the binary. The build function must generate
#            this file.
# - JITOS:   the host operating system
# - JITARCH: the host architecture.
#
function build() {
  # -- prepare go packages if needed.
  if [ -z "${GOPATH:-}" ]; then
    GOPATH=$HOME/jit.golang
  fi

  local packages=$(
    < $SOURCE grep "// JIT_GO_PACKAGES="     | sed "sx// JIT_GO_PACKAGES=xx"
  )

  for package in "$packages" ; do
    go get $package
  done

  if [[ $JIT_DIST_BUILD = y ]]; then
    log "Building for distribution"
    local platforms=${JIT_GO_TARGET_PLATFORMS:-darwin.amd64 linux.amd64 linux.arm linux.386}
  else
    local platforms=$JITOS.$JITARCH
  fi

  local packages=$(
    < $SOURCE grep "// JIT_GO_PACKAGES="     | sed "sx// JIT_GO_PACKAGES=xx"
  )

  for package in "$packages" ; do
    go get $package
  done

  local binary_base=$(echo $BINARY | sed 's-[.][^.]*[.][^.]*$--')

  local platform
  for platform in $platforms ; do
    local goos=$(cut -d . -f 1 <<<$platform)
    local goarch=$(cut -d . -f 2 <<<$platform)
	if ! env GOARCH=$goarch GOOS=$goos go build -ldflags "-s" -o $binary_base.$goos.$goarch $PREPROCESSED_SOURCE ; then
		exit 1
	fi

	log Built $binary_base.$goos.$goarch
  done
}

JIT_EXT=go
jit_inc=$(echo "$0" | sed 's-[.][^.]*$-.inc-')
. "$jit_inc"
<|endoftext|>


<commit_before>// Copyright (C) 2019-2021, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package handler

import (
	"sync"

	"github.com/prometheus/client_golang/prometheus"

	"github.com/ava-labs/avalanchego/ids"
	"github.com/ava-labs/avalanchego/message"
	"github.com/ava-labs/avalanchego/snow/networking/tracker"
	"github.com/ava-labs/avalanchego/snow/validators"
	"github.com/ava-labs/avalanchego/utils/logging"
	"github.com/ava-labs/avalanchego/utils/timer/mockable"
)

var _ MessageQueue = &messageQueue{}

type MessageQueue interface {
	// Add a message.
	//
	// If called after [Shutdown], the message will immediately be marked as
	// having been handled.
	Push(message.InboundMessage)

	// Get and remove a message.
	//
	// If there are no available messages, this function will block until a
	// message becomes available or the queue is [Shutdown].
	Pop() (message.InboundMessage, bool)

	// Returns the number of messages currently on the queue
	Len() int

	// Shutdown and empty the queue.
	Shutdown()
}

// TODO: Use a better data structure for this.
// We can do something better than pushing to the back of a queue. A multi-level
// queue?
type messageQueue struct {
	// Useful for faking time in tests
	clock   mockable.Clock
	metrics messageQueueMetrics

	log logging.Logger
	// Validator set for the chain associated with this
	vdrs validators.Set
	// Tracks CPU utilization of each node
	cpuTracker tracker.TimeTracker

	cond   *sync.Cond
	closed bool
	// Node ID --> Messages this node has in [msgs]
	nodeToUnprocessedMsgs map[ids.ShortID]int
	// Unprocessed messages
	msgs []message.InboundMessage
}

func NewMessageQueue(
	log logging.Logger,
	vdrs validators.Set,
	cpuTracker tracker.TimeTracker,
	metricsNamespace string,
	metricsRegisterer prometheus.Registerer,
	ops []message.Op,
) (MessageQueue, error) {
	m := &messageQueue{
		log:                   log,
		vdrs:                  vdrs,
		cpuTracker:            cpuTracker,
		cond:                  sync.NewCond(&sync.Mutex{}),
		nodeToUnprocessedMsgs: make(map[ids.ShortID]int),
	}
	return m, m.metrics.initialize(metricsNamespace, metricsRegisterer, ops)
}

func (m *messageQueue) Push(msg message.InboundMessage) {
	m.cond.L.Lock()
	defer m.cond.L.Unlock()

	if m.closed {
		msg.OnFinishedHandling()
		return
	}

	// Add the message to the queue
	m.msgs = append(m.msgs, msg)
	m.nodeToUnprocessedMsgs[msg.NodeID()]++

	// Update metrics
	m.metrics.nodesWithMessages.Set(float64(len(m.nodeToUnprocessedMsgs)))
	m.metrics.len.Inc()
	m.metrics.ops[msg.Op()].Inc()

	// Signal a waiting thread
	m.cond.Signal()
}

// FIFO, but skip over messages whose senders whose messages have caused us to
// use excessive CPU recently.
func (m *messageQueue) Pop() (message.InboundMessage, bool) {
	m.cond.L.Lock()
	defer m.cond.L.Unlock()

	for {
		if m.closed {
			return nil, false
		}
		if len(m.msgs) != 0 {
			break
		}
		m.cond.Wait()
	}

	n := len(m.msgs)
	i := 0
	for {
		if i == n {
			m.log.Debug("canPop is false for all %d unprocessed messages", n)
		}
		msg := m.msgs[0]
		nodeID := msg.NodeID()
		// See if it's OK to process [msg] next
		if m.canPop(msg) || i == n { // i should never == n but handle anyway as a fail-safe
			if cap(m.msgs) == 1 {
				m.msgs = nil // Give back memory if possible
			} else {
				m.msgs = m.msgs[1:]
			}
			m.nodeToUnprocessedMsgs[nodeID]--
			if m.nodeToUnprocessedMsgs[nodeID] == 0 {
				delete(m.nodeToUnprocessedMsgs, nodeID)
			}
			m.metrics.nodesWithMessages.Set(float64(len(m.nodeToUnprocessedMsgs)))
			m.metrics.len.Dec()
			m.metrics.ops[msg.Op()].Dec()
			return msg, true
		}
		// [msg.nodeID] is causing excessive CPU usage.
		// Push [msg] to back of [m.msgs] and handle it later.
		m.msgs = append(m.msgs, msg)
		m.msgs = m.msgs[1:]
		i++
		m.metrics.numExcessiveCPU.Inc()
	}
}

func (m *messageQueue) Len() int {
	m.cond.L.Lock()
	defer m.cond.L.Unlock()

	return len(m.msgs)
}

func (m *messageQueue) Shutdown() {
	m.cond.L.Lock()
	defer m.cond.L.Unlock()

	// Remove all the current messages from the queue
	for _, msg := range m.msgs {
		msg.OnFinishedHandling()
	}
	m.msgs = nil
	m.nodeToUnprocessedMsgs = nil

	// Update metrics
	m.metrics.nodesWithMessages.Set(0)
	m.metrics.len.Set(0)

	// Mark the queue as closed
	m.closed = true
	m.cond.Broadcast()
}

// canPop will return true for at least one message in [m.msgs]
func (m *messageQueue) canPop(msg message.InboundMessage) bool {
	// Always pop connected and disconnected messages.
	if op := msg.Op(); op == message.Connected || op == message.Disconnected {
		return true
	}

	// If the deadline to handle [msg] has passed, always pop it.
	// It will be dropped immediately.
	if expirationTime := msg.ExpirationTime(); !expirationTime.IsZero() && m.clock.Time().After(expirationTime) {
		return true
	}
	// Every node has some allowed CPU allocation depending on
	// the number of nodes with unprocessed messages.
	baseMaxCPU := 1 / float64(len(m.nodeToUnprocessedMsgs))
	nodeID := msg.NodeID()
	weight, isVdr := m.vdrs.GetWeight(nodeID)
	if !isVdr {
		weight = 0
	}
	// The sum of validator weights should never be 0, but handle
	// that case for completeness here to avoid divide by 0.
	portionWeight := float64(0)
	totalVdrsWeight := m.vdrs.Weight()
	if totalVdrsWeight != 0 {
		portionWeight = float64(weight) / float64(totalVdrsWeight)
	}
	// Validators are allowed to use more CPm. More weight --> more CPU use allowed.
	recentCPUUtilized := m.cpuTracker.Utilization(nodeID, m.clock.Time())
	maxCPU := baseMaxCPU + (1.0-baseMaxCPU)*portionWeight
	return recentCPUUtilized <= maxCPU
}
<commit_msg>Remove reference to message so it can be garbage collected (#1245)<commit_after>// Copyright (C) 2019-2021, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package handler

import (
	"sync"

	"github.com/prometheus/client_golang/prometheus"

	"github.com/ava-labs/avalanchego/ids"
	"github.com/ava-labs/avalanchego/message"
	"github.com/ava-labs/avalanchego/snow/networking/tracker"
	"github.com/ava-labs/avalanchego/snow/validators"
	"github.com/ava-labs/avalanchego/utils/logging"
	"github.com/ava-labs/avalanchego/utils/timer/mockable"
)

var _ MessageQueue = &messageQueue{}

type MessageQueue interface {
	// Add a message.
	//
	// If called after [Shutdown], the message will immediately be marked as
	// having been handled.
	Push(message.InboundMessage)

	// Get and remove a message.
	//
	// If there are no available messages, this function will block until a
	// message becomes available or the queue is [Shutdown].
	Pop() (message.InboundMessage, bool)

	// Returns the number of messages currently on the queue
	Len() int

	// Shutdown and empty the queue.
	Shutdown()
}

// TODO: Use a better data structure for this.
// We can do something better than pushing to the back of a queue. A multi-level
// queue?
type messageQueue struct {
	// Useful for faking time in tests
	clock   mockable.Clock
	metrics messageQueueMetrics

	log logging.Logger
	// Validator set for the chain associated with this
	vdrs validators.Set
	// Tracks CPU utilization of each node
	cpuTracker tracker.TimeTracker

	cond   *sync.Cond
	closed bool
	// Node ID --> Messages this node has in [msgs]
	nodeToUnprocessedMsgs map[ids.ShortID]int
	// Unprocessed messages
	msgs []message.InboundMessage
}

func NewMessageQueue(
	log logging.Logger,
	vdrs validators.Set,
	cpuTracker tracker.TimeTracker,
	metricsNamespace string,
	metricsRegisterer prometheus.Registerer,
	ops []message.Op,
) (MessageQueue, error) {
	m := &messageQueue{
		log:                   log,
		vdrs:                  vdrs,
		cpuTracker:            cpuTracker,
		cond:                  sync.NewCond(&sync.Mutex{}),
		nodeToUnprocessedMsgs: make(map[ids.ShortID]int),
	}
	return m, m.metrics.initialize(metricsNamespace, metricsRegisterer, ops)
}

func (m *messageQueue) Push(msg message.InboundMessage) {
	m.cond.L.Lock()
	defer m.cond.L.Unlock()

	if m.closed {
		msg.OnFinishedHandling()
		return
	}

	// Add the message to the queue
	m.msgs = append(m.msgs, msg)
	m.nodeToUnprocessedMsgs[msg.NodeID()]++

	// Update metrics
	m.metrics.nodesWithMessages.Set(float64(len(m.nodeToUnprocessedMsgs)))
	m.metrics.len.Inc()
	m.metrics.ops[msg.Op()].Inc()

	// Signal a waiting thread
	m.cond.Signal()
}

// FIFO, but skip over messages whose senders whose messages have caused us to
// use excessive CPU recently.
func (m *messageQueue) Pop() (message.InboundMessage, bool) {
	m.cond.L.Lock()
	defer m.cond.L.Unlock()

	for {
		if m.closed {
			return nil, false
		}
		if len(m.msgs) != 0 {
			break
		}
		m.cond.Wait()
	}

	n := len(m.msgs)
	i := 0
	for {
		if i == n {
			m.log.Debug("canPop is false for all %d unprocessed messages", n)
		}
		msg := m.msgs[0]
		m.msgs[0] = nil
		nodeID := msg.NodeID()
		// See if it's OK to process [msg] next
		if m.canPop(msg) || i == n { // i should never == n but handle anyway as a fail-safe
			if cap(m.msgs) == 1 {
				m.msgs = nil // Give back memory if possible
			} else {
				m.msgs = m.msgs[1:]
			}
			m.nodeToUnprocessedMsgs[nodeID]--
			if m.nodeToUnprocessedMsgs[nodeID] == 0 {
				delete(m.nodeToUnprocessedMsgs, nodeID)
			}
			m.metrics.nodesWithMessages.Set(float64(len(m.nodeToUnprocessedMsgs)))
			m.metrics.len.Dec()
			m.metrics.ops[msg.Op()].Dec()
			return msg, true
		}
		// [msg.nodeID] is causing excessive CPU usage.
		// Push [msg] to back of [m.msgs] and handle it later.
		m.msgs = append(m.msgs, msg)
		m.msgs = m.msgs[1:]
		i++
		m.metrics.numExcessiveCPU.Inc()
	}
}

func (m *messageQueue) Len() int {
	m.cond.L.Lock()
	defer m.cond.L.Unlock()

	return len(m.msgs)
}

func (m *messageQueue) Shutdown() {
	m.cond.L.Lock()
	defer m.cond.L.Unlock()

	// Remove all the current messages from the queue
	for _, msg := range m.msgs {
		msg.OnFinishedHandling()
	}
	m.msgs = nil
	m.nodeToUnprocessedMsgs = nil

	// Update metrics
	m.metrics.nodesWithMessages.Set(0)
	m.metrics.len.Set(0)

	// Mark the queue as closed
	m.closed = true
	m.cond.Broadcast()
}

// canPop will return true for at least one message in [m.msgs]
func (m *messageQueue) canPop(msg message.InboundMessage) bool {
	// Always pop connected and disconnected messages.
	if op := msg.Op(); op == message.Connected || op == message.Disconnected {
		return true
	}

	// If the deadline to handle [msg] has passed, always pop it.
	// It will be dropped immediately.
	if expirationTime := msg.ExpirationTime(); !expirationTime.IsZero() && m.clock.Time().After(expirationTime) {
		return true
	}
	// Every node has some allowed CPU allocation depending on
	// the number of nodes with unprocessed messages.
	baseMaxCPU := 1 / float64(len(m.nodeToUnprocessedMsgs))
	nodeID := msg.NodeID()
	weight, isVdr := m.vdrs.GetWeight(nodeID)
	if !isVdr {
		weight = 0
	}
	// The sum of validator weights should never be 0, but handle
	// that case for completeness here to avoid divide by 0.
	portionWeight := float64(0)
	totalVdrsWeight := m.vdrs.Weight()
	if totalVdrsWeight != 0 {
		portionWeight = float64(weight) / float64(totalVdrsWeight)
	}
	// Validators are allowed to use more CPm. More weight --> more CPU use allowed.
	recentCPUUtilized := m.cpuTracker.Utilization(nodeID, m.clock.Time())
	maxCPU := baseMaxCPU + (1.0-baseMaxCPU)*portionWeight
	return recentCPUUtilized <= maxCPU
}
<|endoftext|>