<commit_before>// From Go GUI with Fyne, Chap 4.  I believe it will be enhanced in later chapters, but this is what is it for now.
/*
REVISION HISTORY
-------- -------
 9 Aug 21 -- I realized that this will not be enhanced, as I went thru more of the book.  I'll have to enhance it myself.
             First, I'm changing the function constants to the version that's more readable to me.  That's working, but I had to
             import more parts of fyne.io than the unmodified version.
12 Aug 21 -- Now called img.go, so I can display 1 image.  I'll start here.
13 Aug 21 -- Now called imgfyne.go.  Same purpose as img.go, but so I can test non-fyne code there and fyne code here.
15 Aug 21 -- Copied back to img.go after the code works and displays a single image from the command line.
               Will use imgfyne to add the arrow key navigation and img display.
*/

package main

import (
	"flag"
	"fmt"
	"fyne.io/fyne/v2/layout"
	"image"
	"image/color"
	_ "image/gif"
	_ "image/jpeg"
	_ "image/png"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"strings"
	"time"

	//_ "golang.org/x/image"

	"fyne.io/fyne/v2"
	"fyne.io/fyne/v2/app"
	"fyne.io/fyne/v2/canvas"
	"fyne.io/fyne/v2/container"
	"fyne.io/fyne/v2/storage"

	"github.com/nfnt/resize"
)

const LastModified = "August 16, 2021"
const maxWidth = 2500
const maxHeight = 2000

func isNotImageStr(name string) bool {
	ext := filepath.Ext(name)
	isImage := ext == ".png" || ext == ".jpg" || ext == ".jpeg" || ext == ".gif" || ext == ".webp"
	return !isImage
}

func main() {
	flag.Parse()
	if flag.NArg() < 1 {
		fmt.Fprintln(os.Stderr, " Usage: img <image file name>")
		os.Exit(1)
	}

	str := fmt.Sprintf("Single Image Viewer last modified %s, compiled using %s", LastModified, runtime.Version())
	fmt.Println(str) // this works as intended

	imgfilename := flag.Arg(0)
	_, err := os.Stat(imgfilename)
	if err != nil {
		fmt.Fprintln(os.Stderr, " Error from os.Stat(", imgfilename, ") is", err)
		os.Exit(1)
	}

	if isNotImageStr(imgfilename) {
		fmt.Fprintln(os.Stderr, imgfilename, "does not have an image extension.")
		os.Exit(1)
	}

	basefilename := filepath.Base(imgfilename)
	fullFilename, err := filepath.Abs(imgfilename)
	if err != nil {
		fmt.Fprintln(os.Stderr, " Error from filepath.Abs on", imgfilename, "is", err)
		os.Exit(1)
	}

	imgFileHandle, err := os.Open(fullFilename)
	if err != nil {
		fmt.Fprintln(os.Stderr, " Error from opening", fullFilename, "is", err)
		os.Exit(1)
	}

	imgConfig, _, err := image.DecodeConfig(imgFileHandle) // img is of type image.Config
	if err != nil {
		fmt.Fprintln(os.Stderr, " Error from decode config on", fullFilename, "is", err)
		os.Exit(1)
	}
	imgFileHandle.Close()

	var width = float32(imgConfig.Width)
	var height = float32(imgConfig.Height)
	var aspectRatio = width/height
	if aspectRatio > 1 {
		aspectRatio = 1/aspectRatio
	}

	fmt.Println(" image.Config", imgfilename, fullFilename, basefilename, "width =", width, ", height =", height, "and aspect ratio =", aspectRatio)

	if width > maxWidth || height > maxHeight {
		width = maxWidth * aspectRatio
		height = maxHeight * aspectRatio
	}

	fmt.Println()
	//fmt.Printf(" Type for DecodeConfig is %T \n", imgConfig) // answer is image.Config
	fmt.Println(" adjusted image.Config width =", width, ", height =", height, " but these values are not used to show the image.")
	fmt.Println()

	a := app.New()
	w := a.NewWindow(str)

	//cwd, _ := os.Getwd()
	cwd := filepath.Dir(fullFilename)
	imageURI := storage.NewFileURI(fullFilename) // needs to be a type = fyne.CanvasObject
	imgRead, err := storage.Reader(imageURI)
	if err != nil {
		fmt.Fprintln(os.Stderr, " Error from storage.Reader of", fullFilename, "is", err)
		os.Exit(1)
	}
	defer imgRead.Close()
	img, imgFmtName, err := image.Decode(imgRead)  // imgFmtName is a string of the format name used during format registration by the init function.
	bounds := img.Bounds()
	imgHeight := bounds.Max.Y
	imgWidth := bounds.Max.X
	fmt.Println(" image.Decode, width=", imgWidth, "and height=", imgHeight, ", imgFmtName=", imgFmtName, "and cwd=", cwd)
	fmt.Println()
	if imgWidth > maxWidth {
		img = resize.Resize(maxWidth, 0, img, resize.Lanczos3)
	} else if imgHeight > maxHeight {
		img = resize.Resize(0, maxHeight, img, resize.Lanczos3)
	}

	label := canvas.NewText(imgfilename, color.Gray{128})  // MakeImageItem in orig viewerf1.go
	label.Alignment = fyne.TextAlignCenter                    // MakeImageItem in orig viewerf1.go
	//bgColor := &color.NRGBA{R: 255, G: 255, B: 255, A: 224}  // MakeImageItem in orig viewerf1.go
	//bg := canvas.NewRectangle(bgColor)  // MakeImageItem in orig viewerf1.go
	//fade := canvas.NewLinearGradient(color.Transparent, bgColor, 0)  // MakeImageItem in orig viewerf1.go
	//imagelayoutliteral := imgLayout{text: label, bg: bg, gradient: fade}  // MakeImageItem in orig viewerf1.go, but this syntax not allowed here.

	loadedimg := canvas.NewImageFromImage(img) // from web, not in original viewerf1.go
	loadedimg.FillMode = canvas.ImageFillContain      // loadimage in orig viewerf1.go

	//imgcontainer := container.New(&imgLayout{text: label, bg: bg, gradient: fade}, loadedimg, bg, fade, label) //MakeImageItem in viewerf1.go
	imgContainer := container.New(layout.NewVBoxLayout(), label, loadedimg)

	//picContainer := container.New(layout.NewVBoxLayout(), label, imgRect)  from my notes of the book.  Not needed now.
	//imgRect := canvas.NewRectangle(color.Black)
	//imgPic := canvas.NewImageFromResource(imageURI)  // code above has nil here
	//imgPic.FillMode = canvas.ImageFillContain
	//imgAndTitle := container.NewBorder(nil, label, nil, nil, imgRect) // top, left and right are all nil here.  bottom=label, center=imgRect

	imgFileInfoChan := make(chan []os.FileInfo)  // unbuffered channel
	go MyReadDirForImages(cwd, imgFileInfoChan)
	w.SetContent(imgContainer)
	imgtitle := fmt.Sprintf("%s, %d x %d", imgfilename, imgWidth, imgHeight)
	w.SetTitle(imgtitle)
	w.SetContent(loadedimg)
	w.Resize(fyne.NewSize(float32(imgWidth), float32(imgHeight)))

	var imageInfo []os.FileInfo

	select { // this syntax works and is blocking.
	case imageInfo = <- imgFileInfoChan :  // this ackward syntax is what's needed to read from a channel.
	}

	t0:= time.Now()
	sortfcn := func(i, j int) bool { // this is a closure anonymous function
		return imageInfo[i].ModTime().After(imageInfo[j].ModTime()) // I want a newest-first sort.  Changed 12/20/20
	}
	sort.Slice(imageInfo, sortfcn)
	elapsedtime := time.Since(t0)

	fmt.Printf(" Length of the image fileinfo slice is %d, and sorted in %s\n", len(imageInfo), elapsedtime.String())
	fmt.Println()

	t0 = time.Now()
	indexchan := make(chan int)
	index := 0
	go filenameIndex(imageInfo, basefilename, indexchan)
	select {
	case index = <- indexchan:
	}
	elapsedtime = time.Since(t0)

	fmt.Printf(" %s index is %d in the fileinfo slice; linear sequential search took %s.\n", basefilename, index, elapsedtime)
	fmt.Printf(" As a check, imageInfo[%d] = %s.\n", index, imageInfo[index].Name())
	fmt.Println()

	w.CenterOnScreen()
	w.ShowAndRun()

} // end main

// -------------------------------          filenameIndex --------------------------------------
func filenameIndex(fileinfos []os.FileInfo, name string, intchan chan int) {
	for i, fi := range fileinfos {
		if fi.Name() == name {
			intchan <- i
			return
		}
	}
	intchan <- -1
	return
}

// ----------------------------------isImage ----------------------------------------------
func isImage(file string) bool {
	ext := strings.ToLower(filepath.Ext(file))
	ext = strings.ToLower(ext)

	return ext == ".png" || ext == ".jpg" || ext == ".jpeg" || ext == ".gif"
}

// ------------------------------- MyReadDirForImages -----------------------------------

func MyReadDirForImages(dir string, imageInfoChan chan []os.FileInfo) {

	dirname, err := os.Open(dir)
	if err != nil {
		return
	}
	defer dirname.Close()

	names, err := dirname.Readdirnames(0) // zero means read all names into the returned []string
	if err != nil {
		return
	}

	fi := make([]os.FileInfo, 0, len(names))
	for _, name := range names {
            if isImage(name) {
		imgInfo, err := os.Lstat(name)
		if err != nil {
			fmt.Fprintln(os.Stderr, " Error from os.Lstat ", err)
			continue
		}
		fi = append(fi, imgInfo)
            }
	}

	imageInfoChan <- fi
	return
} // MyReadDirForImages

<commit_msg>08/18/2021 05:10:27 PM  On branch img  Changes to be committed: 	modified:   img/img.go<commit_after>// From Go GUI with Fyne, Chap 4.  I believe it will be enhanced in later chapters, but this is what is it for now.
/*
REVISION HISTORY
-------- -------
 9 Aug 21 -- I realized that this will not be enhanced, as I went thru more of the book.  I'll have to enhance it myself.
             First, I'm changing the function constants to the version that's more readable to me.  That's working, but I had to
             import more parts of fyne.io than the unmodified version.
12 Aug 21 -- Now called img.go, so I can display 1 image.  I'll start here.
13 Aug 21 -- Now called imgfyne.go.  Same purpose as img.go, but so I can test non-fyne code there and fyne code here.
15 Aug 21 -- Copied back to img.go after the code works and displays a single image from the command line.
               Will use imgfyne to add the arrow key navigation and img display.
*/

package main

import (
	"flag"
	"fmt"
	//w "fyne.io/fyne/v2/internal/widget"
	//"fyne.io/fyne/v2/layout"
	"image"
	//"image/color"
	_ "image/gif"
	_ "image/jpeg"
	_ "image/png"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"strings"
	"time"

	//_ "golang.org/x/image"

	"fyne.io/fyne/v2"
	"fyne.io/fyne/v2/app"
	"fyne.io/fyne/v2/canvas"
	//"fyne.io/fyne/v2/container"
	"fyne.io/fyne/v2/storage"

	"github.com/nfnt/resize"
)

const LastModified = "August 18, 2021"
const maxWidth = 2500
const maxHeight = 2000

var index int
var loadedimg *canvas.Image
var cwd string
var imageInfo []os.FileInfo
var a fyne.App

func isNotImageStr(name string) bool {
	ext := filepath.Ext(name)
	isImage := ext == ".png" || ext == ".jpg" || ext == ".jpeg" || ext == ".gif" || ext == ".webp"
	return !isImage
}

func main() {
	flag.Parse()
	if flag.NArg() < 1 {
		fmt.Fprintln(os.Stderr, " Usage: img <image file name>")
		os.Exit(1)
	}

	str := fmt.Sprintf("Single Image Viewer last modified %s, compiled using %s", LastModified, runtime.Version())
	fmt.Println(str) // this works as intended

	imgfilename := flag.Arg(0)
	_, err := os.Stat(imgfilename)
	if err != nil {
		fmt.Fprintln(os.Stderr, " Error from os.Stat(", imgfilename, ") is", err)
		os.Exit(1)
	}

	if isNotImageStr(imgfilename) {
		fmt.Fprintln(os.Stderr, imgfilename, "does not have an image extension.")
		os.Exit(1)
	}

	basefilename := filepath.Base(imgfilename)
	fullFilename, err := filepath.Abs(imgfilename)
	if err != nil {
		fmt.Fprintln(os.Stderr, " Error from filepath.Abs on", imgfilename, "is", err)
		os.Exit(1)
	}

	imgFileHandle, err := os.Open(fullFilename)
	if err != nil {
		fmt.Fprintln(os.Stderr, " Error from opening", fullFilename, "is", err)
		os.Exit(1)
	}

	imgConfig, _, err := image.DecodeConfig(imgFileHandle) // img is of type image.Config
	if err != nil {
		fmt.Fprintln(os.Stderr, " Error from decode config on", fullFilename, "is", err)
		os.Exit(1)
	}
	imgFileHandle.Close()

	var width = float32(imgConfig.Width)
	var height = float32(imgConfig.Height)
	var aspectRatio = width/height
	if aspectRatio > 1 {
		aspectRatio = 1/aspectRatio
	}

	fmt.Println(" image.Config", imgfilename, fullFilename, basefilename, "width =", width, ", height =", height, "and aspect ratio =", aspectRatio)

	if width > maxWidth || height > maxHeight {
		width = maxWidth * aspectRatio
		height = maxHeight * aspectRatio
	}

	fmt.Println()
	//fmt.Printf(" Type for DecodeConfig is %T \n", imgConfig) // answer is image.Config
	fmt.Println(" adjusted image.Config width =", width, ", height =", height, " but these values are not used to show the image.")
	fmt.Println()

	cwd = filepath.Dir(fullFilename)
	imageURI := storage.NewFileURI(fullFilename) // needs to be a type = fyne.CanvasObject
	imgRead, err := storage.Reader(imageURI)
	if err != nil {
		fmt.Fprintln(os.Stderr, " Error from storage.Reader of", fullFilename, "is", err)
		os.Exit(1)
	}
	defer imgRead.Close()
	img, imgFmtName, err := image.Decode(imgRead)  // imgFmtName is a string of the format name used during format registration by the init function.
	if err != nil {
		fmt.Fprintln(os.Stderr, " Error from image.Decode is", err)
		os.Exit(1)
	}
	bounds := img.Bounds()
	imgHeight := bounds.Max.Y
	imgWidth := bounds.Max.X
	fmt.Println(" image.Decode, width=", imgWidth, "and height=", imgHeight, ", imgFmtName=", imgFmtName, "and cwd=", cwd)
	fmt.Println()
	if imgWidth > maxWidth {
		img = resize.Resize(maxWidth, 0, img, resize.Lanczos3)
	} else if imgHeight > maxHeight {
		img = resize.Resize(0, maxHeight, img, resize.Lanczos3)
	}

	imgFileInfoChan := make(chan []os.FileInfo)  // unbuffered channel
	go MyReadDirForImages(cwd, imgFileInfoChan)

	loadedimg = canvas.NewImageFromImage(img)
	loadedimg.FillMode = canvas.ImageFillContain

	a = app.New()
	w := a.NewWindow(str)
	w.Canvas().SetOnTypedKey(keyTyped)

	imgtitle := fmt.Sprintf("%s, %d x %d", imgfilename, imgWidth, imgHeight)
	w.SetTitle(imgtitle)
	w.Resize(fyne.NewSize(float32(imgWidth), float32(imgHeight)))

	select { // this syntax works and is blocking.
	case imageInfo = <- imgFileInfoChan :  // this ackward syntax is what's needed to read from a channel.
	}

	t0:= time.Now()
	sortfcn := func(i, j int) bool { // this is a closure anonymous function
		return imageInfo[i].ModTime().After(imageInfo[j].ModTime()) // I want a newest-first sort.  Changed 12/20/20
	}
	sort.Slice(imageInfo, sortfcn)
	elapsedtime := time.Since(t0)

	fmt.Printf(" Length of the image fileinfo slice is %d, and sorted in %s\n", len(imageInfo), elapsedtime.String())
	fmt.Println()

	t0 = time.Now()
	indexchan := make(chan int)

	go filenameIndex(imageInfo, basefilename, indexchan)
	select {
	case index = <- indexchan:
	}
	elapsedtime = time.Since(t0)

	fmt.Printf(" %s index is %d in the fileinfo slice; linear sequential search took %s.\n", basefilename, index, elapsedtime)
	fmt.Printf(" As a check, imageInfo[%d] = %s.\n", index, imageInfo[index].Name())
	fmt.Println()

	w.CenterOnScreen()
	w.ShowAndRun()

} // end main

// --------------------------------------------------- loadTheImage ------------------------------
func loadTheImage(indx int) *canvas.Image {
	imgname := imageInfo[indx].Name()
	fullfilename := cwd + string(filepath.Separator) + imgname
	imageURI := storage.NewFileURI(fullfilename)
	imgRead, err := storage.Reader(imageURI)
	defer imgRead.Close()
	if err != nil {
		fmt.Fprintln(os.Stderr, " Error from storage.Reader of", fullfilename, "is", err)
		os.Exit(1)
	}

	img, imgFmtName, err := image.Decode(imgRead)  // imgFmtName is a string of the format name used during format registration by the init function.
	if err != nil {
		fmt.Fprintln(os.Stderr, " Error from image.Decode is", err)
		os.Exit(1)
	}
	bounds := img.Bounds()
	imgHeight := bounds.Max.Y
	imgWidth := bounds.Max.X
	fmt.Println( imgname, " image.Decode, width=", imgWidth, "and height=", imgHeight, ", imgFmtName=", imgFmtName, "and cwd=", cwd)
	fmt.Println()
	if imgWidth > maxWidth {
		img = resize.Resize(maxWidth, 0, img, resize.Lanczos3)
	} else if imgHeight > maxHeight {
		img = resize.Resize(0, maxHeight, img, resize.Lanczos3)
	}
	loadedimg = canvas.NewImageFromImage(img)
	loadedimg.FillMode = canvas.ImageFillContain

	return loadedimg
}

// ------------------------------- filenameIndex --------------------------------------
func filenameIndex(fileinfos []os.FileInfo, name string, intchan chan int) {
	for i, fi := range fileinfos {
		if fi.Name() == name {
			intchan <- i
			return
		}
	}
	intchan <- -1
	return
}

// ----------------------------------isImage ----------------------------------------------
func isImage(file string) bool {
	ext := strings.ToLower(filepath.Ext(file))
	ext = strings.ToLower(ext)

	return ext == ".png" || ext == ".jpg" || ext == ".jpeg" || ext == ".gif"
}

// ------------------------------- MyReadDirForImages -----------------------------------

func MyReadDirForImages(dir string, imageInfoChan chan []os.FileInfo) {

	dirname, err := os.Open(dir)
	if err != nil {
		return
	}
	defer dirname.Close()

	names, err := dirname.Readdirnames(0) // zero means read all names into the returned []string
	if err != nil {
		return
	}

	fi := make([]os.FileInfo, 0, len(names))
	for _, name := range names {
            if isImage(name) {
		imgInfo, err := os.Lstat(name)
		if err != nil {
			fmt.Fprintln(os.Stderr, " Error from os.Lstat ", err)
			continue
		}
		fi = append(fi, imgInfo)
            }
	}

	imageInfoChan <- fi
	return
} // MyReadDirForImages

// ---------------------------------------------- nextImage -----------------------------------------------------
func nextImage(indx int) *canvas.Image {

	indx++
	if indx >= len(imageInfo) {
		indx--
	}
	return loadTheImage(indx)

} // end nextImage

// ------------------------------------------ prevImage -------------------------------------------------------
func prevImage(indx int) *canvas.Image {
	indx--
	if indx < 0 {
		indx++
	}
	return loadTheImage(indx)
} // end prevImage

// ------------------------------------------------------------ keyTyped ------------------------------
func keyTyped(e *fyne.KeyEvent) { // index is a global var
	switch e.Name {
	case fyne.KeyUp:
		prevImage(index)
	case fyne.KeyDown:
		nextImage(index)
	case fyne.KeyLeft:
		prevImage(index)
	case fyne.KeyRight:
		nextImage(index)
	case fyne.KeyEscape:
		a.Quit()
	}
}

<|endoftext|>


<commit_before>package executor

import (
	"fmt"
	"io/ioutil"
	"log"
	"pilosa/config"
	"pilosa/core"
	"pilosa/db"
	"pilosa/query"
	"pilosa/util"
	"tux21b.org/v1/gocql/uuid"

	"github.com/davecgh/go-spew/spew"
)

type Executor struct {
	service *core.Service
	inbox   chan *db.Message
}

func (self *Executor) Init() error {
	log.Println("Starting Executor")
	return nil
}

func (self *Executor) Close() {
	log.Println("Shutting down Executor")
}

func (self *Executor) NewJob(job *db.Message) {
	switch job.Data.(type) {
	case query.CountQueryStep:
		self.service.CountQueryStepHandler(job)
	case query.UnionQueryStep:
		self.service.UnionQueryStepHandler(job)
	case query.IntersectQueryStep:
		self.service.IntersectQueryStepHandler(job)
	case query.CatQueryStep:
		self.service.CatQueryStepHandler(job)
	case query.GetQueryStep:
		self.service.GetQueryStepHandler(job)
	case query.SetQueryStep:
		self.service.SetQueryStepHandler(job)
	default:
		fmt.Println("unknown")
	}
}

type stringSlice []string

func (slice stringSlice) pos(value string) int {
	for p, v := range slice {
		if v == value {
			return p
		}
	}
	return -1
}

func (self *Executor) RunQueryTest(database_name string, pql string) string {
	return pql
}

type queryItem struct {
	label string
	pql   string
}
type queryItemCopy struct {
	id    *uuid.UUID
	label string
}
type QueryItemResult struct {
	Label  string
	Result interface{}
}

func (self *Executor) runQuery(database *db.Database, qry *query.Query) {
	process, err := self.service.GetProcess()
	if err != nil {
		spew.Dump(err)
	}
	process_id := process.Id()
	fragment_id := util.SUUID(0)
	destination := db.Location{&process_id, fragment_id}

	spew.Dump("QUERY.ID:", qry.Id)
	query_plan := query.QueryPlanForQuery(database, qry, &destination)
	// loop over the query steps and send to Transport
	for _, qs := range *query_plan {
		msg := new(db.Message)
		msg.Data = qs
		switch step := qs.(type) {
		case query.PortableQueryStep:
			self.service.Transport.Send(msg, step.GetLocation().ProcessId)
		}
	}
}

func (self *Executor) RunPQL(database_name string, pql string) interface{} {
	database := self.service.Cluster.GetOrCreateDatabase(database_name)

	// see if the outer query function is a custom query
	reserved_functions := stringSlice{"get", "set", "union", "intersect", "count"}
	tokens := query.Lex(pql)
	outer_token := tokens[0].Text

	if reserved_functions.pos(outer_token) != -1 {

		qry := query.QueryForTokens(tokens)
		go self.runQuery(database, qry)

		var final interface{}
		final, err := self.service.Hold.Get(qry.Id, 10)
		if err != nil {
			spew.Dump(err)
		}
		return final

	} else {
		macros_dir := config.Get("macros").(string)
		macros_file := macros_dir + "/" + outer_token + ".js"
		file_data, err := ioutil.ReadFile(macros_file)
		if err != nil {
			spew.Dump(err)
		}
		spew.Dump(file_data)

		// CUSTOM QUERY LIST ///////////////
		var query_list []queryItem
		query_list = append(query_list, queryItem{"set1", "set(20, 1)"})
		query_list = append(query_list, queryItem{"set2", "set(20, 1)"})
		query_list = append(query_list, queryItem{"set3", "set(20, 2)"})
		query_list = append(query_list, queryItem{"set4", "set(20, 3)"})
		query_list = append(query_list, queryItem{"set5", "set(20, 4)"})
		query_list = append(query_list, queryItem{"count", "count(get(20))"})
		var query_list_copy []*queryItemCopy
		////////////////////////////////////

		for _, qi := range query_list {
			qry := query.QueryForPQL(qi.pql)
			go self.runQuery(database, qry)
			query_list_copy = append(query_list_copy, &queryItemCopy{qry.Id, qi.label})
		}

		var final_result []*QueryItemResult
		for _, qlc := range query_list_copy {
			final, err := self.service.Hold.Get(qlc.id, 10)
			if err != nil {
				spew.Dump(err)
			}
			spew.Dump(final)
			final_result = append(final_result, &QueryItemResult{qlc.label, final})
		}

		return final_result
	}

}

func (self *Executor) Run() {
	log.Println("Executor Run...")
}

func NewExecutor(service *core.Service) *Executor {
	return &Executor{service, make(chan *db.Message)}
}
<commit_msg>removed some of the slice range copy<commit_after>package executor

import (
	"fmt"
	"io/ioutil"
	"log"
	"pilosa/config"
	"pilosa/core"
	"pilosa/db"
	"pilosa/query"
	"pilosa/util"
	"tux21b.org/v1/gocql/uuid"

	"github.com/davecgh/go-spew/spew"
)

type Executor struct {
	service *core.Service
	inbox   chan *db.Message
}

func (self *Executor) Init() error {
	log.Println("Starting Executor")
	return nil
}

func (self *Executor) Close() {
	log.Println("Shutting down Executor")
}

func (self *Executor) NewJob(job *db.Message) {
	switch job.Data.(type) {
	case query.CountQueryStep:
		self.service.CountQueryStepHandler(job)
	case query.UnionQueryStep:
		self.service.UnionQueryStepHandler(job)
	case query.IntersectQueryStep:
		self.service.IntersectQueryStepHandler(job)
	case query.CatQueryStep:
		self.service.CatQueryStepHandler(job)
	case query.GetQueryStep:
		self.service.GetQueryStepHandler(job)
	case query.SetQueryStep:
		self.service.SetQueryStepHandler(job)
	default:
		fmt.Println("unknown")
	}
}

type stringSlice []string

func (slice stringSlice) pos(value string) int {
	for p, v := range slice {
		if v == value {
			return p
		}
	}
	return -1
}

func (self *Executor) RunQueryTest(database_name string, pql string) string {
	return pql
}

type queryListItem struct {
	id    *uuid.UUID
	label string
	pql   string
}

func (self *Executor) runQuery(database *db.Database, qry *query.Query) {
	process, err := self.service.GetProcess()
	if err != nil {
		spew.Dump(err)
	}
	process_id := process.Id()
	fragment_id := util.SUUID(0)
	destination := db.Location{&process_id, fragment_id}

	spew.Dump("QUERY.ID:", qry.Id)
	query_plan := query.QueryPlanForQuery(database, qry, &destination)
	// loop over the query steps and send to Transport
	for _, qs := range *query_plan {
		msg := new(db.Message)
		msg.Data = qs
		switch step := qs.(type) {
		case query.PortableQueryStep:
			self.service.Transport.Send(msg, step.GetLocation().ProcessId)
		}
	}
}

func (self *Executor) RunPQL(database_name string, pql string) interface{} {
	database := self.service.Cluster.GetOrCreateDatabase(database_name)

	// see if the outer query function is a custom query
	reserved_functions := stringSlice{"get", "set", "union", "intersect", "count"}
	tokens := query.Lex(pql)
	outer_token := tokens[0].Text

	if reserved_functions.pos(outer_token) != -1 {

		qry := query.QueryForTokens(tokens)
		go self.runQuery(database, qry)

		var final interface{}
		final, err := self.service.Hold.Get(qry.Id, 10)
		if err != nil {
			spew.Dump(err)
		}
		return final

	} else {
		macros_dir := config.Get("macros").(string)
		macros_file := macros_dir + "/" + outer_token + ".js"
		file_data, err := ioutil.ReadFile(macros_file)
		if err != nil {
			spew.Dump(err)
		}
		spew.Dump(file_data)

		// CUSTOM QUERY LIST ///////////////
		var query_list []queryListItem
		query_list = append(query_list, queryListItem{label: "set1", pql: "set(20, 1)"})
		query_list = append(query_list, queryListItem{label: "set2", pql: "set(20, 1)"})
		query_list = append(query_list, queryListItem{label: "set3", pql: "set(20, 2)"})
		query_list = append(query_list, queryListItem{label: "set4", pql: "set(20, 3)"})
		query_list = append(query_list, queryListItem{label: "set5", pql: "set(20, 4)"})
		query_list = append(query_list, queryListItem{label: "count", pql: "count(get(20))"})
		////////////////////////////////////

		for i, _ := range query_list {
			qry := query.QueryForPQL(query_list[i].pql)
			go self.runQuery(database, qry)
			query_list[i].id = qry.Id
		}

		// technically, this is blocking on the hold for each query, but it may be ok, since we need them all for the final anyway.
		final_result := make(map[string]interface{})
		for i, _ := range query_list {
			final, err := self.service.Hold.Get(query_list[i].id, 10)
			if err != nil {
				spew.Dump(err)
			}
			spew.Dump(final)
			final_result[query_list[i].label] = final
		}

		return final_result
	}

}

func (self *Executor) Run() {
	log.Println("Executor Run...")
}

func NewExecutor(service *core.Service) *Executor {
	return &Executor{service, make(chan *db.Message)}
}
<|endoftext|>


<commit_before>package executor

import (
	"fmt"
	"log"
	"pilosa/config"
	"pilosa/core"
	"pilosa/db"
	"pilosa/query"
	"pilosa/util"

	"github.com/davecgh/go-spew/spew"
)

type Executor struct {
	service *core.Service
	inbox   chan *db.Message
}

func (self *Executor) Init() error {
	log.Println("Starting Executor")
	return nil
}

func (self *Executor) Close() {
	log.Println("Shutting down Executor")
}

func (self *Executor) NewJob(job *db.Message) {
	switch job.Data.(type) {
	case query.CountQueryStep:
		self.service.CountQueryStepHandler(job)
	case query.TopNQueryStep:
		self.service.TopNQueryStepHandler(job)
	case query.UnionQueryStep:
		self.service.UnionQueryStepHandler(job)
	case query.IntersectQueryStep:
		self.service.IntersectQueryStepHandler(job)
	case query.CatQueryStep:
		self.service.CatQueryStepHandler(job)
	case query.GetQueryStep:
		self.service.GetQueryStepHandler(job)
	case query.SetQueryStep:
		self.service.SetQueryStepHandler(job)
	default:
		fmt.Println("unknown")
	}
}

type stringSlice []string

func (slice stringSlice) pos(value string) int {
	for p, v := range slice {
		if v == value {
			return p
		}
	}
	return -1
}

func (self *Executor) RunQueryTest(database_name string, pql string) string {
	return pql
}

func (self *Executor) runQuery(database *db.Database, qry *query.Query) error {
	process, err := self.service.GetProcess()
	if err != nil {
		return err
	}
	process_id := process.Id()
	fragment_id := util.SUUID(0)
	destination := db.Location{&process_id, fragment_id}

	query_plan, err := query.QueryPlanForQuery(database, qry, &destination)
	if err != nil {
		switch obj := err.(type) {
		//case *query.InvalidFrame:
		case *query.FragmentNotFound:
			self.service.TopologyMapper.MakeFragments(obj.Db, obj.Slice)
		}
		self.service.Hold.Set(qry.Id, err, 30)
		return err
	}
	// loop over the query steps and send to Transport
	for _, qs := range *query_plan {
		msg := new(db.Message)
		msg.Data = qs
		switch step := qs.(type) {
		case query.PortableQueryStep:
			self.service.Transport.Send(msg, step.GetLocation().ProcessId)
		}
	}
	return nil
}

func (self *Executor) RunPQL(database_name string, pql string) (interface{}, error) {
	database := self.service.Cluster.GetOrCreateDatabase(database_name)

	// see if the outer query function is a custom query
	reserved_functions := stringSlice{"get", "set", "union", "intersect", "count", "top-n"}
	tokens, err := query.Lex(pql)
	if err != nil {
		return nil, err
	}
	outer_token := tokens[0].Text

	if reserved_functions.pos(outer_token) != -1 {

		qry, err := query.QueryForTokens(tokens)
		if err != nil {
			return nil, err
		}
		go self.runQuery(database, qry)

		var final interface{}
		final, err = self.service.Hold.Get(qry.Id, 10)
		if err != nil {
			return nil, err
		}
		return final, nil

	} else {
		plugins_dir := config.GetString("plugins")
		plugins_file := plugins_dir + "/" + outer_token + ".js"
		filter := query.TokensToString(tokens)
		query_list := GetMacro(plugins_file, filter).(query.PqlList)

		for i, _ := range query_list {
			qry, err := query.QueryForPQL(query_list[i].PQL)
			if err != nil {
				return nil, err
			}
			go self.runQuery(database, qry)
			query_list[i].Id = qry.Id
		}

		// technically, this is blocking on the hold for each query, but it may be ok, since we need them all for the final anyway.
		final_result := make(map[string]interface{})
		for i, _ := range query_list {
			final, err := self.service.Hold.Get(query_list[i].Id, 10)
			if err != nil {
				spew.Dump(err)
			}
			final_result[query_list[i].Label] = final
		}

		return final_result, nil
	}

}

func (self *Executor) Run() {
	log.Println("Executor Run...")
}

func NewExecutor(service *core.Service) *Executor {
	return &Executor{service, make(chan *db.Message)}
}
<commit_msg>made hold retrival async from exector<commit_after>package executor

import (
	"fmt"
	"log"
	"pilosa/config"
	"pilosa/core"
	"pilosa/db"
	"pilosa/query"
	"pilosa/util"

	"github.com/davecgh/go-spew/spew"
)

type Executor struct {
	service *core.Service
	inbox   chan *db.Message
}

func (self *Executor) Init() error {
	log.Println("Starting Executor")
	return nil
}

func (self *Executor) Close() {
	log.Println("Shutting down Executor")
}

func (self *Executor) NewJob(job *db.Message) {
	switch job.Data.(type) {
	case query.CountQueryStep:
		self.service.CountQueryStepHandler(job)
	case query.TopNQueryStep:
		self.service.TopNQueryStepHandler(job)
	case query.UnionQueryStep:
		self.service.UnionQueryStepHandler(job)
	case query.IntersectQueryStep:
		self.service.IntersectQueryStepHandler(job)
	case query.CatQueryStep:
		self.service.CatQueryStepHandler(job)
	case query.GetQueryStep:
		self.service.GetQueryStepHandler(job)
	case query.SetQueryStep:
		self.service.SetQueryStepHandler(job)
	default:
		fmt.Println("unknown")
	}
}

type stringSlice []string

func (slice stringSlice) pos(value string) int {
	for p, v := range slice {
		if v == value {
			return p
		}
	}
	return -1
}

func (self *Executor) RunQueryTest(database_name string, pql string) string {
	return pql
}

func (self *Executor) runQuery(database *db.Database, qry *query.Query) error {
	process, err := self.service.GetProcess()
	if err != nil {
		return err
	}
	process_id := process.Id()
	fragment_id := util.SUUID(0)
	destination := db.Location{&process_id, fragment_id}

	query_plan, err := query.QueryPlanForQuery(database, qry, &destination)
	if err != nil {
		switch obj := err.(type) {
		//case *query.InvalidFrame:
		case *query.FragmentNotFound:
			self.service.TopologyMapper.MakeFragments(obj.Db, obj.Slice)
		}
		self.service.Hold.Set(qry.Id, err, 30)
		return err
	}
	// loop over the query steps and send to Transport
	for _, qs := range *query_plan {
		msg := new(db.Message)
		msg.Data = qs
		switch step := qs.(type) {
		case query.PortableQueryStep:
			self.service.Transport.Send(msg, step.GetLocation().ProcessId)
		}
	}
	return nil
}

func (self *Executor) RunPQL(database_name string, pql string) (interface{}, error) {
	database := self.service.Cluster.GetOrCreateDatabase(database_name)

	// see if the outer query function is a custom query
	reserved_functions := stringSlice{"get", "set", "union", "intersect", "count", "top-n"}
	tokens, err := query.Lex(pql)
	if err != nil {
		return nil, err
	}
	outer_token := tokens[0].Text

	if reserved_functions.pos(outer_token) != -1 {

		qry, err := query.QueryForTokens(tokens)
		if err != nil {
			return nil, err
		}
		go self.runQuery(database, qry)

		var final interface{}
		final, err = self.service.Hold.Get(qry.Id, 10)
		if err != nil {
			return nil, err
		}
		return final, nil

	} else {
		plugins_dir := config.GetString("plugins")
		plugins_file := plugins_dir + "/" + outer_token + ".js"
		filter := query.TokensToString(tokens)
		query_list := GetMacro(plugins_file, filter).(query.PqlList)

		for i, _ := range query_list {
			qry, err := query.QueryForPQL(query_list[i].PQL)
			if err != nil {
				return nil, err
			}
			go self.runQuery(database, qry)
			query_list[i].Id = qry.Id
		}

		final_result := make(map[string]interface{})
		result := make(chan struct {
			final interface{}
			label string
			err   error
		})
		x := 0
		for i, _ := range query_list {
			x++
			go func(q query.PqlListItem, reply chan struct {
				final interface{}
				label string
				err   error
			}) {
				final, err := self.service.Hold.Get(q.Id, 10)
				result <- struct {
					final interface{}
					label string
					err   error
				}{final, q.Label, err}
			}(query_list[i], result)
			if err != nil {
				spew.Dump(err)
			}
		}
		for z := 0; z < x; z++ {
			ans := <-result
			final_result[ans.label] = ans.final
		}

		return final_result, nil
	}

}

func (self *Executor) Run() {
	log.Println("Executor Run...")
}

func NewExecutor(service *core.Service) *Executor {
	return &Executor{service, make(chan *db.Message)}
}
<|endoftext|>


<commit_before>/*
 Copyright 2020 The GoPlus Authors (goplus.org)

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package parser

import (
	"bytes"
	"errors"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/qiniu/goplus/ast"
	"github.com/qiniu/goplus/scanner"
	"github.com/qiniu/goplus/token"
)

// -----------------------------------------------------------------------------

// FileSystem represents a file system.
type FileSystem interface {
	ReadDir(dirname string) ([]os.FileInfo, error)
	ReadFile(filename string) ([]byte, error)
	Join(elem ...string) string
}

type localFS struct{}

func (p localFS) ReadDir(dirname string) ([]os.FileInfo, error) {
	return ioutil.ReadDir(dirname)
}

func (p localFS) ReadFile(filename string) ([]byte, error) {
	return ioutil.ReadFile(filename)
}

func (p localFS) Join(elem ...string) string {
	return filepath.Join(elem...)
}

var local FileSystem = localFS{}

// ParseGOPFiles parses the Go+ source files under directory or single Go+ source file.
// The target specifies the directory or single Go+ source file.
//
// The ParseGOPFiles should return the map of packages to run Go+ script, even the target is single file.
//
// If the file or directory couldn't be read, a nil map and the respective error are
// returned.
// If the target is directory and a parse error occurred, a non-nil but incomplete map and the
// first error encountered are returned.
func ParseGopFiles(fset *token.FileSet, target string, mode Mode) (pkgs map[string]*ast.Package, err error) {
	if strings.HasSuffix(target, ".gop") {
		file, err := ParseFile(fset, target, nil, mode)
		if err != nil {
			return pkgs, err
		}
		pkgs = make(map[string]*ast.Package)
		pkg := astFileToPkg(file, target)
		pkgs[file.Name.Name] = pkg
		return pkgs, nil
	}
	return ParseDir(fset, target, nil, mode)
}

// -----------------------------------------------------------------------------
// ParseDir calls ParseFSDir by passing a local filesystem.
//
func ParseDir(fset *token.FileSet, path string, filter func(os.FileInfo) bool, mode Mode) (pkgs map[string]*ast.Package, first error) {
	return ParseFSDir(fset, local, path, filter, mode)
}

// ParseFSDir calls ParseFile for all files with names ending in ".gop" in the
// directory specified by path and returns a map of package name -> package
// AST with all the packages found.
//
// If filter != nil, only the files with os.FileInfo entries passing through
// the filter (and ending in ".gop") are considered. The mode bits are passed
// to ParseFile unchanged. Position information is recorded in fset, which
// must not be nil.
//
// If the directory couldn't be read, a nil map and the respective error are
// returned. If a parse error occurred, a non-nil but incomplete map and the
// first error encountered are returned.
//
func ParseFSDir(fset *token.FileSet, fs FileSystem, path string, filter func(os.FileInfo) bool, mode Mode) (pkgs map[string]*ast.Package, first error) {
	list, err := fs.ReadDir(path)
	if err != nil {
		return nil, err
	}
	pkgs = make(map[string]*ast.Package)
	for _, d := range list {
		if strings.HasSuffix(d.Name(), ".gop") && (filter == nil || filter(d)) {
			filename := fs.Join(path, d.Name())
			if filedata, err := fs.ReadFile(filename); err == nil {
				if src, err := ParseFSFile(fset, fs, filename, filedata, mode); err == nil {
					name := src.Name.Name
					pkg, found := pkgs[name]
					if !found {
						pkg = &ast.Package{
							Name:  name,
							Files: make(map[string]*ast.File),
						}
						pkgs[name] = pkg
					}
					pkg.Files[filename] = src
				} else if first == nil {
					first = err
				}
			} else if first == nil {
				first = err
			}
		}
	}
	return
}

// -----------------------------------------------------------------------------

// ParseFile parses the source code of a single Go+ source file and returns the corresponding ast.File node.
func ParseFile(fset *token.FileSet, filename string, src interface{}, mode Mode) (f *ast.File, err error) {
	return ParseFSFile(fset, local, filename, src, mode)
}

// ParseFSFile parses the source code of a single Go+ source file and returns the corresponding ast.File node.
func ParseFSFile(fset *token.FileSet, fs FileSystem, filename string, src interface{}, mode Mode) (f *ast.File, err error) {
	var code []byte
	if src == nil {
		code, err = fs.ReadFile(filename)
	} else {
		code, err = readSource(src)
	}
	if err != nil {
		return
	}
	return parseFileEx(fset, filename, code, mode)
}

// TODO: should not add package info and init|main function.
// If do this, parsing will display error line number when error occur
func parseFileEx(fset *token.FileSet, filename string, code []byte, mode Mode) (f *ast.File, err error) {
	var b []byte
	var isMod, hasUnnamed bool
	var fsetTmp = token.NewFileSet()
	f, err = parseFile(fsetTmp, filename, code, PackageClauseOnly)
	if err != nil {
		n := len(code)
		b = make([]byte, n+28)
		copy(b, "package main;")
		copy(b[13:], code)
		code = b[:n+13]
	} else {
		isMod = f.Name.Name != "main"
	}
	_, err = parseFile(fsetTmp, filename, code, mode)
	if err != nil {
		if errlist, ok := err.(scanner.ErrorList); ok {
			if e := errlist[0]; strings.HasPrefix(e.Msg, "expected declaration") {
				n := len(code)
				idx := e.Pos.Offset
				if b == nil {
					b = make([]byte, n+14)
					copy(b, code[:idx])
				}
				copy(b[idx+12:], code[idx:n])
				if isMod {
					copy(b[idx:], "func init(){")
				} else {
					copy(b[idx:], "func main(){")
				}
				b[n+12] = '\n'
				b[n+13] = '}'
				code = b[:n+14]
				hasUnnamed = true
				err = nil
			}
		}
	}
	if err == nil {
		f, err = parseFile(fset, filename, code, mode)
		if err == nil {
			f.HasUnnamed = hasUnnamed
		}
	}
	return
}

var (
	errInvalidSource = errors.New("invalid source")
)

func readSource(src interface{}) ([]byte, error) {
	switch s := src.(type) {
	case string:
		return []byte(s), nil
	case []byte:
		return s, nil
	case *bytes.Buffer:
		// is io.Reader, but src is already available in []byte form
		if s != nil {
			return s.Bytes(), nil
		}
	case io.Reader:
		return ioutil.ReadAll(s)
	}
	return nil, errInvalidSource
}

// -----------------------------------------------------------------------------

func (p *parser) parseSliceLit(lbrack token.Pos, len ast.Expr) ast.Expr {
	elts := make([]ast.Expr, 1, 8)
	elts[0] = len
	for p.tok == token.COMMA {
		p.next()
		elt := p.parseRHS()
		elts = append(elts, elt)
	}
	rbrack := p.expect(token.RBRACK)
	return &ast.SliceLit{Lbrack: lbrack, Elts: elts, Rbrack: rbrack}
}

func newSliceLit(lbrack, rbrack token.Pos, len ast.Expr) ast.Expr {
	var elts []ast.Expr
	if len != nil {
		elts = []ast.Expr{len}
	}
	return &ast.SliceLit{Lbrack: lbrack, Elts: elts, Rbrack: rbrack}
}

// astFileToPkg translate ast.File to ast.Package
func astFileToPkg(file *ast.File, fileName string) (pkg *ast.Package) {
	pkg = &ast.Package{
		Name:  file.Name.Name,
		Files: make(map[string]*ast.File),
	}
	pkg.Files[fileName] = file
	return
}

// -----------------------------------------------------------------------------
<commit_msg>comments fix<commit_after>/*
 Copyright 2020 The GoPlus Authors (goplus.org)

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package parser

import (
	"bytes"
	"errors"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/qiniu/goplus/ast"
	"github.com/qiniu/goplus/scanner"
	"github.com/qiniu/goplus/token"
)

// -----------------------------------------------------------------------------

// FileSystem represents a file system.
type FileSystem interface {
	ReadDir(dirname string) ([]os.FileInfo, error)
	ReadFile(filename string) ([]byte, error)
	Join(elem ...string) string
}

type localFS struct{}

func (p localFS) ReadDir(dirname string) ([]os.FileInfo, error) {
	return ioutil.ReadDir(dirname)
}

func (p localFS) ReadFile(filename string) ([]byte, error) {
	return ioutil.ReadFile(filename)
}

func (p localFS) Join(elem ...string) string {
	return filepath.Join(elem...)
}

var local FileSystem = localFS{}

// ParseGOPFiles parses the Go+ source files under directory or single Go+ source file.
// The target specifies the directory or single Go+ source file.
//
// The ParseGOPFiles should return the map of packages to run Go+ script, even the target is single file.
//
// If the file or directory couldn't be read, a nil map and the respective error are
// returned.
// If the target is directory and a parse error occurred, a non-nil but incomplete map and the first error encountered are returned.
func ParseGopFiles(fset *token.FileSet, target string, mode Mode) (pkgs map[string]*ast.Package, err error) {
	if strings.HasSuffix(target, ".gop") {
		file, err := ParseFile(fset, target, nil, mode)
		if err != nil {
			return pkgs, err
		}
		pkgs = make(map[string]*ast.Package)
		pkg := astFileToPkg(file, target)
		pkgs[file.Name.Name] = pkg
		return pkgs, nil
	}
	return ParseDir(fset, target, nil, mode)
}

// -----------------------------------------------------------------------------
// ParseDir calls ParseFSDir by passing a local filesystem.
//
func ParseDir(fset *token.FileSet, path string, filter func(os.FileInfo) bool, mode Mode) (pkgs map[string]*ast.Package, first error) {
	return ParseFSDir(fset, local, path, filter, mode)
}

// ParseFSDir calls ParseFile for all files with names ending in ".gop" in the
// directory specified by path and returns a map of package name -> package
// AST with all the packages found.
//
// If filter != nil, only the files with os.FileInfo entries passing through
// the filter (and ending in ".gop") are considered. The mode bits are passed
// to ParseFile unchanged. Position information is recorded in fset, which
// must not be nil.
//
// If the directory couldn't be read, a nil map and the respective error are
// returned. If a parse error occurred, a non-nil but incomplete map and the
// first error encountered are returned.
//
func ParseFSDir(fset *token.FileSet, fs FileSystem, path string, filter func(os.FileInfo) bool, mode Mode) (pkgs map[string]*ast.Package, first error) {
	list, err := fs.ReadDir(path)
	if err != nil {
		return nil, err
	}
	pkgs = make(map[string]*ast.Package)
	for _, d := range list {
		if strings.HasSuffix(d.Name(), ".gop") && (filter == nil || filter(d)) {
			filename := fs.Join(path, d.Name())
			if filedata, err := fs.ReadFile(filename); err == nil {
				if src, err := ParseFSFile(fset, fs, filename, filedata, mode); err == nil {
					name := src.Name.Name
					pkg, found := pkgs[name]
					if !found {
						pkg = &ast.Package{
							Name:  name,
							Files: make(map[string]*ast.File),
						}
						pkgs[name] = pkg
					}
					pkg.Files[filename] = src
				} else if first == nil {
					first = err
				}
			} else if first == nil {
				first = err
			}
		}
	}
	return
}

// -----------------------------------------------------------------------------

// ParseFile parses the source code of a single Go+ source file and returns the corresponding ast.File node.
func ParseFile(fset *token.FileSet, filename string, src interface{}, mode Mode) (f *ast.File, err error) {
	return ParseFSFile(fset, local, filename, src, mode)
}

// ParseFSFile parses the source code of a single Go+ source file and returns the corresponding ast.File node.
func ParseFSFile(fset *token.FileSet, fs FileSystem, filename string, src interface{}, mode Mode) (f *ast.File, err error) {
	var code []byte
	if src == nil {
		code, err = fs.ReadFile(filename)
	} else {
		code, err = readSource(src)
	}
	if err != nil {
		return
	}
	return parseFileEx(fset, filename, code, mode)
}

// TODO: should not add package info and init|main function.
// If do this, parsing will display error line number when error occur
func parseFileEx(fset *token.FileSet, filename string, code []byte, mode Mode) (f *ast.File, err error) {
	var b []byte
	var isMod, hasUnnamed bool
	var fsetTmp = token.NewFileSet()
	f, err = parseFile(fsetTmp, filename, code, PackageClauseOnly)
	if err != nil {
		n := len(code)
		b = make([]byte, n+28)
		copy(b, "package main;")
		copy(b[13:], code)
		code = b[:n+13]
	} else {
		isMod = f.Name.Name != "main"
	}
	_, err = parseFile(fsetTmp, filename, code, mode)
	if err != nil {
		if errlist, ok := err.(scanner.ErrorList); ok {
			if e := errlist[0]; strings.HasPrefix(e.Msg, "expected declaration") {
				n := len(code)
				idx := e.Pos.Offset
				if b == nil {
					b = make([]byte, n+14)
					copy(b, code[:idx])
				}
				copy(b[idx+12:], code[idx:n])
				if isMod {
					copy(b[idx:], "func init(){")
				} else {
					copy(b[idx:], "func main(){")
				}
				b[n+12] = '\n'
				b[n+13] = '}'
				code = b[:n+14]
				hasUnnamed = true
				err = nil
			}
		}
	}
	if err == nil {
		f, err = parseFile(fset, filename, code, mode)
		if err == nil {
			f.HasUnnamed = hasUnnamed
		}
	}
	return
}

var (
	errInvalidSource = errors.New("invalid source")
)

func readSource(src interface{}) ([]byte, error) {
	switch s := src.(type) {
	case string:
		return []byte(s), nil
	case []byte:
		return s, nil
	case *bytes.Buffer:
		// is io.Reader, but src is already available in []byte form
		if s != nil {
			return s.Bytes(), nil
		}
	case io.Reader:
		return ioutil.ReadAll(s)
	}
	return nil, errInvalidSource
}

// -----------------------------------------------------------------------------

func (p *parser) parseSliceLit(lbrack token.Pos, len ast.Expr) ast.Expr {
	elts := make([]ast.Expr, 1, 8)
	elts[0] = len
	for p.tok == token.COMMA {
		p.next()
		elt := p.parseRHS()
		elts = append(elts, elt)
	}
	rbrack := p.expect(token.RBRACK)
	return &ast.SliceLit{Lbrack: lbrack, Elts: elts, Rbrack: rbrack}
}

func newSliceLit(lbrack, rbrack token.Pos, len ast.Expr) ast.Expr {
	var elts []ast.Expr
	if len != nil {
		elts = []ast.Expr{len}
	}
	return &ast.SliceLit{Lbrack: lbrack, Elts: elts, Rbrack: rbrack}
}

// astFileToPkg translate ast.File to ast.Package
func astFileToPkg(file *ast.File, fileName string) (pkg *ast.Package) {
	pkg = &ast.Package{
		Name:  file.Name.Name,
		Files: make(map[string]*ast.File),
	}
	pkg.Files[fileName] = file
	return
}

// -----------------------------------------------------------------------------
<|endoftext|>


<commit_before>package parser

import (
	"github.com/apparentlymart/go-textseg/textseg"
	"github.com/cirbo-lang/cirbo/source"
)

// Token represents a sequence of bytes from some zcl code that has been
// tagged with a type and its range within the source file.
type Token struct {
	Type  TokenType
	Bytes []byte
	Range source.Range
}

// Tokens is a slice of Token.
type Tokens []Token

// TokenType is an enumeration used for the Type field on Token.
type TokenType rune

const (
	// Single-character tokens are represented by their own character, for
	// convenience in producing these within the scanner. However, the values
	// are otherwise arbitrary and just intended to be mnemonic for humans
	// who might see them in debug output.

	TokenOBrace TokenType = '{'
	TokenCBrace TokenType = '}'
	TokenOBrack TokenType = '['
	TokenCBrack TokenType = ']'
	TokenOParen TokenType = '('
	TokenCParen TokenType = ')'

	TokenStar    TokenType = '*'
	TokenSlash   TokenType = '/'
	TokenPlus    TokenType = '+'
	TokenMinus   TokenType = '-'
	TokenPercent TokenType = '%'
	TokenCaret   TokenType = '^'

	TokenAssign        TokenType = '='
	TokenEqual         TokenType = '≔'
	TokenNotEqual      TokenType = '≠'
	TokenLessThan      TokenType = '<'
	TokenLessThanEq    TokenType = '≤'
	TokenGreaterThan   TokenType = '>'
	TokenGreaterThanEq TokenType = '≥'
	TokenDashDash      TokenType = '—'

	TokenAnd  TokenType = '∧'
	TokenOr   TokenType = '∨'
	TokenBang TokenType = '!'

	TokenDot   TokenType = '.'
	TokenComma TokenType = ','

	TokenQuestion  TokenType = '?'
	TokenColon     TokenType = ':'
	TokenSemicolon TokenType = ';'

	TokenStringLit TokenType = 'S'
	TokenNumberLit TokenType = 'N'
	TokenIdent     TokenType = 'I'

	TokenComment    TokenType = 'C'
	TokenWhitespace TokenType = ' '

	TokenEOF TokenType = '␄'

	// The following are not used in the language but recognized by the scanner
	// so we can generate good diagnostics in the parser when users try to write
	// things that might work in other languages they are familiar with, or
	// simply make incorrect assumptions about the cirbo language.

	TokenBitwiseAnd TokenType = '&'
	TokenBitwiseOr  TokenType = '|'
	TokenBitwiseNot TokenType = '~'
	TokenInvalid    TokenType = '�'
	TokenBadUTF8    TokenType = '💩'

	// TokenNil is a placeholder for when a token is required but none is
	// available, e.g. when reporting errors. The scanner will never produce
	// this as part of a token stream.
	TokenNil TokenType = '\x00'
)

/*func (t TokenType) GoString() string {
	return fmt.Sprintf("parser.%s", t.String())
}*/

type scanMode int

const (
	scanNormal scanMode = iota
	scanTemplate
)

type tokenAccum struct {
	Filename string
	Bytes    []byte
	Pos      source.Pos
	Tokens   []Token
}

func (f *tokenAccum) emitToken(ty TokenType, startOfs, endOfs int) {
	// Walk through our buffer to figure out how much we need to adjust
	// the start pos to get our end pos.

	start := f.Pos
	start.Column += startOfs - f.Pos.Byte // Safe because only ASCII whitespace can be in the offset (we count a tab character as one column)
	start.Byte = startOfs

	end := start
	end.Byte = endOfs
	b := f.Bytes[startOfs:endOfs]
	for len(b) > 0 {
		advance, seq, _ := textseg.ScanGraphemeClusters(b, true)
		if len(seq) == 1 && seq[0] == '\n' {
			end.Line++
			end.Column = 1
		} else {
			end.Column++
		}
		b = b[advance:]
	}

	f.Pos = end

	f.Tokens = append(f.Tokens, Token{
		Type:  ty,
		Bytes: f.Bytes[startOfs:endOfs],
		Range: source.Range{
			Filename: f.Filename,
			Start:    start,
			End:      end,
		},
	})
}

type heredocInProgress struct {
	Marker      []byte
	StartOfLine bool
}
<commit_msg>parser: GoString for TokenType<commit_after>package parser

import (
	"fmt"

	"github.com/apparentlymart/go-textseg/textseg"
	"github.com/cirbo-lang/cirbo/source"
)

// Token represents a sequence of bytes from some zcl code that has been
// tagged with a type and its range within the source file.
type Token struct {
	Type  TokenType
	Bytes []byte
	Range source.Range
}

// Tokens is a slice of Token.
type Tokens []Token

// TokenType is an enumeration used for the Type field on Token.
type TokenType rune

const (
	// Single-character tokens are represented by their own character, for
	// convenience in producing these within the scanner. However, the values
	// are otherwise arbitrary and just intended to be mnemonic for humans
	// who might see them in debug output.

	TokenOBrace TokenType = '{'
	TokenCBrace TokenType = '}'
	TokenOBrack TokenType = '['
	TokenCBrack TokenType = ']'
	TokenOParen TokenType = '('
	TokenCParen TokenType = ')'

	TokenStar    TokenType = '*'
	TokenSlash   TokenType = '/'
	TokenPlus    TokenType = '+'
	TokenMinus   TokenType = '-'
	TokenPercent TokenType = '%'
	TokenCaret   TokenType = '^'

	TokenAssign        TokenType = '='
	TokenEqual         TokenType = '≔'
	TokenNotEqual      TokenType = '≠'
	TokenLessThan      TokenType = '<'
	TokenLessThanEq    TokenType = '≤'
	TokenGreaterThan   TokenType = '>'
	TokenGreaterThanEq TokenType = '≥'
	TokenDashDash      TokenType = '—'

	TokenAnd  TokenType = '∧'
	TokenOr   TokenType = '∨'
	TokenBang TokenType = '!'

	TokenDot   TokenType = '.'
	TokenComma TokenType = ','

	TokenQuestion  TokenType = '?'
	TokenColon     TokenType = ':'
	TokenSemicolon TokenType = ';'

	TokenStringLit TokenType = 'S'
	TokenNumberLit TokenType = 'N'
	TokenIdent     TokenType = 'I'

	TokenComment    TokenType = 'C'
	TokenWhitespace TokenType = ' '

	TokenEOF TokenType = '␄'

	// The following are not used in the language but recognized by the scanner
	// so we can generate good diagnostics in the parser when users try to write
	// things that might work in other languages they are familiar with, or
	// simply make incorrect assumptions about the cirbo language.

	TokenBitwiseAnd TokenType = '&'
	TokenBitwiseOr  TokenType = '|'
	TokenBitwiseNot TokenType = '~'
	TokenInvalid    TokenType = '�'
	TokenBadUTF8    TokenType = '💩'

	// TokenNil is a placeholder for when a token is required but none is
	// available, e.g. when reporting errors. The scanner will never produce
	// this as part of a token stream.
	TokenNil TokenType = '\x00'
)

func (t TokenType) GoString() string {
	return fmt.Sprintf("parser.%s", t.String())
}

type scanMode int

const (
	scanNormal scanMode = iota
	scanTemplate
)

type tokenAccum struct {
	Filename string
	Bytes    []byte
	Pos      source.Pos
	Tokens   []Token
}

func (f *tokenAccum) emitToken(ty TokenType, startOfs, endOfs int) {
	// Walk through our buffer to figure out how much we need to adjust
	// the start pos to get our end pos.

	start := f.Pos
	start.Column += startOfs - f.Pos.Byte // Safe because only ASCII whitespace can be in the offset (we count a tab character as one column)
	start.Byte = startOfs

	end := start
	end.Byte = endOfs
	b := f.Bytes[startOfs:endOfs]
	for len(b) > 0 {
		advance, seq, _ := textseg.ScanGraphemeClusters(b, true)
		if len(seq) == 1 && seq[0] == '\n' {
			end.Line++
			end.Column = 1
		} else {
			end.Column++
		}
		b = b[advance:]
	}

	f.Pos = end

	f.Tokens = append(f.Tokens, Token{
		Type:  ty,
		Bytes: f.Bytes[startOfs:endOfs],
		Range: source.Range{
			Filename: f.Filename,
			Start:    start,
			End:      end,
		},
	})
}

type heredocInProgress struct {
	Marker      []byte
	StartOfLine bool
}
<|endoftext|>


<commit_before>//go:generate -command mkcode go run mkcode.go --
//go:generate mkcode $GOFILE

// Information taken from
// - http://www.ntinternals.net/: UserMode / NTDLL / System Information
// - http://www.geoffchappell.com/studies/windows/km/index.htm

package ntdll

import (
	"reflect"
	"unsafe"
)

/*
func:
NTSTATUS NtQuerySystemInformation(
  _In_  SYSTEM_INFORMATION_CLASS SystemInformationClass,
  _Out_ PVOID                    SystemInformation,
  _In_  ULONG                    SystemInformationLength,
  _Out_ PULONG                   ReturnLength
);
*/

/*
func:
NTSTATUS NtSetSystemInformation(
  _In_ SYSTEM_INFORMATION_CLASS SystemInformationClass,
  _In_ PVOID                    SystemInformation,
  _In_ ULONG                    SystemInformationLength
);
*/

/*
enum:
typedef enum _SYSTEM_INFORMATION_CLASS {
    SystemBasicInformation,
    SystemProcessorInformation,
    SystemPerformanceInformation,
    SystemTimeOfDayInformation,
    SystemPathInformation,
    SystemProcessInformation,
    SystemCallCountInformation,
    SystemDeviceInformation,
    SystemProcessorPerformanceInformation,
    SystemFlagsInformation,
    SystemCallTimeInformation,
    SystemModuleInformation,
    SystemLocksInformation,
    SystemStackTraceInformation,
    SystemPagedPoolInformation,
    SystemNonPagedPoolInformation,
    SystemHandleInformation,
    SystemObjectInformation,
    SystemPageFileInformation,
    SystemVdmInstemulInformation,
    SystemVdmBopInformation,
    SystemFileCacheInformation,
    SystemPoolTagInformation,
    SystemInterruptInformation,
    SystemDpcBehaviorInformation,
    SystemFullMemoryInformation,
    SystemLoadGdiDriverInformation,
    SystemUnloadGdiDriverInformation,
    SystemTimeAdjustmentInformation,
    SystemSummaryMemoryInformation,
    SystemNextEventIdInformation,
    SystemEventIdsInformation,
    SystemCrashDumpInformation,
    SystemExceptionInformation,
    SystemCrashDumpStateInformation,
    SystemKernelDebuggerInformation,
    SystemContextSwitchInformation,
    SystemRegistryQuotaInformation,
    SystemExtendServiceTableInformation,
    SystemPrioritySeperation,
    SystemPlugPlayBusInformation,
    SystemDockInformation,
    SystemPowerInformation,
    SystemProcessorSpeedInformation,
    SystemCurrentTimeZoneInformation,
    SystemLookasideInformation
} SYSTEM_INFORMATION_CLASS, *PSYSTEM_INFORMATION_CLASS;
*/

/*
type:
typedef struct _SYSTEM_MODULE {
  PVOID                Section;
  PVOID                MappedBase;
  PVOID                ImageBase;
  ULONG                ImageSize;
  ULONG                Flags;
  WORD                 LoadOrderIndex;
  WORD                 InitOrderIndex;
  WORD                 LoadCount;
  WORD                 NameOffset;
  BYTE                 Name[256];
} SYSTEM_MODULE, *PSYSTEM_MODULE;
*/

/*
type:
typedef struct _SYSTEM_BASIC_INFORMATION {
  BYTE Reserved1[4];
  ULONG MaximumIncrement;
  ULONG PhysicalPageSize;
  ULONG NumberOfPhysicalPages;
  ULONG LowestPhysicalPage;
  ULONG HighestPhysicalPage;
  ULONG AllocationGranularity;
  ULONG_PTR LowestUserAddress;
  ULONG_PTR HighestUserAddress;
  ULONG_PTR ActiveProcessors;
  CCHAR NumberOfProcessors;
} SYSTEM_BASIC_INFORMATION, *PSYSTEM_BASIC_INFORMATION;
*/

/*
type:
typedef struct _SYSTEM_PROCESSOR_INFORMATION {
  USHORT ProcessorArchitecture;
  USHORT ProcessorLevel;
  USHORT ProcessorRevision;
  USHORT Unknown;
  ULONG FeatureBits;
} SYSTEM_PROCESSOR_INFORMATION, *PSYSTEM_PROCESSOR_INFORMATION;
*/

/*
type:
typedef struct _SYSTEM_PROCESSOR_PERFORMANCE_INFORMATION {
    LARGE_INTEGER IdleTime;
    LARGE_INTEGER KernelTime;
    LARGE_INTEGER UserTime;
    LARGE_INTEGER Reserved1[2];
    ULONG Reserved2;
} SYSTEM_PROCESSOR_PERFORMANCE_INFORMATION, *PSYSTEM_PROCESSOR_PERFORMANCE_INFORMATION;
*/

/*
type:
typedef struct _IO_COUNTERS {
  ULONGLONG ReadOperationCount;
  ULONGLONG WriteOperationCount;
  ULONGLONG OtherOperationCount;
  ULONGLONG ReadTransferCount;
  ULONGLONG WriteTransferCount;
  ULONGLONG OtherTransferCount;
} IO_COUNTERS, *PIO_COUNTERS;
*/

/*
type:
typedef struct _VM_COUNTERS {
  SIZE_T PeakVirtualSize;
  SIZE_T VirtualSize;
  ULONG PageFaultCount;
  SIZE_T PeakWorkingSetSize;
  SIZE_T WorkingSetSize;
  SIZE_T QuotaPeakPagedPoolUsage;
  SIZE_T QuotaPagedPoolUsage;
  SIZE_T QuotaPeakNonPagedPoolUsage;
  SIZE_T QuotaNonPagedPoolUsage;
  SIZE_T PagefileUsage;
  SIZE_T PeakPagefileUsage;
} VM_COUNTERS, *PVM_COUNTERS;
*/

/*
type:
typedef struct _SYSTEM_PROCESS_INFORMATION {
  ULONG NextEntryOffset;
  ULONG NumberOfThreads;
  LARGE_INTEGER Reserved[3];
  LARGE_INTEGER CreateTime;
  LARGE_INTEGER UserTime;
  LARGE_INTEGER KernelTime;
  UNICODE_STRING ImageName;
  KPRIORITY BasePriority;
  HANDLE UniqueProcessId;
  HANDLE InheritedFromUniqueProcessId;
  ULONG HandleCount;
  ULONG SessionId;
  ULONG PageDirectoryBase;
  VM_COUNTERS VirtualMemoryCounters;
  SIZE_T PrivatePageCount;
  IO_COUNTERS IoCounters;
} SYSTEM_PROCESS_INFORMATION, *PSYSTEM_PROCESS_INFORMATION;
*/

type Kpriority uint64

/*
type:
typedef struct _CLIENT_ID {
  HANDLE UniqueProcess;
  HANDLE UniqueThread;
} CLIENT_ID, *PCLIENT_ID;
*/

/*
type:
typedef struct _SYSTEM_THREAD_INFORMATION {
    LARGE_INTEGER Reserved1[3];
    ULONG Reserved2;
    PVOID StartAddress;
    CLIENT_ID ClientId;
    KPRIORITY Priority;
    LONG BasePriority;
    ULONG Reserved3;
    ULONG ThreadState;
    ULONG WaitReason;
} SYSTEM_THREAD_INFORMATION, *PSYSTEM_THREAD_INFORMATION;
*/

/*
type:
typedef struct _SYSTEM_REGISTRY_QUOTA_INFORMATION {
    ULONG RegistryQuotaAllowed;
    ULONG RegistryQuotaUsed;
    PVOID Reserved1;
} SYSTEM_REGISTRY_QUOTA_INFORMATION, *PSYSTEM_REGISTRY_QUOTA_INFORMATION;
*/

/*
type:
typedef struct _SYSTEM_TIMEOFDAY_INFORMATION {
  LARGE_INTEGER BootTime;
  LARGE_INTEGER CurrentTime;
  LARGE_INTEGER TimeZoneBias;
  ULONG CurrentTimeZoneId;
  BYTE Reserved1[20];
} SYSTEM_TIMEOFDAY_INFORMATION,*PSYSTEM_TIMEOFDAY_INFORMATION;
*/

/*
type:
typedef struct _SYSTEM_PERFORMANCE_INFORMATION {
  LARGE_INTEGER IdleTime;
  LARGE_INTEGER ReadTransferCount;
  LARGE_INTEGER WriteTransferCount;
  LARGE_INTEGER OtherTransferCount;
  ULONG ReadOperationCount;
  ULONG WriteOperationCount;
  ULONG OtherOperationCount;
  ULONG AvailablePages;
  ULONG TotalCommittedPages;
  ULONG TotalCommitLimit;
  ULONG PeakCommitment;
  ULONG PageFaults;
  ULONG WriteCopyFaults;
  ULONG TransitionFaults;
  ULONG CacheTransitionFaults;
  ULONG DemandZeroFaults;
  ULONG PagesRead;
  ULONG PageReadIos;
  ULONG CacheReads;
  ULONG CacheIos;
  ULONG PagefilePagesWritten;
  ULONG PagefilePageWriteIos;
  ULONG MappedFilePagesWritten;
  ULONG MappedFilePageWriteIos;
  ULONG PagedPoolUsage;
  ULONG NonPagedPoolUsage;
  ULONG PagedPoolAllocs;
  ULONG PagedPoolFrees;
  ULONG NonPagedPoolAllocs;
  ULONG NonPagedPoolFrees;
  ULONG TotalFreeSystemPtes;
  ULONG SystemCodePage;
  ULONG TotalSystemDriverPages;
  ULONG TotalSystemCodePages;
  ULONG SmallNonPagedLookasideListAllocateHits;
  ULONG SmallPagedLookasideListAllocateHits;
  ULONG Reserved3;
  ULONG MmSystemCachePage;
  ULONG PagedPoolPage;
  ULONG SystemDriverPage;
  ULONG FastReadNoWait;
  ULONG FastReadWait;
  ULONG FastReadResourceMiss;
  ULONG FastReadNotPossible;
  ULONG FastMdlReadNoWait;
  ULONG FastMdlReadWait;
  ULONG FastMdlReadResourceMiss;
  ULONG FastMdlReadNotPossible;
  ULONG MapDataNoWait;
  ULONG MapDataWait;
  ULONG MapDataNoWaitMiss;
  ULONG MapDataWaitMiss;
  ULONG PinMappedDataCount;
  ULONG PinReadNoWait;
  ULONG PinReadWait;
  ULONG PinReadNoWaitMiss;
  ULONG PinReadWaitMiss;
  ULONG CopyReadNoWait;
  ULONG CopyReadWait;
  ULONG CopyReadNoWaitMiss;
  ULONG CopyReadWaitMiss;
  ULONG MdlReadNoWait;
  ULONG MdlReadWait;
  ULONG MdlReadNoWaitMiss;
  ULONG MdlReadWaitMiss;
  ULONG ReadAheadIos;
  ULONG LazyWriteIos;
  ULONG LazyWritePages;
  ULONG DataFlushes;
  ULONG DataPages;
  ULONG ContextSwitches;
  ULONG FirstLevelTbFills;
  ULONG SecondLevelTbFills;
  ULONG SystemCalls;
} SYSTEM_PERFORMANCE_INFORMATION, *PSYSTEM_PERFORMANCE_INFORMATION;
*/

/*
type:
typedef struct _SYSTEM_EXCEPTION_INFORMATION {
    BYTE Reserved1[16];
} SYSTEM_EXCEPTION_INFORMATION, *PSYSTEM_EXCEPTION_INFORMATION;
*/

/*
type:
typedef struct _SYSTEM_LOOKASIDE_INFORMATION {
    BYTE Reserved1[32];
} SYSTEM_LOOKASIDE_INFORMATION, *PSYSTEM_LOOKASIDE_INFORMATION;
*/

/*
type:
typedef struct _SYSTEM_INTERRUPT_INFORMATION {
    BYTE Reserved1[24];
} SYSTEM_INTERRUPT_INFORMATION, *PSYSTEM_INTERRUPT_INFORMATION;
*/

/*
type:
typedef struct _SYSTEM_POLICY_INFORMATION {
    PVOID Reserved1[2];
    ULONG Reserved2[3];
} SYSTEM_POLICY_INFORMATION, *PSYSTEM_POLICY_INFORMATION;
*/

/*
type:
typedef struct _SYSTEM_HANDLE_ENTRY {
  ULONG OwnerPid;
  BYTE ObjectType;
  BYTE HandleFlags;
  USHORT HandleValue;
  PVOID ObjectPointer;
  ULONG AccessMask;
} SYSTEM_HANDLE_ENTRY, *PSYSTEM_HANDLE_ENTRY;
*/

/*
type:
typedef struct _SYSTEM_HANDLE_INFORMATION {
  ULONG Count;
  SYSTEM_HANDLE_ENTRY Handle[1];
} SYSTEM_HANDLE_INFORMATION, *PSYSTEM_HANDLE_INFORMATION;
*/

/*
type:
typedef struct _SYSTEM_PAGEFILE_INFORMATION {
  ULONG NextEntryOffset;
  ULONG CurrentSize;
  ULONG TotalUsed;
  ULONG PeakUsed;
  UNICODE_STRING FileName;
} SYSTEM_PAGEFILE_INFORMATION, *PSYSTEM_PAGEFILE_INFORMATION;
*/

/*
type:
typedef struct _SYSTEM_MODULE_INFORMATION {
  ULONG                ModulesCount;
  SYSTEM_MODULE        Modules[ANYSIZE_ARRAY];
} SYSTEM_MODULE_INFORMATION, *PSYSTEM_MODULE_INFORMATION;
*/

func (mi *SystemModuleInformationT) GetModules() []SystemModule {
	modules := make([]SystemModule, int(mi.ModulesCount))
	hdr := (*reflect.SliceHeader)(unsafe.Pointer(&modules))
	hdr.Data = uintptr(unsafe.Pointer(&mi.Modules[0]))
	return modules
}
<commit_msg>Update comment about struct info sources<commit_after>//go:generate -command mkcode go run mkcode.go --
//go:generate mkcode $GOFILE

// Information taken from
// - http://www.ntinternals.net/: UserMode / NTDLL / System Information
// - http://www.geoffchappell.com/studies/windows/km/index.htm
// - MinGW headers

package ntdll

import (
	"reflect"
	"unsafe"
)

/*
func:
NTSTATUS NtQuerySystemInformation(
  _In_  SYSTEM_INFORMATION_CLASS SystemInformationClass,
  _Out_ PVOID                    SystemInformation,
  _In_  ULONG                    SystemInformationLength,
  _Out_ PULONG                   ReturnLength
);
*/

/*
func:
NTSTATUS NtSetSystemInformation(
  _In_ SYSTEM_INFORMATION_CLASS SystemInformationClass,
  _In_ PVOID                    SystemInformation,
  _In_ ULONG                    SystemInformationLength
);
*/

/*
enum:
typedef enum _SYSTEM_INFORMATION_CLASS {
    SystemBasicInformation,
    SystemProcessorInformation,
    SystemPerformanceInformation,
    SystemTimeOfDayInformation,
    SystemPathInformation,
    SystemProcessInformation,
    SystemCallCountInformation,
    SystemDeviceInformation,
    SystemProcessorPerformanceInformation,
    SystemFlagsInformation,
    SystemCallTimeInformation,
    SystemModuleInformation,
    SystemLocksInformation,
    SystemStackTraceInformation,
    SystemPagedPoolInformation,
    SystemNonPagedPoolInformation,
    SystemHandleInformation,
    SystemObjectInformation,
    SystemPageFileInformation,
    SystemVdmInstemulInformation,
    SystemVdmBopInformation,
    SystemFileCacheInformation,
    SystemPoolTagInformation,
    SystemInterruptInformation,
    SystemDpcBehaviorInformation,
    SystemFullMemoryInformation,
    SystemLoadGdiDriverInformation,
    SystemUnloadGdiDriverInformation,
    SystemTimeAdjustmentInformation,
    SystemSummaryMemoryInformation,
    SystemNextEventIdInformation,
    SystemEventIdsInformation,
    SystemCrashDumpInformation,
    SystemExceptionInformation,
    SystemCrashDumpStateInformation,
    SystemKernelDebuggerInformation,
    SystemContextSwitchInformation,
    SystemRegistryQuotaInformation,
    SystemExtendServiceTableInformation,
    SystemPrioritySeperation,
    SystemPlugPlayBusInformation,
    SystemDockInformation,
    SystemPowerInformation,
    SystemProcessorSpeedInformation,
    SystemCurrentTimeZoneInformation,
    SystemLookasideInformation
} SYSTEM_INFORMATION_CLASS, *PSYSTEM_INFORMATION_CLASS;
*/

/*
type:
typedef struct _SYSTEM_MODULE {
  PVOID                Section;
  PVOID                MappedBase;
  PVOID                ImageBase;
  ULONG                ImageSize;
  ULONG                Flags;
  WORD                 LoadOrderIndex;
  WORD                 InitOrderIndex;
  WORD                 LoadCount;
  WORD                 NameOffset;
  BYTE                 Name[256];
} SYSTEM_MODULE, *PSYSTEM_MODULE;
*/

/*
type:
typedef struct _SYSTEM_BASIC_INFORMATION {
  BYTE Reserved1[4];
  ULONG MaximumIncrement;
  ULONG PhysicalPageSize;
  ULONG NumberOfPhysicalPages;
  ULONG LowestPhysicalPage;
  ULONG HighestPhysicalPage;
  ULONG AllocationGranularity;
  ULONG_PTR LowestUserAddress;
  ULONG_PTR HighestUserAddress;
  ULONG_PTR ActiveProcessors;
  CCHAR NumberOfProcessors;
} SYSTEM_BASIC_INFORMATION, *PSYSTEM_BASIC_INFORMATION;
*/

/*
type:
typedef struct _SYSTEM_PROCESSOR_INFORMATION {
  USHORT ProcessorArchitecture;
  USHORT ProcessorLevel;
  USHORT ProcessorRevision;
  USHORT Unknown;
  ULONG FeatureBits;
} SYSTEM_PROCESSOR_INFORMATION, *PSYSTEM_PROCESSOR_INFORMATION;
*/

/*
type:
typedef struct _SYSTEM_PROCESSOR_PERFORMANCE_INFORMATION {
    LARGE_INTEGER IdleTime;
    LARGE_INTEGER KernelTime;
    LARGE_INTEGER UserTime;
    LARGE_INTEGER Reserved1[2];
    ULONG Reserved2;
} SYSTEM_PROCESSOR_PERFORMANCE_INFORMATION, *PSYSTEM_PROCESSOR_PERFORMANCE_INFORMATION;
*/

/*
type:
typedef struct _IO_COUNTERS {
  ULONGLONG ReadOperationCount;
  ULONGLONG WriteOperationCount;
  ULONGLONG OtherOperationCount;
  ULONGLONG ReadTransferCount;
  ULONGLONG WriteTransferCount;
  ULONGLONG OtherTransferCount;
} IO_COUNTERS, *PIO_COUNTERS;
*/

/*
type:
typedef struct _VM_COUNTERS {
  SIZE_T PeakVirtualSize;
  SIZE_T VirtualSize;
  ULONG PageFaultCount;
  SIZE_T PeakWorkingSetSize;
  SIZE_T WorkingSetSize;
  SIZE_T QuotaPeakPagedPoolUsage;
  SIZE_T QuotaPagedPoolUsage;
  SIZE_T QuotaPeakNonPagedPoolUsage;
  SIZE_T QuotaNonPagedPoolUsage;
  SIZE_T PagefileUsage;
  SIZE_T PeakPagefileUsage;
} VM_COUNTERS, *PVM_COUNTERS;
*/

/*
type:
typedef struct _SYSTEM_PROCESS_INFORMATION {
  ULONG NextEntryOffset;
  ULONG NumberOfThreads;
  LARGE_INTEGER Reserved[3];
  LARGE_INTEGER CreateTime;
  LARGE_INTEGER UserTime;
  LARGE_INTEGER KernelTime;
  UNICODE_STRING ImageName;
  KPRIORITY BasePriority;
  HANDLE UniqueProcessId;
  HANDLE InheritedFromUniqueProcessId;
  ULONG HandleCount;
  ULONG SessionId;
  ULONG PageDirectoryBase;
  VM_COUNTERS VirtualMemoryCounters;
  SIZE_T PrivatePageCount;
  IO_COUNTERS IoCounters;
} SYSTEM_PROCESS_INFORMATION, *PSYSTEM_PROCESS_INFORMATION;
*/

type Kpriority uint64

/*
type:
typedef struct _CLIENT_ID {
  HANDLE UniqueProcess;
  HANDLE UniqueThread;
} CLIENT_ID, *PCLIENT_ID;
*/

/*
type:
typedef struct _SYSTEM_THREAD_INFORMATION {
    LARGE_INTEGER Reserved1[3];
    ULONG Reserved2;
    PVOID StartAddress;
    CLIENT_ID ClientId;
    KPRIORITY Priority;
    LONG BasePriority;
    ULONG Reserved3;
    ULONG ThreadState;
    ULONG WaitReason;
} SYSTEM_THREAD_INFORMATION, *PSYSTEM_THREAD_INFORMATION;
*/

/*
type:
typedef struct _SYSTEM_REGISTRY_QUOTA_INFORMATION {
    ULONG RegistryQuotaAllowed;
    ULONG RegistryQuotaUsed;
    PVOID Reserved1;
} SYSTEM_REGISTRY_QUOTA_INFORMATION, *PSYSTEM_REGISTRY_QUOTA_INFORMATION;
*/

/*
type:
typedef struct _SYSTEM_TIMEOFDAY_INFORMATION {
  LARGE_INTEGER BootTime;
  LARGE_INTEGER CurrentTime;
  LARGE_INTEGER TimeZoneBias;
  ULONG CurrentTimeZoneId;
  BYTE Reserved1[20];
} SYSTEM_TIMEOFDAY_INFORMATION,*PSYSTEM_TIMEOFDAY_INFORMATION;
*/

/*
type:
typedef struct _SYSTEM_PERFORMANCE_INFORMATION {
  LARGE_INTEGER IdleTime;
  LARGE_INTEGER ReadTransferCount;
  LARGE_INTEGER WriteTransferCount;
  LARGE_INTEGER OtherTransferCount;
  ULONG ReadOperationCount;
  ULONG WriteOperationCount;
  ULONG OtherOperationCount;
  ULONG AvailablePages;
  ULONG TotalCommittedPages;
  ULONG TotalCommitLimit;
  ULONG PeakCommitment;
  ULONG PageFaults;
  ULONG WriteCopyFaults;
  ULONG TransitionFaults;
  ULONG CacheTransitionFaults;
  ULONG DemandZeroFaults;
  ULONG PagesRead;
  ULONG PageReadIos;
  ULONG CacheReads;
  ULONG CacheIos;
  ULONG PagefilePagesWritten;
  ULONG PagefilePageWriteIos;
  ULONG MappedFilePagesWritten;
  ULONG MappedFilePageWriteIos;
  ULONG PagedPoolUsage;
  ULONG NonPagedPoolUsage;
  ULONG PagedPoolAllocs;
  ULONG PagedPoolFrees;
  ULONG NonPagedPoolAllocs;
  ULONG NonPagedPoolFrees;
  ULONG TotalFreeSystemPtes;
  ULONG SystemCodePage;
  ULONG TotalSystemDriverPages;
  ULONG TotalSystemCodePages;
  ULONG SmallNonPagedLookasideListAllocateHits;
  ULONG SmallPagedLookasideListAllocateHits;
  ULONG Reserved3;
  ULONG MmSystemCachePage;
  ULONG PagedPoolPage;
  ULONG SystemDriverPage;
  ULONG FastReadNoWait;
  ULONG FastReadWait;
  ULONG FastReadResourceMiss;
  ULONG FastReadNotPossible;
  ULONG FastMdlReadNoWait;
  ULONG FastMdlReadWait;
  ULONG FastMdlReadResourceMiss;
  ULONG FastMdlReadNotPossible;
  ULONG MapDataNoWait;
  ULONG MapDataWait;
  ULONG MapDataNoWaitMiss;
  ULONG MapDataWaitMiss;
  ULONG PinMappedDataCount;
  ULONG PinReadNoWait;
  ULONG PinReadWait;
  ULONG PinReadNoWaitMiss;
  ULONG PinReadWaitMiss;
  ULONG CopyReadNoWait;
  ULONG CopyReadWait;
  ULONG CopyReadNoWaitMiss;
  ULONG CopyReadWaitMiss;
  ULONG MdlReadNoWait;
  ULONG MdlReadWait;
  ULONG MdlReadNoWaitMiss;
  ULONG MdlReadWaitMiss;
  ULONG ReadAheadIos;
  ULONG LazyWriteIos;
  ULONG LazyWritePages;
  ULONG DataFlushes;
  ULONG DataPages;
  ULONG ContextSwitches;
  ULONG FirstLevelTbFills;
  ULONG SecondLevelTbFills;
  ULONG SystemCalls;
} SYSTEM_PERFORMANCE_INFORMATION, *PSYSTEM_PERFORMANCE_INFORMATION;
*/

/*
type:
typedef struct _SYSTEM_EXCEPTION_INFORMATION {
    BYTE Reserved1[16];
} SYSTEM_EXCEPTION_INFORMATION, *PSYSTEM_EXCEPTION_INFORMATION;
*/

/*
type:
typedef struct _SYSTEM_LOOKASIDE_INFORMATION {
    BYTE Reserved1[32];
} SYSTEM_LOOKASIDE_INFORMATION, *PSYSTEM_LOOKASIDE_INFORMATION;
*/

/*
type:
typedef struct _SYSTEM_INTERRUPT_INFORMATION {
    BYTE Reserved1[24];
} SYSTEM_INTERRUPT_INFORMATION, *PSYSTEM_INTERRUPT_INFORMATION;
*/

/*
type:
typedef struct _SYSTEM_POLICY_INFORMATION {
    PVOID Reserved1[2];
    ULONG Reserved2[3];
} SYSTEM_POLICY_INFORMATION, *PSYSTEM_POLICY_INFORMATION;
*/

/*
type:
typedef struct _SYSTEM_HANDLE_ENTRY {
  ULONG OwnerPid;
  BYTE ObjectType;
  BYTE HandleFlags;
  USHORT HandleValue;
  PVOID ObjectPointer;
  ULONG AccessMask;
} SYSTEM_HANDLE_ENTRY, *PSYSTEM_HANDLE_ENTRY;
*/

/*
type:
typedef struct _SYSTEM_HANDLE_INFORMATION {
  ULONG Count;
  SYSTEM_HANDLE_ENTRY Handle[1];
} SYSTEM_HANDLE_INFORMATION, *PSYSTEM_HANDLE_INFORMATION;
*/

/*
type:
typedef struct _SYSTEM_PAGEFILE_INFORMATION {
  ULONG NextEntryOffset;
  ULONG CurrentSize;
  ULONG TotalUsed;
  ULONG PeakUsed;
  UNICODE_STRING FileName;
} SYSTEM_PAGEFILE_INFORMATION, *PSYSTEM_PAGEFILE_INFORMATION;
*/

/*
type:
typedef struct _SYSTEM_MODULE_INFORMATION {
  ULONG                ModulesCount;
  SYSTEM_MODULE        Modules[ANYSIZE_ARRAY];
} SYSTEM_MODULE_INFORMATION, *PSYSTEM_MODULE_INFORMATION;
*/

func (mi *SystemModuleInformationT) GetModules() []SystemModule {
	modules := make([]SystemModule, int(mi.ModulesCount))
	hdr := (*reflect.SliceHeader)(unsafe.Pointer(&modules))
	hdr.Data = uintptr(unsafe.Pointer(&mi.Modules[0]))
	return modules
}
<|endoftext|>


<commit_before>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"syscall"
	"time"

	"github.com/google/syzkaller/pkg/log"
	"github.com/google/syzkaller/pkg/osutil"
	"github.com/google/syzkaller/pkg/vcs"
	"github.com/google/syzkaller/syz-manager/mgrconfig"
)

const (
	syzkallerRebuildPeriod = 12 * time.Hour
	buildRetryPeriod       = 10 * time.Minute // used for both syzkaller and kernel
)

// SyzUpdater handles everything related to syzkaller updates.
// As kernel builder, it maintains 2 builds:
//  - latest: latest known good syzkaller build
//  - current: currently used syzkaller build
// Additionally it updates and restarts the current executable as necessary.
// Current executable is always built on the same revision as the rest of syzkaller binaries.
type SyzUpdater struct {
	repo         vcs.Repo
	exe          string
	repoAddress  string
	branch       string
	descriptions string
	gopathDir    string
	syzkallerDir string
	latestDir    string
	currentDir   string
	syzFiles     map[string]bool
	targets      map[string]bool
}

func NewSyzUpdater(cfg *Config) *SyzUpdater {
	wd, err := os.Getwd()
	if err != nil {
		log.Fatalf("failed to get wd: %v", err)
	}
	bin := os.Args[0]
	if !filepath.IsAbs(bin) {
		bin = filepath.Join(wd, bin)
	}
	bin = filepath.Clean(bin)
	exe := filepath.Base(bin)
	if wd != filepath.Dir(bin) {
		log.Fatalf("%v executable must be in cwd (it will be overwritten on update)", exe)
	}

	gopath := filepath.Join(wd, "gopath")
	os.Setenv("GOROOT", cfg.Goroot)
	os.Setenv("PATH", filepath.Join(cfg.Goroot, "bin")+
		string(filepath.ListSeparator)+os.Getenv("PATH"))
	syzkallerDir := filepath.Join(gopath, "src", "github.com", "google", "syzkaller")
	osutil.MkdirAll(syzkallerDir)

	// List of required files in syzkaller build (contents of latest/current dirs).
	files := map[string]bool{
		"tag":             true, // contains syzkaller repo git hash
		"bin/syz-ci":      true, // these are just copied from syzkaller dir
		"bin/syz-manager": true,
	}
	targets := make(map[string]bool)
	for _, mgr := range cfg.Managers {
		mgrcfg, err := mgrconfig.LoadPartialData(mgr.ManagerConfig)
		if err != nil {
			log.Fatalf("failed to load manager %v config: %v", mgr.Name, err)
		}
		os, vmarch, arch := mgrcfg.TargetOS, mgrcfg.TargetVMArch, mgrcfg.TargetArch
		targets[os+"/"+vmarch+"/"+arch] = true
		files[fmt.Sprintf("bin/%v_%v/syz-fuzzer", os, vmarch)] = true
		files[fmt.Sprintf("bin/%v_%v/syz-execprog", os, vmarch)] = true
		files[fmt.Sprintf("bin/%v_%v/syz-executor", os, arch)] = true
	}
	syzFiles := make(map[string]bool)
	for f := range files {
		syzFiles[f] = true
	}
	return &SyzUpdater{
		repo:         vcs.NewSyzkallerRepo(syzkallerDir),
		exe:          exe,
		repoAddress:  cfg.SyzkallerRepo,
		branch:       cfg.SyzkallerBranch,
		descriptions: cfg.SyzkallerDescriptions,
		gopathDir:    gopath,
		syzkallerDir: syzkallerDir,
		latestDir:    filepath.Join("syzkaller", "latest"),
		currentDir:   filepath.Join("syzkaller", "current"),
		syzFiles:     syzFiles,
		targets:      targets,
	}
}

// UpdateOnStart does 3 things:
//  - ensures that the current executable is fresh
//  - ensures that we have a working syzkaller build in current
func (upd *SyzUpdater) UpdateOnStart(shutdown chan struct{}) {
	os.RemoveAll(upd.currentDir)
	exeTag, exeMod := readTag(upd.exe + ".tag")
	latestTag := upd.checkLatest()
	if exeTag == latestTag && time.Since(exeMod) < time.Minute {
		// Have a freash up-to-date build, probably just restarted.
		log.Logf(0, "current executable is up-to-date (%v)", exeTag)
		if err := osutil.LinkFiles(upd.latestDir, upd.currentDir, upd.syzFiles); err != nil {
			log.Fatal(err)
		}
		return
	}
	if exeTag == "" {
		log.Logf(0, "current executable is bootstrap")
	} else {
		log.Logf(0, "current executable is on %v", exeTag)
		log.Logf(0, "latest syzkaller build is on %v", latestTag)
	}

	// No syzkaller build or executable is stale.
	lastCommit := exeTag
	for {
		lastCommit = upd.pollAndBuild(lastCommit)
		latestTag := upd.checkLatest()
		if latestTag != "" {
			// The build was successful or we had the latest build from previous runs.
			// Either way, use the latest build.
			log.Logf(0, "using syzkaller built on %v", latestTag)
			if err := osutil.LinkFiles(upd.latestDir, upd.currentDir, upd.syzFiles); err != nil {
				log.Fatal(err)
			}
			if exeTag != latestTag {
				upd.UpdateAndRestart()
			}
			return
		}

		// No good build at all, try again later.
		log.Logf(0, "retrying in %v", buildRetryPeriod)
		select {
		case <-time.After(buildRetryPeriod):
		case <-shutdown:
			os.Exit(0)
		}
	}
}

// WaitForUpdate polls and rebuilds syzkaller.
// Returns when we have a new good build in latest.
func (upd *SyzUpdater) WaitForUpdate() {
	time.Sleep(syzkallerRebuildPeriod)
	latestTag := upd.checkLatest()
	lastCommit := latestTag
	for {
		lastCommit = upd.pollAndBuild(lastCommit)
		if latestTag != upd.checkLatest() {
			break
		}
		time.Sleep(buildRetryPeriod)
	}
	log.Logf(0, "syzkaller: update available, restarting")
}

// UpdateAndRestart updates and restarts the current executable.
// Does not return.
func (upd *SyzUpdater) UpdateAndRestart() {
	log.Logf(0, "restarting executable for update")
	latestBin := filepath.Join(upd.latestDir, "bin", upd.exe)
	latestTag := filepath.Join(upd.latestDir, "tag")
	if err := osutil.CopyFile(latestBin, upd.exe); err != nil {
		log.Fatal(err)
	}
	if err := osutil.CopyFile(latestTag, upd.exe+".tag"); err != nil {
		log.Fatal(err)
	}
	if err := syscall.Exec(upd.exe, os.Args, os.Environ()); err != nil {
		log.Fatal(err)
	}
	log.Fatalf("not reachable")
}

func (upd *SyzUpdater) pollAndBuild(lastCommit string) string {
	commit, err := upd.repo.Poll(upd.repoAddress, upd.branch)
	if err != nil {
		log.Logf(0, "syzkaller: failed to poll: %v", err)
		return lastCommit
	}
	log.Logf(0, "syzkaller: poll: %v (%v)", commit.Hash, commit.Title)
	if lastCommit != commit.Hash {
		log.Logf(0, "syzkaller: building ...")
		lastCommit = commit.Hash
		if err := upd.build(commit); err != nil {
			log.Logf(0, "syzkaller: %v", err)
		}
	}
	return lastCommit
}

func (upd *SyzUpdater) build(commit *vcs.Commit) error {
	if upd.descriptions != "" {
		files, err := ioutil.ReadDir(upd.descriptions)
		if err != nil {
			return fmt.Errorf("failed to read descriptions dir: %v", err)
		}
		for _, f := range files {
			src := filepath.Join(upd.descriptions, f.Name())
			dst := filepath.Join(upd.syzkallerDir, "sys", "linux", f.Name())
			if err := osutil.CopyFile(src, dst); err != nil {
				return err
			}
		}
	}
	cmd := osutil.Command("make", "generate")
	cmd.Dir = upd.syzkallerDir
	cmd.Env = append([]string{"GOPATH=" + upd.gopathDir}, os.Environ()...)
	if _, err := osutil.Run(time.Hour, cmd); err != nil {
		return fmt.Errorf("build failed: %v", err)
	}
	cmd = osutil.Command("make", "host", "ci")
	cmd.Dir = upd.syzkallerDir
	cmd.Env = append([]string{"GOPATH=" + upd.gopathDir}, os.Environ()...)
	if _, err := osutil.Run(time.Hour, cmd); err != nil {
		return fmt.Errorf("build failed: %v", err)
	}
	for target := range upd.targets {
		parts := strings.Split(target, "/")
		cmd := osutil.Command("make", "target")
		cmd.Dir = upd.syzkallerDir
		cmd.Env = append([]string{}, os.Environ()...)
		cmd.Env = append(cmd.Env,
			"GOPATH="+upd.gopathDir,
			"TARGETOS="+parts[0],
			"TARGETVMARCH="+parts[1],
			"TARGETARCH="+parts[2],
		)
		if _, err := osutil.Run(time.Hour, cmd); err != nil {
			return fmt.Errorf("build failed: %v", err)
		}
	}
	cmd = osutil.Command("go", "test", "-short", "./...")
	cmd.Dir = upd.syzkallerDir
	cmd.Env = append([]string{"GOPATH=" + upd.gopathDir}, os.Environ()...)
	if _, err := osutil.Run(time.Hour, cmd); err != nil {
		return fmt.Errorf("tests failed: %v", err)
	}
	tagFile := filepath.Join(upd.syzkallerDir, "tag")
	if err := osutil.WriteFile(tagFile, []byte(commit.Hash)); err != nil {
		return fmt.Errorf("filed to write tag file: %v", err)
	}
	if err := osutil.CopyFiles(upd.syzkallerDir, upd.latestDir, upd.syzFiles); err != nil {
		return fmt.Errorf("filed to copy syzkaller: %v", err)
	}
	return nil
}

// checkLatest returns tag of the latest build,
// or an empty string if latest build is missing/broken.
func (upd *SyzUpdater) checkLatest() string {
	if !osutil.FilesExist(upd.latestDir, upd.syzFiles) {
		return ""
	}
	tag, _ := readTag(filepath.Join(upd.latestDir, "tag"))
	return tag
}

func readTag(file string) (tag string, mod time.Time) {
	data, _ := ioutil.ReadFile(file)
	tag = string(data)
	if st, err := os.Stat(file); err == nil {
		mod = st.ModTime()
	}
	if tag == "" || mod.IsZero() {
		tag = ""
		mod = time.Time{}
	}
	return
}
<commit_msg>syz-ci: unset GOPATH<commit_after>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"syscall"
	"time"

	"github.com/google/syzkaller/pkg/log"
	"github.com/google/syzkaller/pkg/osutil"
	"github.com/google/syzkaller/pkg/vcs"
	"github.com/google/syzkaller/syz-manager/mgrconfig"
)

const (
	syzkallerRebuildPeriod = 12 * time.Hour
	buildRetryPeriod       = 10 * time.Minute // used for both syzkaller and kernel
)

// SyzUpdater handles everything related to syzkaller updates.
// As kernel builder, it maintains 2 builds:
//  - latest: latest known good syzkaller build
//  - current: currently used syzkaller build
// Additionally it updates and restarts the current executable as necessary.
// Current executable is always built on the same revision as the rest of syzkaller binaries.
type SyzUpdater struct {
	repo         vcs.Repo
	exe          string
	repoAddress  string
	branch       string
	descriptions string
	gopathDir    string
	syzkallerDir string
	latestDir    string
	currentDir   string
	syzFiles     map[string]bool
	targets      map[string]bool
}

func NewSyzUpdater(cfg *Config) *SyzUpdater {
	wd, err := os.Getwd()
	if err != nil {
		log.Fatalf("failed to get wd: %v", err)
	}
	bin := os.Args[0]
	if !filepath.IsAbs(bin) {
		bin = filepath.Join(wd, bin)
	}
	bin = filepath.Clean(bin)
	exe := filepath.Base(bin)
	if wd != filepath.Dir(bin) {
		log.Fatalf("%v executable must be in cwd (it will be overwritten on update)", exe)
	}

	gopath := filepath.Join(wd, "gopath")
	os.Setenv("GOROOT", cfg.Goroot)
	os.Unsetenv("GOPATH")
	os.Setenv("PATH", filepath.Join(cfg.Goroot, "bin")+
		string(filepath.ListSeparator)+os.Getenv("PATH"))
	syzkallerDir := filepath.Join(gopath, "src", "github.com", "google", "syzkaller")
	osutil.MkdirAll(syzkallerDir)

	// List of required files in syzkaller build (contents of latest/current dirs).
	files := map[string]bool{
		"tag":             true, // contains syzkaller repo git hash
		"bin/syz-ci":      true, // these are just copied from syzkaller dir
		"bin/syz-manager": true,
	}
	targets := make(map[string]bool)
	for _, mgr := range cfg.Managers {
		mgrcfg, err := mgrconfig.LoadPartialData(mgr.ManagerConfig)
		if err != nil {
			log.Fatalf("failed to load manager %v config: %v", mgr.Name, err)
		}
		os, vmarch, arch := mgrcfg.TargetOS, mgrcfg.TargetVMArch, mgrcfg.TargetArch
		targets[os+"/"+vmarch+"/"+arch] = true
		files[fmt.Sprintf("bin/%v_%v/syz-fuzzer", os, vmarch)] = true
		files[fmt.Sprintf("bin/%v_%v/syz-execprog", os, vmarch)] = true
		files[fmt.Sprintf("bin/%v_%v/syz-executor", os, arch)] = true
	}
	syzFiles := make(map[string]bool)
	for f := range files {
		syzFiles[f] = true
	}
	return &SyzUpdater{
		repo:         vcs.NewSyzkallerRepo(syzkallerDir),
		exe:          exe,
		repoAddress:  cfg.SyzkallerRepo,
		branch:       cfg.SyzkallerBranch,
		descriptions: cfg.SyzkallerDescriptions,
		gopathDir:    gopath,
		syzkallerDir: syzkallerDir,
		latestDir:    filepath.Join("syzkaller", "latest"),
		currentDir:   filepath.Join("syzkaller", "current"),
		syzFiles:     syzFiles,
		targets:      targets,
	}
}

// UpdateOnStart does 3 things:
//  - ensures that the current executable is fresh
//  - ensures that we have a working syzkaller build in current
func (upd *SyzUpdater) UpdateOnStart(shutdown chan struct{}) {
	os.RemoveAll(upd.currentDir)
	exeTag, exeMod := readTag(upd.exe + ".tag")
	latestTag := upd.checkLatest()
	if exeTag == latestTag && time.Since(exeMod) < time.Minute {
		// Have a freash up-to-date build, probably just restarted.
		log.Logf(0, "current executable is up-to-date (%v)", exeTag)
		if err := osutil.LinkFiles(upd.latestDir, upd.currentDir, upd.syzFiles); err != nil {
			log.Fatal(err)
		}
		return
	}
	if exeTag == "" {
		log.Logf(0, "current executable is bootstrap")
	} else {
		log.Logf(0, "current executable is on %v", exeTag)
		log.Logf(0, "latest syzkaller build is on %v", latestTag)
	}

	// No syzkaller build or executable is stale.
	lastCommit := exeTag
	for {
		lastCommit = upd.pollAndBuild(lastCommit)
		latestTag := upd.checkLatest()
		if latestTag != "" {
			// The build was successful or we had the latest build from previous runs.
			// Either way, use the latest build.
			log.Logf(0, "using syzkaller built on %v", latestTag)
			if err := osutil.LinkFiles(upd.latestDir, upd.currentDir, upd.syzFiles); err != nil {
				log.Fatal(err)
			}
			if exeTag != latestTag {
				upd.UpdateAndRestart()
			}
			return
		}

		// No good build at all, try again later.
		log.Logf(0, "retrying in %v", buildRetryPeriod)
		select {
		case <-time.After(buildRetryPeriod):
		case <-shutdown:
			os.Exit(0)
		}
	}
}

// WaitForUpdate polls and rebuilds syzkaller.
// Returns when we have a new good build in latest.
func (upd *SyzUpdater) WaitForUpdate() {
	time.Sleep(syzkallerRebuildPeriod)
	latestTag := upd.checkLatest()
	lastCommit := latestTag
	for {
		lastCommit = upd.pollAndBuild(lastCommit)
		if latestTag != upd.checkLatest() {
			break
		}
		time.Sleep(buildRetryPeriod)
	}
	log.Logf(0, "syzkaller: update available, restarting")
}

// UpdateAndRestart updates and restarts the current executable.
// Does not return.
func (upd *SyzUpdater) UpdateAndRestart() {
	log.Logf(0, "restarting executable for update")
	latestBin := filepath.Join(upd.latestDir, "bin", upd.exe)
	latestTag := filepath.Join(upd.latestDir, "tag")
	if err := osutil.CopyFile(latestBin, upd.exe); err != nil {
		log.Fatal(err)
	}
	if err := osutil.CopyFile(latestTag, upd.exe+".tag"); err != nil {
		log.Fatal(err)
	}
	if err := syscall.Exec(upd.exe, os.Args, os.Environ()); err != nil {
		log.Fatal(err)
	}
	log.Fatalf("not reachable")
}

func (upd *SyzUpdater) pollAndBuild(lastCommit string) string {
	commit, err := upd.repo.Poll(upd.repoAddress, upd.branch)
	if err != nil {
		log.Logf(0, "syzkaller: failed to poll: %v", err)
		return lastCommit
	}
	log.Logf(0, "syzkaller: poll: %v (%v)", commit.Hash, commit.Title)
	if lastCommit != commit.Hash {
		log.Logf(0, "syzkaller: building ...")
		lastCommit = commit.Hash
		if err := upd.build(commit); err != nil {
			log.Logf(0, "syzkaller: %v", err)
		}
	}
	return lastCommit
}

func (upd *SyzUpdater) build(commit *vcs.Commit) error {
	if upd.descriptions != "" {
		files, err := ioutil.ReadDir(upd.descriptions)
		if err != nil {
			return fmt.Errorf("failed to read descriptions dir: %v", err)
		}
		for _, f := range files {
			src := filepath.Join(upd.descriptions, f.Name())
			dst := filepath.Join(upd.syzkallerDir, "sys", "linux", f.Name())
			if err := osutil.CopyFile(src, dst); err != nil {
				return err
			}
		}
	}
	cmd := osutil.Command("make", "generate")
	cmd.Dir = upd.syzkallerDir
	cmd.Env = append([]string{"GOPATH=" + upd.gopathDir}, os.Environ()...)
	if _, err := osutil.Run(time.Hour, cmd); err != nil {
		return fmt.Errorf("build failed: %v", err)
	}
	cmd = osutil.Command("make", "host", "ci")
	cmd.Dir = upd.syzkallerDir
	cmd.Env = append([]string{"GOPATH=" + upd.gopathDir}, os.Environ()...)
	if _, err := osutil.Run(time.Hour, cmd); err != nil {
		return fmt.Errorf("build failed: %v", err)
	}
	for target := range upd.targets {
		parts := strings.Split(target, "/")
		cmd := osutil.Command("make", "target")
		cmd.Dir = upd.syzkallerDir
		cmd.Env = append([]string{}, os.Environ()...)
		cmd.Env = append(cmd.Env,
			"GOPATH="+upd.gopathDir,
			"TARGETOS="+parts[0],
			"TARGETVMARCH="+parts[1],
			"TARGETARCH="+parts[2],
		)
		if _, err := osutil.Run(time.Hour, cmd); err != nil {
			return fmt.Errorf("build failed: %v", err)
		}
	}
	cmd = osutil.Command("go", "test", "-short", "./...")
	cmd.Dir = upd.syzkallerDir
	cmd.Env = append([]string{"GOPATH=" + upd.gopathDir}, os.Environ()...)
	if _, err := osutil.Run(time.Hour, cmd); err != nil {
		return fmt.Errorf("tests failed: %v", err)
	}
	tagFile := filepath.Join(upd.syzkallerDir, "tag")
	if err := osutil.WriteFile(tagFile, []byte(commit.Hash)); err != nil {
		return fmt.Errorf("filed to write tag file: %v", err)
	}
	if err := osutil.CopyFiles(upd.syzkallerDir, upd.latestDir, upd.syzFiles); err != nil {
		return fmt.Errorf("filed to copy syzkaller: %v", err)
	}
	return nil
}

// checkLatest returns tag of the latest build,
// or an empty string if latest build is missing/broken.
func (upd *SyzUpdater) checkLatest() string {
	if !osutil.FilesExist(upd.latestDir, upd.syzFiles) {
		return ""
	}
	tag, _ := readTag(filepath.Join(upd.latestDir, "tag"))
	return tag
}

func readTag(file string) (tag string, mod time.Time) {
	data, _ := ioutil.ReadFile(file)
	tag = string(data)
	if st, err := os.Stat(file); err == nil {
		mod = st.ModTime()
	}
	if tag == "" || mod.IsZero() {
		tag = ""
		mod = time.Time{}
	}
	return
}
<|endoftext|>


<commit_before>// autogenerated by windowsconstgen; do not edit
package ui

const _BCM_GETIDEALSIZE = 5633
const _BM_GETCHECK = 240
const _BN_CLICKED = 0
const _BST_CHECKED = 1
const _BS_AUTOCHECKBOX = 3
const _BS_PUSHBUTTON = 0
const _CBS_AUTOHSCROLL = 64
const _CBS_DROPDOWN = 2
const _CBS_DROPDOWNLIST = 3
const _CB_ADDSTRING = 323
const _CB_DELETESTRING = 324
const _CB_ERR = -1
const _CB_ERRSPACE = -2
const _CB_GETCOUNT = 326
const _CB_GETCURSEL = 327
const _CB_INSERTSTRING = 330
const _COLOR_BTNFACE = 15
const _CW_USEDEFAULT = -2147483648
const _ERROR = 0
const _ES_AUTOHSCROLL = 128
const _ES_PASSWORD = 32
const _GWL_STYLE = -16
const _ICC_PROGRESS_CLASS = 32
const _LBS_EXTENDEDSEL = 2048
const _LBS_NOINTEGRALHEIGHT = 256
const _LBS_NOTIFY = 1
const _LB_ADDSTRING = 384
const _LB_DELETESTRING = 386
const _LB_ERR = -1
const _LB_ERRSPACE = -2
const _LB_GETCOUNT = 395
const _LB_GETCURSEL = 392
const _LB_GETSELCOUNT = 400
const _LB_GETSELITEMS = 401
const _LB_GETTEXT = 393
const _LB_GETTEXTLEN = 394
const _LB_INSERTSTRING = 385
const _PBM_SETMARQUEE = 1034
const _PBM_SETPOS = 1026
const _PBS_MARQUEE = 8
const _PBS_SMOOTH = 1
const _SB_HORZ = 0
const _SB_LEFT = 6
const _SB_LINELEFT = 0
const _SB_LINERIGHT = 1
const _SB_PAGELEFT = 2
const _SB_PAGERIGHT = 3
const _SB_RIGHT = 7
const _SB_THUMBPOSITION = 4
const _SB_THUMBTRACK = 5
const _SB_VERT = 1
const _SIF_PAGE = 2
const _SIF_POS = 4
const _SIF_RANGE = 1
const _SIF_TRACKPOS = 16
const _SS_LEFTNOWORDWRAP = 12
const _SS_NOPREFIX = 128
const _SW_ERASE = 4
const _SW_HIDE = 0
const _SW_INVALIDATE = 2
const _SW_SHOW = 5
const _SW_SHOWDEFAULT = 10
const _WM_CLOSE = 16
const _WM_ERASEBKGND = 20
const _WM_GETMINMAXINFO = 36
const _WM_GETTEXT = 13
const _WM_GETTEXTLENGTH = 14
const _WM_HSCROLL = 276
const _WM_SETFONT = 48
const _WM_SIZE = 5
const _WM_VSCROLL = 277
const _WS_CHILD = 1073741824
const _WS_EX_CLIENTEDGE = 512
const _WS_HSCROLL = 1048576
const _WS_OVERLAPPEDWINDOW = 13565952
const _WS_TABSTOP = 65536
const _WS_VISIBLE = 268435456
const _WS_VSCROLL = 2097152
const _INVALID_HANDLE_VALUE = 4294967295
<commit_msg>And merged the old events_windows.go constants into zconstants_windows_GOARCH.go.<commit_after>// autogenerated by windowsconstgen; do not edit
package ui
const _BCM_GETIDEALSIZE = 5633
const _BM_GETCHECK = 240
const _BN_CLICKED = 0
const _BST_CHECKED = 1
const _BS_AUTOCHECKBOX = 3
const _BS_PUSHBUTTON = 0
const _CBS_AUTOHSCROLL = 64
const _CBS_DROPDOWN = 2
const _CBS_DROPDOWNLIST = 3
const _CB_ADDSTRING = 323
const _CB_DELETESTRING = 324
const _CB_ERR = -1
const _CB_ERRSPACE = -2
const _CB_GETCOUNT = 326
const _CB_GETCURSEL = 327
const _CB_INSERTSTRING = 330
const _COLOR_BTNFACE = 15
const _CW_USEDEFAULT = -2147483648
const _ERROR = 0
const _ES_AUTOHSCROLL = 128
const _ES_PASSWORD = 32
const _GWL_STYLE = -16
const _ICC_PROGRESS_CLASS = 32
const _LBS_EXTENDEDSEL = 2048
const _LBS_NOINTEGRALHEIGHT = 256
const _LBS_NOTIFY = 1
const _LB_ADDSTRING = 384
const _LB_DELETESTRING = 386
const _LB_ERR = -1
const _LB_ERRSPACE = -2
const _LB_GETCOUNT = 395
const _LB_GETCURSEL = 392
const _LB_GETSELCOUNT = 400
const _LB_GETSELITEMS = 401
const _LB_GETTEXT = 393
const _LB_GETTEXTLEN = 394
const _LB_INSERTSTRING = 385
const _MK_LBUTTON = 1
const _MK_MBUTTON = 16
const _MK_RBUTTON = 2
const _MK_XBUTTON1 = 32
const _MK_XBUTTON2 = 64
const _PBM_SETMARQUEE = 1034
const _PBM_SETPOS = 1026
const _PBS_MARQUEE = 8
const _PBS_SMOOTH = 1
const _SB_HORZ = 0
const _SB_LEFT = 6
const _SB_LINELEFT = 0
const _SB_LINERIGHT = 1
const _SB_PAGELEFT = 2
const _SB_PAGERIGHT = 3
const _SB_RIGHT = 7
const _SB_THUMBPOSITION = 4
const _SB_THUMBTRACK = 5
const _SB_VERT = 1
const _SIF_PAGE = 2
const _SIF_POS = 4
const _SIF_RANGE = 1
const _SIF_TRACKPOS = 16
const _SS_LEFTNOWORDWRAP = 12
const _SS_NOPREFIX = 128
const _SW_ERASE = 4
const _SW_HIDE = 0
const _SW_INVALIDATE = 2
const _SW_SHOW = 5
const _SW_SHOWDEFAULT = 10
const _VK_ADD = 107
const _VK_CONTROL = 17
const _VK_DELETE = 46
const _VK_DIVIDE = 111
const _VK_DOWN = 40
const _VK_END = 35
const _VK_ESCAPE = 27
const _VK_F1 = 112
const _VK_F10 = 121
const _VK_F11 = 122
const _VK_F12 = 123
const _VK_F2 = 113
const _VK_F3 = 114
const _VK_F4 = 115
const _VK_F5 = 116
const _VK_F6 = 117
const _VK_F7 = 118
const _VK_F8 = 119
const _VK_F9 = 120
const _VK_HOME = 36
const _VK_INSERT = 45
const _VK_LEFT = 37
const _VK_LWIN = 91
const _VK_MENU = 18
const _VK_MULTIPLY = 106
const _VK_NEXT = 34
const _VK_PRIOR = 33
const _VK_RETURN = 13
const _VK_RIGHT = 39
const _VK_RWIN = 92
const _VK_SHIFT = 16
const _VK_SUBTRACT = 109
const _VK_UP = 38
const _WM_ACTIVATE = 6
const _WM_CLOSE = 16
const _WM_ERASEBKGND = 20
const _WM_GETMINMAXINFO = 36
const _WM_GETTEXT = 13
const _WM_GETTEXTLENGTH = 14
const _WM_HSCROLL = 276
const _WM_KEYDOWN = 256
const _WM_KEYUP = 257
const _WM_LBUTTONDOWN = 513
const _WM_LBUTTONUP = 514
const _WM_MBUTTONDOWN = 519
const _WM_MBUTTONUP = 520
const _WM_MOUSEACTIVATE = 33
const _WM_MOUSEMOVE = 512
const _WM_RBUTTONDOWN = 516
const _WM_RBUTTONUP = 517
const _WM_SETFONT = 48
const _WM_SIZE = 5
const _WM_SYSKEYDOWN = 260
const _WM_SYSKEYUP = 261
const _WM_VSCROLL = 277
const _WM_XBUTTONDOWN = 523
const _WM_XBUTTONUP = 524
const _WS_CHILD = 1073741824
const _WS_EX_CLIENTEDGE = 512
const _WS_HSCROLL = 1048576
const _WS_OVERLAPPEDWINDOW = 13565952
const _WS_TABSTOP = 65536
const _WS_VISIBLE = 268435456
const _WS_VSCROLL = 2097152
const _INVALID_HANDLE_VALUE = 4294967295
<|endoftext|>