<commit_before>package revel

import (
	"html/template"
	"io"
	"log"
	"strings"
)

const GO_TEMPLATE = "go"

// Adapter for Go Templates.
type GoTemplate struct {
	*template.Template
	engine *GoEngine
	*TemplateView
}

// return a 'revel.Template' from Go's template.
func (gotmpl GoTemplate) Render(wr io.Writer, arg interface{}) error {
	return gotmpl.Execute(wr, arg)
}

type GoEngine struct {
	loader      *TemplateLoader
	templateSet *template.Template
	// TemplatesBylowerName is a map from lower case template name to the real template.
	templatesBylowerName map[string]*GoTemplate
	splitDelims          []string
	CaseInsensitiveMode bool
}

func (i *GoEngine) ConvertPath(path string) string {
	if i.CaseInsensitiveMode {
		return strings.ToLower(path)
	}
	return path
}

func (i *GoEngine) Handles(templateView *TemplateView) bool{
	return EngineHandles(i, templateView)
}

func (engine *GoEngine) ParseAndAdd(baseTemplate *TemplateView) error {
	// If alternate delimiters set for the project, change them for this set
	if engine.splitDelims != nil && baseTemplate.Location() == ViewsPath {
		engine.templateSet.Delims(engine.splitDelims[0], engine.splitDelims[1])
	} else {
		// Reset to default otherwise
		engine.templateSet.Delims("", "")
	}
	templateSource := string(baseTemplate.FileBytes)
	lowerTemplateName := engine.ConvertPath(baseTemplate.TemplateName)
	tpl, err := engine.templateSet.New(baseTemplate.TemplateName).Parse(templateSource)
	if nil != err {
		_, line, description := ParseTemplateError(err)
		return &Error{
			Title:       "Template Compilation Error",
			Path:        baseTemplate.TemplateName,
			Description: description,
			Line:        line,
			SourceLines: strings.Split(templateSource, "\n"),
		}
	}
	engine.templatesBylowerName[lowerTemplateName] = &GoTemplate{Template: tpl, engine: engine, TemplateView: baseTemplate}
	return nil
}

func (engine *GoEngine) Lookup(templateName string) Template {
	// Case-insensitive matching of template file name
	if tpl, found := engine.templatesBylowerName[engine.ConvertPath(templateName)]; found {
		return tpl
	}
	return nil
}
func (engine *GoEngine) Name() string {
	return GO_TEMPLATE
}
func (engine *GoEngine) Event(action int, i interface{}) {
	if action == TEMPLATE_REFRESH_REQUESTED {
		// At this point all the templates have been passed into the
		engine.templatesBylowerName = map[string]*GoTemplate{}
		engine.templateSet = template.New("__root__").Funcs(TemplateFuncs)
		// Check to see what should be used for case sensitivity
		engine.CaseInsensitiveMode = Config.StringDefault("go.template.path", "lower") != "case"
	}
}
func init() {
	RegisterTemplateLoader(GO_TEMPLATE, func(loader *TemplateLoader) (TemplateEngine, error) {
		// Set the template delimiters for the project if present, then split into left
		// and right delimiters around a space character

		TemplateDelims := Config.StringDefault("template.go.delimiters", "")
		var splitDelims []string
		if TemplateDelims != "" {
			splitDelims = strings.Split(TemplateDelims, " ")
			if len(splitDelims) != 2 {
				log.Fatalln("app.conf: Incorrect format for template.delimiters")
			}
		}

		return &GoEngine{
			loader:               loader,
			templateSet:          template.New("__root__").Funcs(TemplateFuncs),
			templatesBylowerName: map[string]*GoTemplate{},
			splitDelims:          splitDelims,
		}, nil
	})
}
<commit_msg>Fixes template delimiters issue Closes #1199<commit_after>package revel

import (
	"html/template"
	"io"
	"log"
	"strings"
)

const GO_TEMPLATE = "go"

// Adapter for Go Templates.
type GoTemplate struct {
	*template.Template
	engine *GoEngine
	*TemplateView
}

// return a 'revel.Template' from Go's template.
func (gotmpl GoTemplate) Render(wr io.Writer, arg interface{}) error {
	return gotmpl.Execute(wr, arg)
}

type GoEngine struct {
	loader      *TemplateLoader
	templateSet *template.Template
	// TemplatesBylowerName is a map from lower case template name to the real template.
	templatesBylowerName map[string]*GoTemplate
	splitDelims          []string
	CaseInsensitiveMode bool
}

func (i *GoEngine) ConvertPath(path string) string {
	if i.CaseInsensitiveMode {
		return strings.ToLower(path)
	}
	return path
}

func (i *GoEngine) Handles(templateView *TemplateView) bool{
	return EngineHandles(i, templateView)
}

func (engine *GoEngine) ParseAndAdd(baseTemplate *TemplateView) error {
	// If alternate delimiters set for the project, change them for this set
	if engine.splitDelims != nil && strings.Index(baseTemplate.Location(), ViewsPath)>-1 {
		engine.templateSet.Delims(engine.splitDelims[0], engine.splitDelims[1])
	} else {
		// Reset to default otherwise
		engine.templateSet.Delims("", "")
	}
	templateSource := string(baseTemplate.FileBytes)
	lowerTemplateName := engine.ConvertPath(baseTemplate.TemplateName)
	tpl, err := engine.templateSet.New(baseTemplate.TemplateName).Parse(templateSource)
	if nil != err {
		_, line, description := ParseTemplateError(err)
		return &Error{
			Title:       "Template Compilation Error",
			Path:        baseTemplate.TemplateName,
			Description: description,
			Line:        line,
			SourceLines: strings.Split(templateSource, "\n"),
		}
	}
	engine.templatesBylowerName[lowerTemplateName] = &GoTemplate{Template: tpl, engine: engine, TemplateView: baseTemplate}
	return nil
}

func (engine *GoEngine) Lookup(templateName string) Template {
	// Case-insensitive matching of template file name
	if tpl, found := engine.templatesBylowerName[engine.ConvertPath(templateName)]; found {
		return tpl
	}
	return nil
}
func (engine *GoEngine) Name() string {
	return GO_TEMPLATE
}
func (engine *GoEngine) Event(action int, i interface{}) {
	if action == TEMPLATE_REFRESH_REQUESTED {
		// At this point all the templates have been passed into the
		engine.templatesBylowerName = map[string]*GoTemplate{}
		engine.templateSet = template.New("__root__").Funcs(TemplateFuncs)
		// Check to see what should be used for case sensitivity
		engine.CaseInsensitiveMode = Config.StringDefault("go.template.path", "lower") != "case"
	}
}
func init() {
	RegisterTemplateLoader(GO_TEMPLATE, func(loader *TemplateLoader) (TemplateEngine, error) {
		// Set the template delimiters for the project if present, then split into left
		// and right delimiters around a space character

		TemplateDelims := Config.StringDefault("template.go.delimiters", "")
		var splitDelims []string
		if TemplateDelims != "" {
			splitDelims = strings.Split(TemplateDelims, " ")
			if len(splitDelims) != 2 {
				log.Fatalln("app.conf: Incorrect format for template.delimiters")
			}
		}

		return &GoEngine{
			loader:               loader,
			templateSet:          template.New("__root__").Funcs(TemplateFuncs),
			templatesBylowerName: map[string]*GoTemplate{},
			splitDelims:          splitDelims,
		}, nil
	})
}
<|endoftext|>


<commit_before>// Copyright 2014 The Cayley Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package db

import (
	"fmt"
	"io"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"time"

	"github.com/peterh/liner"

	"github.com/google/cayley/config"
	"github.com/google/cayley/graph"
	"github.com/google/cayley/quad/cquads"
	"github.com/google/cayley/query"
	"github.com/google/cayley/query/gremlin"
	"github.com/google/cayley/query/mql"
	"github.com/google/cayley/query/sexp"
)

func trace(s string) (string, time.Time) {
	return s, time.Now()
}

func un(s string, startTime time.Time) {
	endTime := time.Now()

	fmt.Printf(s, float64(endTime.UnixNano()-startTime.UnixNano())/float64(1E6))
}

func Run(query string, ses query.Session) {
	nResults := 0
	startTrace, startTime := trace("Elapsed time: %g ms\n\n")
	defer func() {
		if nResults > 0 {
			un(startTrace, startTime)
		}
	}()
	fmt.Printf("\n")
	c := make(chan interface{}, 5)
	go ses.Execute(query, c, 100)
	for res := range c {
		fmt.Print(ses.Format(res))
		nResults++
	}
	if nResults > 0 {
		fmt.Printf("-----------\n%d Results\n", nResults)
	}
}

const (
	ps1 = "cayley> "
	ps2 = "...     "

	history = ".cayley_history"
)

func Repl(h *graph.Handle, queryLanguage string, cfg *config.Config) error {
	var ses query.Session
	switch queryLanguage {
	case "sexp":
		ses = sexp.NewSession(h.QuadStore)
	case "mql":
		ses = mql.NewSession(h.QuadStore)
	case "gremlin":
		fallthrough
	default:
		ses = gremlin.NewSession(h.QuadStore, cfg.Timeout, true)
	}

	term, err := terminal(history)
	if os.IsNotExist(err) {
		fmt.Printf("creating new history file: %q\n", history)
	}
	defer persist(term, history)

	var (
		prompt = ps1

		code string
	)

	for {
		if len(code) == 0 {
			prompt = ps1
		} else {
			prompt = ps2
		}
		line, err := term.Prompt(prompt)
		if err != nil {
			if err == io.EOF {
				fmt.Println()
				return nil
			}
			return err
		}

		term.AppendHistory(line)

		line = strings.TrimSpace(line)
		if len(line) == 0 || line[0] == '#' {
			continue
		}

		if code == "" {
			cmd, args := splitLine(line)

			switch cmd {
			case ":debug":
				args = strings.TrimSpace(args)
				var debug bool
				switch args {
				case "t":
					debug = true
				case "f":
					// Do nothing.
				default:
					debug, err = strconv.ParseBool(args)
					if err != nil {
						fmt.Printf("Error: cannot parse %q as a valid boolean - acceptable values: 't'|'true' or 'f'|'false'\n", args)
						continue
					}
				}
				ses.Debug(debug)
				fmt.Printf("Debug set to %t\n", debug)
				continue

			case ":a":
				quad, err := cquads.Parse(args)
				if err != nil {
					fmt.Printf("Error: not a valid quad: %v\n", err)
					continue
				}

				h.QuadWriter.AddQuad(quad)
				continue

			case ":d":
				quad, err := cquads.Parse(args)
				if err != nil {
					fmt.Printf("Error: not a valid quad: %v\n", err)
					continue
				}
				h.QuadWriter.RemoveQuad(quad)
				continue

			case "exit":
				term.Close()
				os.Exit(0)

			default:
				if cmd[0] == ':' {
					fmt.Printf("Unknown command: %q\n", cmd)
					continue
				}
			}
		}

		code += line

		result, err := ses.Parse(code)
		switch result {
		case query.Parsed:
			Run(code, ses)
			code = ""
		case query.ParseFail:
			fmt.Println("Error: ", err)
			code = ""
		case query.ParseMore:
		}
	}
}

// Splits a line into a command and its arguments
// e.g. ":a b c d ." will be split into ":a" and " b c d ."
func splitLine(line string) (string, string) {
	var command, arguments string

	line = strings.TrimSpace(line)

	// An empty line/a line consisting of whitespace contains neither command nor arguments
	if len(line) > 0 {
		command = strings.Fields(line)[0]

		// A line containing only a command has no arguments
		if len(line) > len(command) {
			arguments = line[len(command):]
		}
	}

	return command, arguments
}

func terminal(path string) (*liner.State, error) {
	term := liner.NewLiner()

	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, os.Interrupt, os.Kill)
		<-c

		err := persist(term, history)
		if err != nil {
			fmt.Fprintf(os.Stderr, "failed to properly clean up terminal: %v\n", err)
			os.Exit(1)
		}

		os.Exit(0)
	}()

	f, err := os.Open(path)
	if err != nil {
		return term, err
	}
	defer f.Close()
	_, err = term.ReadHistory(f)
	return term, err
}

func persist(term *liner.State, path string) error {
	f, err := os.OpenFile(path, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666)
	if err != nil {
		return fmt.Errorf("could not open %q to append history: %v", path, err)
	}
	defer f.Close()
	_, err = term.WriteHistory(f)
	if err != nil {
		return fmt.Errorf("could not write history to %q: %v", path, err)
	}
	return term.Close()
}
<commit_msg>No plural when only one result<commit_after>// Copyright 2014 The Cayley Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package db

import (
	"fmt"
	"io"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"time"

	"github.com/peterh/liner"

	"github.com/google/cayley/config"
	"github.com/google/cayley/graph"
	"github.com/google/cayley/quad/cquads"
	"github.com/google/cayley/query"
	"github.com/google/cayley/query/gremlin"
	"github.com/google/cayley/query/mql"
	"github.com/google/cayley/query/sexp"
)

func trace(s string) (string, time.Time) {
	return s, time.Now()
}

func un(s string, startTime time.Time) {
	endTime := time.Now()

	fmt.Printf(s, float64(endTime.UnixNano()-startTime.UnixNano())/float64(1E6))
}

func Run(query string, ses query.Session) {
	nResults := 0
	startTrace, startTime := trace("Elapsed time: %g ms\n\n")
	defer func() {
		if nResults > 0 {
			un(startTrace, startTime)
		}
	}()
	fmt.Printf("\n")
	c := make(chan interface{}, 5)
	go ses.Execute(query, c, 100)
	for res := range c {
		fmt.Print(ses.Format(res))
		nResults++
	}
	if nResults > 0 {
		results := "Result"
		if nResults > 1 {
			results += "s"
		}
		fmt.Printf("-----------\n%d %s\n", nResults, results)
	}
}

const (
	ps1 = "cayley> "
	ps2 = "...     "

	history = ".cayley_history"
)

func Repl(h *graph.Handle, queryLanguage string, cfg *config.Config) error {
	var ses query.Session
	switch queryLanguage {
	case "sexp":
		ses = sexp.NewSession(h.QuadStore)
	case "mql":
		ses = mql.NewSession(h.QuadStore)
	case "gremlin":
		fallthrough
	default:
		ses = gremlin.NewSession(h.QuadStore, cfg.Timeout, true)
	}

	term, err := terminal(history)
	if os.IsNotExist(err) {
		fmt.Printf("creating new history file: %q\n", history)
	}
	defer persist(term, history)

	var (
		prompt = ps1

		code string
	)

	for {
		if len(code) == 0 {
			prompt = ps1
		} else {
			prompt = ps2
		}
		line, err := term.Prompt(prompt)
		if err != nil {
			if err == io.EOF {
				fmt.Println()
				return nil
			}
			return err
		}

		term.AppendHistory(line)

		line = strings.TrimSpace(line)
		if len(line) == 0 || line[0] == '#' {
			continue
		}

		if code == "" {
			cmd, args := splitLine(line)

			switch cmd {
			case ":debug":
				args = strings.TrimSpace(args)
				var debug bool
				switch args {
				case "t":
					debug = true
				case "f":
					// Do nothing.
				default:
					debug, err = strconv.ParseBool(args)
					if err != nil {
						fmt.Printf("Error: cannot parse %q as a valid boolean - acceptable values: 't'|'true' or 'f'|'false'\n", args)
						continue
					}
				}
				ses.Debug(debug)
				fmt.Printf("Debug set to %t\n", debug)
				continue

			case ":a":
				quad, err := cquads.Parse(args)
				if err != nil {
					fmt.Printf("Error: not a valid quad: %v\n", err)
					continue
				}

				h.QuadWriter.AddQuad(quad)
				continue

			case ":d":
				quad, err := cquads.Parse(args)
				if err != nil {
					fmt.Printf("Error: not a valid quad: %v\n", err)
					continue
				}
				h.QuadWriter.RemoveQuad(quad)
				continue

			case "exit":
				term.Close()
				os.Exit(0)

			default:
				if cmd[0] == ':' {
					fmt.Printf("Unknown command: %q\n", cmd)
					continue
				}
			}
		}

		code += line

		result, err := ses.Parse(code)
		switch result {
		case query.Parsed:
			Run(code, ses)
			code = ""
		case query.ParseFail:
			fmt.Println("Error: ", err)
			code = ""
		case query.ParseMore:
		}
	}
}

// Splits a line into a command and its arguments
// e.g. ":a b c d ." will be split into ":a" and " b c d ."
func splitLine(line string) (string, string) {
	var command, arguments string

	line = strings.TrimSpace(line)

	// An empty line/a line consisting of whitespace contains neither command nor arguments
	if len(line) > 0 {
		command = strings.Fields(line)[0]

		// A line containing only a command has no arguments
		if len(line) > len(command) {
			arguments = line[len(command):]
		}
	}

	return command, arguments
}

func terminal(path string) (*liner.State, error) {
	term := liner.NewLiner()

	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, os.Interrupt, os.Kill)
		<-c

		err := persist(term, history)
		if err != nil {
			fmt.Fprintf(os.Stderr, "failed to properly clean up terminal: %v\n", err)
			os.Exit(1)
		}

		os.Exit(0)
	}()

	f, err := os.Open(path)
	if err != nil {
		return term, err
	}
	defer f.Close()
	_, err = term.ReadHistory(f)
	return term, err
}

func persist(term *liner.State, path string) error {
	f, err := os.OpenFile(path, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666)
	if err != nil {
		return fmt.Errorf("could not open %q to append history: %v", path, err)
	}
	defer f.Close()
	_, err = term.WriteHistory(f)
	if err != nil {
		return fmt.Errorf("could not write history to %q: %v", path, err)
	}
	return term.Close()
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"go/format"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strings"
	"text/template"
)

// ResourceGenerator takes AWS CloudFormation Resource Specification
// documents, and generates Go structs and a JSON Schema from them.
type ResourceGenerator struct {
	filenames map[string]string
	urls      map[string]string
	Results   *ResourceGeneratorResults
}

// ResourceGeneratorResults contains a summary of the items generated
type ResourceGeneratorResults struct {
	UpdatedResources map[string]string
	UpdatedSchemas   map[string]string
	ProcessedCount   int
}

// NewResourceGenerator an array of AWS CloudFormation Resource Specification
// documents, and generates Go structs and a JSON Schema from them.
// The input can be a mix of URLs (https://) or files (file://).
func NewResourceGenerator(urls map[string]string) (*ResourceGenerator, error) {

	rg := &ResourceGenerator{
		filenames: map[string]string{},
		urls:      map[string]string{},
		Results: &ResourceGeneratorResults{
			UpdatedResources: map[string]string{},
			UpdatedSchemas:   map[string]string{},
			ProcessedCount:   0,
		},
	}

	for name, found := range urls {

		uri, err := url.Parse(found)
		if err != nil {
			return nil, err
		}

		switch uri.Scheme {
		case "https":
			rg.urls[name] = uri.String()
		case "http":
			rg.urls[name] = uri.String()
		case "file":
			rg.filenames[name] = uri.String()
		default:
			return nil, fmt.Errorf("invalid URL scheme %s", uri.Scheme)
		}

	}

	return rg, nil

}

// Generate generates Go structs and a JSON Schema from the AWS CloudFormation
// Resource Specifications provided to NewResourceGenerator
func (rg *ResourceGenerator) Generate() error {

	for name, uri := range rg.urls {
		fmt.Printf("Downloading %s specification from %s\n", name, uri)
		data, err := rg.downloadSpec(uri)
		if err != nil {
			return err
		}
		spec, err := rg.processSpec(name, data)
		if err != nil {
			return err
		}
		if err := rg.generateJSONSchema(name, spec); err != nil {
			return err
		}
	}

	for name, filename := range rg.filenames {
		fmt.Printf("Downloading %s specification from %s\n", name, filename)
		data, err := ioutil.ReadFile(strings.Replace(filename, "file://", "", -1))
		if err != nil {
			return err
		}
		spec, err := rg.processSpec(name, data)
		if err != nil {
			return err
		}
		if err := rg.generateJSONSchema(name, spec); err != nil {
			return err
		}

	}

	return nil

}

func (rg *ResourceGenerator) downloadSpec(uri string) ([]byte, error) {

	response, err := http.Get(uri)
	if err != nil {
		return nil, err
	}

	data, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, err
	}

	return data, nil

}

func (rg *ResourceGenerator) processSpec(specname string, data []byte) (*CloudFormationResourceSpecification, error) {

	// Unmarshall the JSON specification
	spec := &CloudFormationResourceSpecification{}
	if err := json.Unmarshal(data, spec); err != nil {
		return nil, err
	}

	// Write all of the resources in the spec file
	for name, resource := range spec.Resources {
		if err := rg.generateResources(name, resource, false, spec); err != nil {
			return nil, err
		}
	}

	// Write all of the custom types in the spec file
	for name, resource := range spec.Properties {
		if err := rg.generateResources(name, resource, true, spec); err != nil {
			return nil, err
		}
	}

	return spec, nil

}

func (rg *ResourceGenerator) generateResources(name string, resource Resource, isCustomProperty bool, spec *CloudFormationResourceSpecification) error {

	// Open the resource template
	tmpl, err := template.ParseFiles("generate/templates/resource.template")
	if err != nil {
		return fmt.Errorf("failed to load resource template: %s", err)
	}

	// Pass in the following information into the template
	sname := structName(name)
	structNameParts := strings.Split(name, ".")
	basename := structName(structNameParts[0])

	templateData := struct {
		Name             string
		StructName       string
		Basename         string
		Resource         Resource
		IsCustomProperty bool
		Version          string
	}{
		Name:             name,
		StructName:       sname,
		Basename:         basename,
		Resource:         resource,
		IsCustomProperty: isCustomProperty,
		Version:          spec.ResourceSpecificationVersion,
	}

	// Execute the template, writing it to a buffer
	var buf bytes.Buffer
	err = tmpl.Execute(&buf, templateData)
	if err != nil {
		return fmt.Errorf("failed to generate resource %s: %s", name, err)
	}

	// Format the generated Go code with gofmt
	formatted, err := format.Source(buf.Bytes())
	if err != nil {
		return fmt.Errorf("failed to format Go file for resource %s: %s", name, err)
	}

	// Check if the file has changed since the last time generate ran
	fn := "cloudformation/" + filename(name)
	current, err := ioutil.ReadFile(fn)

	if err != nil || bytes.Compare(formatted, current) != 0 {

		// Write the file contents out
		if err := ioutil.WriteFile(fn, formatted, 0644); err != nil {
			return fmt.Errorf("failed to write resource file %s: %s", fn, err)
		}

		// Log the updated class name to the results
		rg.Results.UpdatedResources[fn] = templateData.Basename + "_" + templateData.StructName

	}

	rg.Results.ProcessedCount++

	return nil

}

func (rg *ResourceGenerator) generateJSONSchema(specname string, spec *CloudFormationResourceSpecification) error {

	// Open the schema template and setup a counter function that will
	// available in the template to be used to detect when trailing commas
	// are required in the JSON when looping through maps
	tmpl, err := template.New("schema.template").Funcs(template.FuncMap{
		"counter": counter,
	}).ParseFiles("generate/templates/schema.template")

	var buf bytes.Buffer

	// Execute the template, writing it to file
	err = tmpl.Execute(&buf, spec)
	if err != nil {
		return fmt.Errorf("failed to generate JSON Schema: %s", err)
	}

	// Parse it to JSON objects and back again to format it
	var j interface{}
	if err := json.Unmarshal(buf.Bytes(), &j); err != nil {
		return fmt.Errorf("failed to unmarshal JSON Schema: %s", err)
	}

	formatted, err := json.MarshalIndent(j, "", "    ")
	if err != nil {
		return fmt.Errorf("failed to marshal JSON Schema: %s", err)

	}

	filename := fmt.Sprintf("schema/%s.schema.json", specname)
	if err := ioutil.WriteFile(filename, formatted, 0644); err != nil {
		return fmt.Errorf("failed to write JSON Schema: %s", err)
	}

	rg.Results.UpdatedSchemas[filename] = specname

	return nil

}

func generatePolymorphicProperty(name string, property Property) {

	// Open the polymorphic property template
	tmpl, err := template.New("polymorphic-property.template").Funcs(template.FuncMap{
		"convertToGoType": convertTypeToGo,
	}).ParseFiles("generate/templates/polymorphic-property.template")

	nameParts := strings.Split(name, "_")

	types := append([]string{}, property.PrimitiveTypes...)
	types = append(types, property.PrimitiveItemTypes...)
	types = append(types, property.ItemTypes...)
	types = append(types, property.Types...)

	templateData := struct {
		Name        string
		Basename    string
		Property    Property
		Types       []string
		TypesJoined string
	}{
		Name:        name,
		Basename:    nameParts[0],
		Property:    property,
		Types:       types,
		TypesJoined: conjoin("or", types),
	}

	// Execute the template, writing it to file
	var buf bytes.Buffer
	err = tmpl.Execute(&buf, templateData)
	if err != nil {
		fmt.Printf("Error: Failed to generate polymorphic property %s\n%s\n", name, err)
		os.Exit(1)
	}

	// Format the generated Go file with gofmt
	formatted, err := format.Source(buf.Bytes())
	if err != nil {
		fmt.Printf("Error: Failed to format Go file for resource %s\n%s\n", name, err)
		os.Exit(1)
	}

	// Write the file out
	if err := ioutil.WriteFile("cloudformation/"+filename(name), formatted, 0644); err != nil {
		fmt.Printf("Error: Failed to write JSON Schema\n%s\n", err)
		os.Exit(1)
	}

}

// counter is used within the JSON Schema template to determin whether or not
// to put a comma after a JSON resource (i.e. if it's the last element, then no comma)
// see: http://android.wekeepcoding.com/article/10126058/Go+template+remove+the+last+comma+in+range+loop
func counter(length int) func() int {
	i := length
	return func() int {
		i--
		return i
	}
}

func conjoin(conj string, items []string) string {
	if len(items) == 0 {
		return ""
	}
	if len(items) == 1 {
		return items[0]
	}
	if len(items) == 2 { // "a and b" not "a, and b"
		return items[0] + " " + conj + " " + items[1]
	}

	sep := ", "
	pieces := []string{items[0]}
	for _, item := range items[1 : len(items)-1] {
		pieces = append(pieces, sep, item)
	}
	pieces = append(pieces, sep, conj, " ", items[len(items)-1])

	return strings.Join(pieces, "")
}
<commit_msg>Tweaked go generate output<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"go/format"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strings"
	"text/template"
)

// ResourceGenerator takes AWS CloudFormation Resource Specification
// documents, and generates Go structs and a JSON Schema from them.
type ResourceGenerator struct {
	filenames map[string]string
	urls      map[string]string
	Results   *ResourceGeneratorResults
}

// ResourceGeneratorResults contains a summary of the items generated
type ResourceGeneratorResults struct {
	UpdatedResources map[string]string
	UpdatedSchemas   map[string]string
	ProcessedCount   int
}

// NewResourceGenerator an array of AWS CloudFormation Resource Specification
// documents, and generates Go structs and a JSON Schema from them.
// The input can be a mix of URLs (https://) or files (file://).
func NewResourceGenerator(urls map[string]string) (*ResourceGenerator, error) {

	rg := &ResourceGenerator{
		filenames: map[string]string{},
		urls:      map[string]string{},
		Results: &ResourceGeneratorResults{
			UpdatedResources: map[string]string{},
			UpdatedSchemas:   map[string]string{},
			ProcessedCount:   0,
		},
	}

	for name, found := range urls {

		uri, err := url.Parse(found)
		if err != nil {
			return nil, err
		}

		switch uri.Scheme {
		case "https":
			rg.urls[name] = uri.String()
		case "http":
			rg.urls[name] = uri.String()
		case "file":
			rg.filenames[name] = uri.String()
		default:
			return nil, fmt.Errorf("invalid URL scheme %s", uri.Scheme)
		}

	}

	return rg, nil

}

// Generate generates Go structs and a JSON Schema from the AWS CloudFormation
// Resource Specifications provided to NewResourceGenerator
func (rg *ResourceGenerator) Generate() error {

	for name, uri := range rg.urls {
		fmt.Printf("Downloading %s specification from %s\n", name, uri)
		data, err := rg.downloadSpec(uri)
		if err != nil {
			return err
		}
		spec, err := rg.processSpec(name, data)
		if err != nil {
			return err
		}
		if err := rg.generateJSONSchema(name, spec); err != nil {
			return err
		}
	}

	for name, filename := range rg.filenames {
		fmt.Printf("Downloading %s specification from %s\n", name, filename)
		data, err := ioutil.ReadFile(strings.Replace(filename, "file://", "", -1))
		if err != nil {
			return err
		}
		spec, err := rg.processSpec(name, data)
		if err != nil {
			return err
		}
		if err := rg.generateJSONSchema(name, spec); err != nil {
			return err
		}

	}

	return nil

}

func (rg *ResourceGenerator) downloadSpec(uri string) ([]byte, error) {

	response, err := http.Get(uri)
	if err != nil {
		return nil, err
	}

	data, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, err
	}

	return data, nil

}

func (rg *ResourceGenerator) processSpec(specname string, data []byte) (*CloudFormationResourceSpecification, error) {

	// Unmarshall the JSON specification
	spec := &CloudFormationResourceSpecification{}
	if err := json.Unmarshal(data, spec); err != nil {
		return nil, err
	}

	// Write all of the resources in the spec file
	for name, resource := range spec.Resources {
		if err := rg.generateResources(name, resource, false, spec); err != nil {
			return nil, err
		}
	}

	// Write all of the custom types in the spec file
	for name, resource := range spec.Properties {
		if err := rg.generateResources(name, resource, true, spec); err != nil {
			return nil, err
		}
	}

	return spec, nil

}

func (rg *ResourceGenerator) generateResources(name string, resource Resource, isCustomProperty bool, spec *CloudFormationResourceSpecification) error {

	// Open the resource template
	tmpl, err := template.ParseFiles("generate/templates/resource.template")
	if err != nil {
		return fmt.Errorf("failed to load resource template: %s", err)
	}

	// Pass in the following information into the template
	sname := structName(name)
	structNameParts := strings.Split(name, ".")
	basename := structName(structNameParts[0])

	templateData := struct {
		Name             string
		StructName       string
		Basename         string
		Resource         Resource
		IsCustomProperty bool
		Version          string
	}{
		Name:             name,
		StructName:       sname,
		Basename:         basename,
		Resource:         resource,
		IsCustomProperty: isCustomProperty,
		Version:          spec.ResourceSpecificationVersion,
	}

	// Execute the template, writing it to a buffer
	var buf bytes.Buffer
	err = tmpl.Execute(&buf, templateData)
	if err != nil {
		return fmt.Errorf("failed to generate resource %s: %s", name, err)
	}

	// Format the generated Go code with gofmt
	formatted, err := format.Source(buf.Bytes())
	if err != nil {
		return fmt.Errorf("failed to format Go file for resource %s: %s", name, err)
	}

	// Check if the file has changed since the last time generate ran
	fn := "cloudformation/" + filename(name)
	current, err := ioutil.ReadFile(fn)

	if err != nil || bytes.Compare(formatted, current) != 0 {

		// Write the file contents out
		if err := ioutil.WriteFile(fn, formatted, 0644); err != nil {
			return fmt.Errorf("failed to write resource file %s: %s", fn, err)
		}

		// Log the updated resource name to the results
		rg.Results.UpdatedResources[fn] = name

	}

	rg.Results.ProcessedCount++

	return nil

}

func (rg *ResourceGenerator) generateJSONSchema(specname string, spec *CloudFormationResourceSpecification) error {

	// Open the schema template and setup a counter function that will
	// available in the template to be used to detect when trailing commas
	// are required in the JSON when looping through maps
	tmpl, err := template.New("schema.template").Funcs(template.FuncMap{
		"counter": counter,
	}).ParseFiles("generate/templates/schema.template")

	var buf bytes.Buffer

	// Execute the template, writing it to file
	err = tmpl.Execute(&buf, spec)
	if err != nil {
		return fmt.Errorf("failed to generate JSON Schema: %s", err)
	}

	// Parse it to JSON objects and back again to format it
	var j interface{}
	if err := json.Unmarshal(buf.Bytes(), &j); err != nil {
		return fmt.Errorf("failed to unmarshal JSON Schema: %s", err)
	}

	formatted, err := json.MarshalIndent(j, "", "    ")
	if err != nil {
		return fmt.Errorf("failed to marshal JSON Schema: %s", err)

	}

	filename := fmt.Sprintf("schema/%s.schema.json", specname)
	if err := ioutil.WriteFile(filename, formatted, 0644); err != nil {
		return fmt.Errorf("failed to write JSON Schema: %s", err)
	}

	rg.Results.UpdatedSchemas[filename] = specname

	return nil

}

func generatePolymorphicProperty(name string, property Property) {

	// Open the polymorphic property template
	tmpl, err := template.New("polymorphic-property.template").Funcs(template.FuncMap{
		"convertToGoType": convertTypeToGo,
	}).ParseFiles("generate/templates/polymorphic-property.template")

	nameParts := strings.Split(name, "_")

	types := append([]string{}, property.PrimitiveTypes...)
	types = append(types, property.PrimitiveItemTypes...)
	types = append(types, property.ItemTypes...)
	types = append(types, property.Types...)

	templateData := struct {
		Name        string
		Basename    string
		Property    Property
		Types       []string
		TypesJoined string
	}{
		Name:        name,
		Basename:    nameParts[0],
		Property:    property,
		Types:       types,
		TypesJoined: conjoin("or", types),
	}

	// Execute the template, writing it to file
	var buf bytes.Buffer
	err = tmpl.Execute(&buf, templateData)
	if err != nil {
		fmt.Printf("Error: Failed to generate polymorphic property %s\n%s\n", name, err)
		os.Exit(1)
	}

	// Format the generated Go file with gofmt
	formatted, err := format.Source(buf.Bytes())
	if err != nil {
		fmt.Printf("Error: Failed to format Go file for resource %s\n%s\n", name, err)
		os.Exit(1)
	}

	// Write the file out
	if err := ioutil.WriteFile("cloudformation/"+filename(name), formatted, 0644); err != nil {
		fmt.Printf("Error: Failed to write JSON Schema\n%s\n", err)
		os.Exit(1)
	}

}

// counter is used within the JSON Schema template to determin whether or not
// to put a comma after a JSON resource (i.e. if it's the last element, then no comma)
// see: http://android.wekeepcoding.com/article/10126058/Go+template+remove+the+last+comma+in+range+loop
func counter(length int) func() int {
	i := length
	return func() int {
		i--
		return i
	}
}

func conjoin(conj string, items []string) string {
	if len(items) == 0 {
		return ""
	}
	if len(items) == 1 {
		return items[0]
	}
	if len(items) == 2 { // "a and b" not "a, and b"
		return items[0] + " " + conj + " " + items[1]
	}

	sep := ", "
	pieces := []string{items[0]}
	for _, item := range items[1 : len(items)-1] {
		pieces = append(pieces, sep, item)
	}
	pieces = append(pieces, sep, conj, " ", items[len(items)-1])

	return strings.Join(pieces, "")
}
<|endoftext|>


<commit_before>// Package timesafeguard collects the time of other nodes and ensures the
// remote times are not diverging too much from the local time. This is useful
// to ensure the following situation does not happen:
//
// 1. In a network of 3 nodes, node 1 goes down.
//
// 2. Node A comes back up, but with a 1-hour clock drift due to the hardware
// clock being in local time instead of UTC. ntpd refuses to correct the
// time because the drift is too big.
//
// 3. Node A re-joins the network.
//
// 4. At some point, node A becomes the leader. Message timestamps made a jump
// from e.g. 1432323893 to 1432327493, i.e. one hour into the future.
//
// 5. At some point, a different node becomes the leader. When trying to apply
// a message, the node panics because the message timestamp is not
// monotonically increasing.
//
// 6. The network needs to be frozen for an hour to be healthy again.
//
// timesafeguard ensures that the time is not off by more than
// |ElectionTimeout|, which presents the lower bound on how long an election
// takes:
//
// 1. In order for a candidate to win an election, the candidate needs to have
// the most recently committed entry in its log, otherwise it will not get a
// majority of votes.
//
// 2. The only way to get the most recently committed entry is to receive an
// appendEntries RPC, which resets the election timer. Hence, the earliest
// point at which any node can start an election that has any chance to be
// successful is after |ElectionTimeout| passed.
//
// Since during an election, no new messages are committed, a clock drift of
// |ElectionTimeout| is okay: the first message on the new leader will be
// committed late enough that it has a higher timestamp than the last message
// on the old leader.
package timesafeguard

import (
	"flag"
	"fmt"
	"log"
	"strings"
	"sync"
	"time"

	"github.com/robustirc/robustirc/util"
	"github.com/stapelberg/glog"
)

var DisableTimesafeguard = flag.Bool(
	"disable_timesafeguard",
	false,
	"Disables checking whether the time is in sync with other nodes before joining the network (DANGEROUS!)")

const ElectionTimeout = 2 * time.Second

type timeResult struct {
	// Start is the point in local time before we started talking to the target.
	Start time.Time
	// End is the point in local time after we decoded the response.
	End time.Time
	// Result is the remote time of when the target replied.
	Result time.Time
}

func (t *timeResult) String() string {
	return fmt.Sprintf("Local: %v, Remote: %v ± %v", t.Start, t.Result, t.End.Sub(t.Start))
}

func (t timeResult) worstCaseDrift() time.Duration {
	// The worst-case drift is the difference between local time when
	// starting the measurement and remote time plus however long the
	// measurement itself took.
	drift := t.Result.Sub(t.Start)
	if drift < 0 {
		drift = -drift
	}
	drift += t.End.Sub(t.Start)
	return drift
}

func getServerTime(server, networkPassword string) (timeResult, util.ServerStatus, error) {
	start := time.Now()
	status, err := util.GetServerStatus(server, networkPassword)
	return timeResult{
		Start:  start,
		End:    time.Now(),
		Result: status.CurrentTime,
	}, status, err
}

// The first error talking to any of |servers| will be returned.
func collectTime(servers []string, networkPassword string) ([]timeResult, error) {
	var wg sync.WaitGroup
	results := make([]timeResult, len(servers))
	errChan := make(chan error, len(servers))
	for idx, server := range servers {
		wg.Add(1)
		go func(idx int, server string) {
			defer wg.Done()
			result, _, err := getServerTime(server, networkPassword)
			if err != nil {
				errChan <- err
				return
			}
			results[idx] = result
		}(idx, server)
	}
	wg.Wait()
	select {
	case err := <-errChan:
		for _ = range errChan {
		}
		return results, err
	default:
		return results, nil
	}
}

// timeInSync returns whether all remote times are synchronized to
// local time ± |ElectionTimeout|.
func timeInSync(results []timeResult) bool {
	for _, result := range results {
		if result.worstCaseDrift() >= ElectionTimeout {
			return false
		}
	}
	return true
}

func synchronizedWithNetwork(results []timeResult) error {
	log.Printf("Collected time measurements:\n")
	var nonZeroResults []timeResult
	for _, result := range results {
		if result.Result.IsZero() {
			log.Printf("  %s (ignoring)\n", result.String())
		} else {
			log.Printf("  %s\n", result.String())
			nonZeroResults = append(nonZeroResults, result)
		}
	}
	if timeInSync(nonZeroResults) {
		return nil
	}
	var errDetails []string
	for _, result := range nonZeroResults {
		if result.worstCaseDrift() >= ElectionTimeout {
			errDetails = append(errDetails, result.String())
		}
	}

	if *DisableTimesafeguard {
		log.Printf("Local time is too different from remote time, but timesafeguard is disabled.\nConflicting remote times: %s",
			strings.Join(errDetails, "\n"))
		return nil
	}

	return fmt.Errorf("Local time is too different from remote time, refusing to join network.\nConflicting remote times: %s",
		strings.Join(errDetails, "\n"))
}

// SynchronizedWithMasterAndNetwork returns an error if the time of either
// |join| or any other node in the network is too far off the local time.
func SynchronizedWithMasterAndNetwork(peerAddr, join, networkPassword string) error {
	result, status, err := getServerTime(join, networkPassword)
	if err != nil {
		log.Fatalf("Could not join %q: %v\n", join, err)
	}
	var peers []string
	for _, peer := range status.Peers {
		if peer != peerAddr && peer != join {
			peers = append(peers, peer)
		}
	}
	log.Printf("Collecting time of remaining nodes %v\n", peers)
	results, err := collectTime(peers, networkPassword)
	if err != nil {
		glog.Warningf("Could not collect time from all of %v: %v\n", peers, err)
	}
	results = append(results, result)
	return synchronizedWithNetwork(results)
}

// SynchronizedWithNetwork returns an error if the time of any of |peers| is
// too far off the local time.
func SynchronizedWithNetwork(peerAddr string, peers []string, networkPassword string) error {
	var collectPeers []string
	for _, peer := range peers {
		if peer != peerAddr {
			collectPeers = append(collectPeers, peer)
		}
	}
	log.Printf("Collecting time of remaining nodes %v\n", collectPeers)
	results, err := collectTime(collectPeers, networkPassword)
	if err != nil {
		glog.Warningf("Could not collect time from all of %v: %v\n", collectPeers, err)
	}
	return synchronizedWithNetwork(results)
}
<commit_msg>close(errChan) to fix a channel leak<commit_after>// Package timesafeguard collects the time of other nodes and ensures the
// remote times are not diverging too much from the local time. This is useful
// to ensure the following situation does not happen:
//
// 1. In a network of 3 nodes, node 1 goes down.
//
// 2. Node A comes back up, but with a 1-hour clock drift due to the hardware
// clock being in local time instead of UTC. ntpd refuses to correct the
// time because the drift is too big.
//
// 3. Node A re-joins the network.
//
// 4. At some point, node A becomes the leader. Message timestamps made a jump
// from e.g. 1432323893 to 1432327493, i.e. one hour into the future.
//
// 5. At some point, a different node becomes the leader. When trying to apply
// a message, the node panics because the message timestamp is not
// monotonically increasing.
//
// 6. The network needs to be frozen for an hour to be healthy again.
//
// timesafeguard ensures that the time is not off by more than
// |ElectionTimeout|, which presents the lower bound on how long an election
// takes:
//
// 1. In order for a candidate to win an election, the candidate needs to have
// the most recently committed entry in its log, otherwise it will not get a
// majority of votes.
//
// 2. The only way to get the most recently committed entry is to receive an
// appendEntries RPC, which resets the election timer. Hence, the earliest
// point at which any node can start an election that has any chance to be
// successful is after |ElectionTimeout| passed.
//
// Since during an election, no new messages are committed, a clock drift of
// |ElectionTimeout| is okay: the first message on the new leader will be
// committed late enough that it has a higher timestamp than the last message
// on the old leader.
package timesafeguard

import (
	"flag"
	"fmt"
	"log"
	"strings"
	"sync"
	"time"

	"github.com/robustirc/robustirc/util"
	"github.com/stapelberg/glog"
)

var DisableTimesafeguard = flag.Bool(
	"disable_timesafeguard",
	false,
	"Disables checking whether the time is in sync with other nodes before joining the network (DANGEROUS!)")

const ElectionTimeout = 2 * time.Second

type timeResult struct {
	// Start is the point in local time before we started talking to the target.
	Start time.Time
	// End is the point in local time after we decoded the response.
	End time.Time
	// Result is the remote time of when the target replied.
	Result time.Time
}

func (t *timeResult) String() string {
	return fmt.Sprintf("Local: %v, Remote: %v ± %v", t.Start, t.Result, t.End.Sub(t.Start))
}

func (t timeResult) worstCaseDrift() time.Duration {
	// The worst-case drift is the difference between local time when
	// starting the measurement and remote time plus however long the
	// measurement itself took.
	drift := t.Result.Sub(t.Start)
	if drift < 0 {
		drift = -drift
	}
	drift += t.End.Sub(t.Start)
	return drift
}

func getServerTime(server, networkPassword string) (timeResult, util.ServerStatus, error) {
	start := time.Now()
	status, err := util.GetServerStatus(server, networkPassword)
	return timeResult{
		Start:  start,
		End:    time.Now(),
		Result: status.CurrentTime,
	}, status, err
}

// The first error talking to any of |servers| will be returned.
func collectTime(servers []string, networkPassword string) ([]timeResult, error) {
	var wg sync.WaitGroup
	results := make([]timeResult, len(servers))
	errChan := make(chan error, len(servers))
	for idx, server := range servers {
		wg.Add(1)
		go func(idx int, server string) {
			defer wg.Done()
			result, _, err := getServerTime(server, networkPassword)
			if err != nil {
				errChan <- err
				return
			}
			results[idx] = result
		}(idx, server)
	}
	wg.Wait()
	select {
	case err := <-errChan:
		close(errChan)
		for _ = range errChan {
		}
		return results, err
	default:
		return results, nil
	}
}

// timeInSync returns whether all remote times are synchronized to
// local time ± |ElectionTimeout|.
func timeInSync(results []timeResult) bool {
	for _, result := range results {
		if result.worstCaseDrift() >= ElectionTimeout {
			return false
		}
	}
	return true
}

func synchronizedWithNetwork(results []timeResult) error {
	log.Printf("Collected time measurements:\n")
	var nonZeroResults []timeResult
	for _, result := range results {
		if result.Result.IsZero() {
			log.Printf("  %s (ignoring)\n", result.String())
		} else {
			log.Printf("  %s\n", result.String())
			nonZeroResults = append(nonZeroResults, result)
		}
	}
	if timeInSync(nonZeroResults) {
		return nil
	}
	var errDetails []string
	for _, result := range nonZeroResults {
		if result.worstCaseDrift() >= ElectionTimeout {
			errDetails = append(errDetails, result.String())
		}
	}

	if *DisableTimesafeguard {
		log.Printf("Local time is too different from remote time, but timesafeguard is disabled.\nConflicting remote times: %s",
			strings.Join(errDetails, "\n"))
		return nil
	}

	return fmt.Errorf("Local time is too different from remote time, refusing to join network.\nConflicting remote times: %s",
		strings.Join(errDetails, "\n"))
}

// SynchronizedWithMasterAndNetwork returns an error if the time of either
// |join| or any other node in the network is too far off the local time.
func SynchronizedWithMasterAndNetwork(peerAddr, join, networkPassword string) error {
	result, status, err := getServerTime(join, networkPassword)
	if err != nil {
		log.Fatalf("Could not join %q: %v\n", join, err)
	}
	var peers []string
	for _, peer := range status.Peers {
		if peer != peerAddr && peer != join {
			peers = append(peers, peer)
		}
	}
	log.Printf("Collecting time of remaining nodes %v\n", peers)
	results, err := collectTime(peers, networkPassword)
	if err != nil {
		glog.Warningf("Could not collect time from all of %v: %v\n", peers, err)
	}
	results = append(results, result)
	return synchronizedWithNetwork(results)
}

// SynchronizedWithNetwork returns an error if the time of any of |peers| is
// too far off the local time.
func SynchronizedWithNetwork(peerAddr string, peers []string, networkPassword string) error {
	var collectPeers []string
	for _, peer := range peers {
		if peer != peerAddr {
			collectPeers = append(collectPeers, peer)
		}
	}
	log.Printf("Collecting time of remaining nodes %v\n", collectPeers)
	results, err := collectTime(collectPeers, networkPassword)
	if err != nil {
		glog.Warningf("Could not collect time from all of %v: %v\n", collectPeers, err)
	}
	return synchronizedWithNetwork(results)
}
<|endoftext|>


<commit_before>// Copyright 2015/2016 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"sort"
	"strings"
	"sync"
	"text/template"

	"github.com/google/syzkaller/pkg/ast"
	"github.com/google/syzkaller/pkg/cmdprof"
	"github.com/google/syzkaller/pkg/compiler"
	"github.com/google/syzkaller/pkg/hash"
	"github.com/google/syzkaller/pkg/osutil"
	"github.com/google/syzkaller/pkg/serializer"
	"github.com/google/syzkaller/prog"
	"github.com/google/syzkaller/sys/targets"
)

type SyscallData struct {
	Name     string
	CallName string
	NR       int32
	NeedCall bool
	Attrs    []uint64
}

type ArchData struct {
	Revision   string
	ForkServer int
	Shmem      int
	GOARCH     string
	PageSize   uint64
	NumPages   uint64
	DataOffset uint64
	Calls      []SyscallData
}

type OSData struct {
	GOOS  string
	Archs []ArchData
}

type ExecutorData struct {
	OSes      []OSData
	CallAttrs []string
}

var srcDir = flag.String("src", "", "path to root of syzkaller source dir")
var outDir = flag.String("out", "", "path to out dir")

func main() {
	flag.Parse()
	defer cmdprof.Install()()

	var OSList []string
	for OS := range targets.List {
		OSList = append(OSList, OS)
	}
	sort.Strings(OSList)

	data := &ExecutorData{}
	for _, OS := range OSList {
		descriptions := ast.ParseGlob(filepath.Join(*srcDir, "sys", OS, "*.txt"), nil)
		if descriptions == nil {
			os.Exit(1)
		}
		osutil.MkdirAll(filepath.Join(*outDir, "sys", OS, "gen"))

		var archs []string
		for arch := range targets.List[OS] {
			archs = append(archs, arch)
		}
		sort.Strings(archs)

		type Job struct {
			Target      *targets.Target
			OK          bool
			Errors      []string
			Unsupported map[string]bool
			ArchData    ArchData
		}
		var jobs []*Job
		for _, arch := range archs {
			jobs = append(jobs, &Job{
				Target: targets.List[OS][arch],
			})
		}
		sort.Slice(jobs, func(i, j int) bool {
			return jobs[i].Target.Arch < jobs[j].Target.Arch
		})
		var wg sync.WaitGroup
		wg.Add(len(jobs))

		for _, job := range jobs {
			job := job
			go func() {
				defer wg.Done()
				eh := func(pos ast.Pos, msg string) {
					job.Errors = append(job.Errors, fmt.Sprintf("%v: %v\n", pos, msg))
				}
				consts := compiler.DeserializeConstsGlob(filepath.Join(*srcDir, "sys", OS, "*_"+job.Target.Arch+".const"), eh)
				if consts == nil {
					return
				}
				top := descriptions
				if OS == "linux" && job.Target.Arch == "arm" {
					// Hack: KVM is not supported on ARM anymore.
					// Note: syz-extract also ignores this file for arm.
					top = descriptions.Filter(func(n ast.Node) bool {
						pos, _, _ := n.Info()
						return !strings.HasSuffix(pos.File, "_kvm.txt")
					})
				}
				if OS == "test" {
					constInfo := compiler.ExtractConsts(top, job.Target, eh)
					compiler.FabricateSyscallConsts(job.Target, constInfo, consts)
				}
				prog := compiler.Compile(top, consts, job.Target, eh)
				if prog == nil {
					return
				}
				job.Unsupported = prog.Unsupported

				sysFile := filepath.Join(*outDir, "sys", OS, "gen", job.Target.Arch+".go")
				out := new(bytes.Buffer)
				generate(job.Target, prog, consts, out)
				rev := hash.String(out.Bytes())
				fmt.Fprintf(out, "const revision_%v = %q\n", job.Target.Arch, rev)
				writeSource(sysFile, out.Bytes())

				job.ArchData = generateExecutorSyscalls(job.Target, prog.Syscalls, rev)

				// Don't print warnings, they are printed in syz-check.
				job.Errors = nil
				job.OK = true
			}()
		}
		writeEmpty(OS)
		wg.Wait()

		var syscallArchs []ArchData
		unsupported := make(map[string]int)
		for _, job := range jobs {
			if !job.OK {
				fmt.Printf("compilation of %v/%v target failed:\n", job.Target.OS, job.Target.Arch)
				for _, msg := range job.Errors {
					fmt.Print(msg)
				}
				os.Exit(1)
			}
			syscallArchs = append(syscallArchs, job.ArchData)
			for u := range job.Unsupported {
				unsupported[u]++
			}
		}
		data.OSes = append(data.OSes, OSData{
			GOOS:  OS,
			Archs: syscallArchs,
		})

		for what, count := range unsupported {
			if count == len(jobs) {
				failf("%v is unsupported on all arches (typo?)", what)
			}
		}
	}

	attrs := reflect.TypeOf(prog.SyscallAttrs{})
	for i := 0; i < attrs.NumField(); i++ {
		data.CallAttrs = append(data.CallAttrs, prog.CppName(attrs.Field(i).Name))
	}

	writeExecutorSyscalls(data)
}

func generate(target *targets.Target, prg *compiler.Prog, consts map[string]uint64, out io.Writer) {
	tag := fmt.Sprintf("syz_target,syz_os_%v,syz_arch_%v", target.OS, target.Arch)
	if target.VMArch != "" {
		tag += fmt.Sprintf(" syz_target,syz_os_%v,syz_arch_%v", target.OS, target.VMArch)
	}
	fmt.Fprintf(out, "// AUTOGENERATED FILE\n")
	fmt.Fprintf(out, "// +build !codeanalysis\n")
	fmt.Fprintf(out, "// +build !syz_target %v\n\n", tag)
	fmt.Fprintf(out, "package gen\n\n")
	fmt.Fprintf(out, "import . \"github.com/google/syzkaller/prog\"\n")
	fmt.Fprintf(out, "import . \"github.com/google/syzkaller/sys/%v\"\n\n", target.OS)

	fmt.Fprintf(out, "func init() {\n")
	fmt.Fprintf(out, "\tRegisterTarget(&Target{"+
		"OS: %q, Arch: %q, Revision: revision_%v, PtrSize: %v, "+
		"PageSize: %v, NumPages: %v, DataOffset: %v, LittleEndian: %v, Syscalls: syscalls_%v, "+
		"Resources: resources_%v, Consts: consts_%v}, "+
		"types_%v, InitTarget)\n}\n\n",
		target.OS, target.Arch, target.Arch, target.PtrSize,
		target.PageSize, target.NumPages, target.DataOffset,
		target.LittleEndian, target.Arch, target.Arch, target.Arch, target.Arch)

	fmt.Fprintf(out, "var resources_%v = ", target.Arch)
	serializer.Write(out, prg.Resources)
	fmt.Fprintf(out, "\n\n")

	fmt.Fprintf(out, "var syscalls_%v = ", target.Arch)
	serializer.Write(out, prg.Syscalls)
	fmt.Fprintf(out, "\n\n")

	fmt.Fprintf(out, "var types_%v = ", target.Arch)
	serializer.Write(out, prg.Types)
	fmt.Fprintf(out, "\n\n")

	constArr := make([]prog.ConstValue, 0, len(consts))
	for name, val := range consts {
		constArr = append(constArr, prog.ConstValue{Name: name, Value: val})
	}
	sort.Slice(constArr, func(i, j int) bool {
		return constArr[i].Name < constArr[j].Name
	})
	fmt.Fprintf(out, "var consts_%v = ", target.Arch)
	serializer.Write(out, constArr)
	fmt.Fprintf(out, "\n\n")
}

func writeEmpty(OS string) {
	const data = `// AUTOGENERATED FILE
// This file is needed if OS is completely excluded by build tags.
package gen
`
	writeSource(filepath.Join(*outDir, "sys", OS, "gen", "empty.go"), []byte(data))
}

func generateExecutorSyscalls(target *targets.Target, syscalls []*prog.Syscall, rev string) ArchData {
	data := ArchData{
		Revision:   rev,
		GOARCH:     target.Arch,
		PageSize:   target.PageSize,
		NumPages:   target.NumPages,
		DataOffset: target.DataOffset,
	}
	if target.ExecutorUsesForkServer {
		data.ForkServer = 1
	}
	if target.ExecutorUsesShmem {
		data.Shmem = 1
	}
	for _, c := range syscalls {
		var attrVals []uint64
		attrs := reflect.ValueOf(c.Attrs)
		last := -1
		for i := 0; i < attrs.NumField(); i++ {
			attr := attrs.Field(i)
			val := uint64(0)
			switch attr.Type().Kind() {
			case reflect.Bool:
				if attr.Bool() {
					val = 1
				}
			case reflect.Uint64:
				val = attr.Uint()
			default:
				panic("unsupported syscall attribute type")
			}
			attrVals = append(attrVals, val)
			if val != 0 {
				last = i
			}
		}
		data.Calls = append(data.Calls, newSyscallData(target, c, attrVals[:last+1]))
	}
	sort.Slice(data.Calls, func(i, j int) bool {
		return data.Calls[i].Name < data.Calls[j].Name
	})
	return data
}

func newSyscallData(target *targets.Target, sc *prog.Syscall, attrs []uint64) SyscallData {
	callName, patchCallName := target.SyscallTrampolines[sc.Name]
	if !patchCallName {
		callName = sc.CallName
	}
	return SyscallData{
		Name:     sc.Name,
		CallName: callName,
		NR:       int32(sc.NR),
		NeedCall: (!target.SyscallNumbers || strings.HasPrefix(sc.CallName, "syz_") || patchCallName) && !sc.Attrs.Disabled,
		Attrs:    attrs,
	}
}

func writeExecutorSyscalls(data *ExecutorData) {
	osutil.MkdirAll(filepath.Join(*outDir, "executor"))
	sort.Slice(data.OSes, func(i, j int) bool {
		return data.OSes[i].GOOS < data.OSes[j].GOOS
	})
	buf := new(bytes.Buffer)
	if err := defsTempl.Execute(buf, data); err != nil {
		failf("failed to execute defs template: %v", err)
	}
	writeFile(filepath.Join(*outDir, "executor", "defs.h"), buf.Bytes())
	buf.Reset()
	if err := syscallsTempl.Execute(buf, data); err != nil {
		failf("failed to execute syscalls template: %v", err)
	}
	writeFile(filepath.Join(*outDir, "executor", "syscalls.h"), buf.Bytes())
}

func writeSource(file string, data []byte) {
	if oldSrc, err := ioutil.ReadFile(file); err == nil && bytes.Equal(data, oldSrc) {
		return
	}
	writeFile(file, data)
}

func writeFile(file string, data []byte) {
	outf, err := os.Create(file)
	if err != nil {
		failf("failed to create output file: %v", err)
	}
	defer outf.Close()
	outf.Write(data)
}

func failf(msg string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, msg+"\n", args...)
	os.Exit(1)
}

var defsTempl = template.Must(template.New("").Parse(`// AUTOGENERATED FILE

struct call_attrs_t { {{range $attr := $.CallAttrs}}
	uint64_t {{$attr}};{{end}}
};
{{range $os := $.OSes}}
#if GOOS_{{$os.GOOS}}
#define GOOS "{{$os.GOOS}}"
{{range $arch := $os.Archs}}
#if GOARCH_{{$arch.GOARCH}}
#define GOARCH "{{.GOARCH}}"
#define SYZ_REVISION "{{.Revision}}"
#define SYZ_EXECUTOR_USES_FORK_SERVER {{.ForkServer}}
#define SYZ_EXECUTOR_USES_SHMEM {{.Shmem}}
#define SYZ_PAGE_SIZE {{.PageSize}}
#define SYZ_NUM_PAGES {{.NumPages}}
#define SYZ_DATA_OFFSET {{.DataOffset}}
#endif
{{end}}
#endif
{{end}}
`))

// nolint: lll
var syscallsTempl = template.Must(template.New("").Parse(`// AUTOGENERATED FILE
// clang-format off
{{range $os := $.OSes}}
#if GOOS_{{$os.GOOS}}
{{range $arch := $os.Archs}}
#if GOARCH_{{$arch.GOARCH}}
const call_t syscalls[] = {
{{range $c := $arch.Calls}}    {"{{$c.Name}}", {{$c.NR}}{{if or $c.Attrs $c.NeedCall}}, { {{- range $attr := $c.Attrs}}{{$attr}}, {{end}}}{{end}}{{if $c.NeedCall}}, (syscall_t){{$c.CallName}}{{end}}},
{{end}}};
#endif
{{end}}
#endif
{{end}}
`))
<commit_msg>sys/syz-sysgen: ignore *_kvm.txt on linux/riscv64<commit_after>// Copyright 2015/2016 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"sort"
	"strings"
	"sync"
	"text/template"

	"github.com/google/syzkaller/pkg/ast"
	"github.com/google/syzkaller/pkg/cmdprof"
	"github.com/google/syzkaller/pkg/compiler"
	"github.com/google/syzkaller/pkg/hash"
	"github.com/google/syzkaller/pkg/osutil"
	"github.com/google/syzkaller/pkg/serializer"
	"github.com/google/syzkaller/prog"
	"github.com/google/syzkaller/sys/targets"
)

type SyscallData struct {
	Name     string
	CallName string
	NR       int32
	NeedCall bool
	Attrs    []uint64
}

type ArchData struct {
	Revision   string
	ForkServer int
	Shmem      int
	GOARCH     string
	PageSize   uint64
	NumPages   uint64
	DataOffset uint64
	Calls      []SyscallData
}

type OSData struct {
	GOOS  string
	Archs []ArchData
}

type ExecutorData struct {
	OSes      []OSData
	CallAttrs []string
}

var srcDir = flag.String("src", "", "path to root of syzkaller source dir")
var outDir = flag.String("out", "", "path to out dir")

func main() {
	flag.Parse()
	defer cmdprof.Install()()

	var OSList []string
	for OS := range targets.List {
		OSList = append(OSList, OS)
	}
	sort.Strings(OSList)

	data := &ExecutorData{}
	for _, OS := range OSList {
		descriptions := ast.ParseGlob(filepath.Join(*srcDir, "sys", OS, "*.txt"), nil)
		if descriptions == nil {
			os.Exit(1)
		}
		osutil.MkdirAll(filepath.Join(*outDir, "sys", OS, "gen"))

		var archs []string
		for arch := range targets.List[OS] {
			archs = append(archs, arch)
		}
		sort.Strings(archs)

		type Job struct {
			Target      *targets.Target
			OK          bool
			Errors      []string
			Unsupported map[string]bool
			ArchData    ArchData
		}
		var jobs []*Job
		for _, arch := range archs {
			jobs = append(jobs, &Job{
				Target: targets.List[OS][arch],
			})
		}
		sort.Slice(jobs, func(i, j int) bool {
			return jobs[i].Target.Arch < jobs[j].Target.Arch
		})
		var wg sync.WaitGroup
		wg.Add(len(jobs))

		for _, job := range jobs {
			job := job
			go func() {
				defer wg.Done()
				eh := func(pos ast.Pos, msg string) {
					job.Errors = append(job.Errors, fmt.Sprintf("%v: %v\n", pos, msg))
				}
				consts := compiler.DeserializeConstsGlob(filepath.Join(*srcDir, "sys", OS, "*_"+job.Target.Arch+".const"), eh)
				if consts == nil {
					return
				}
				top := descriptions
				if OS == "linux" && (job.Target.Arch == "arm" || job.Target.Arch == "riscv64") {
					// Hack: KVM is not supported on ARM anymore. On riscv64 it
					// is not supported yet but might be in the future.
					// Note: syz-extract also ignores this file for arm and
					// riscv64.
					top = descriptions.Filter(func(n ast.Node) bool {
						pos, _, _ := n.Info()
						return !strings.HasSuffix(pos.File, "_kvm.txt")
					})
				}
				if OS == "test" {
					constInfo := compiler.ExtractConsts(top, job.Target, eh)
					compiler.FabricateSyscallConsts(job.Target, constInfo, consts)
				}
				prog := compiler.Compile(top, consts, job.Target, eh)
				if prog == nil {
					return
				}
				job.Unsupported = prog.Unsupported

				sysFile := filepath.Join(*outDir, "sys", OS, "gen", job.Target.Arch+".go")
				out := new(bytes.Buffer)
				generate(job.Target, prog, consts, out)
				rev := hash.String(out.Bytes())
				fmt.Fprintf(out, "const revision_%v = %q\n", job.Target.Arch, rev)
				writeSource(sysFile, out.Bytes())

				job.ArchData = generateExecutorSyscalls(job.Target, prog.Syscalls, rev)

				// Don't print warnings, they are printed in syz-check.
				job.Errors = nil
				job.OK = true
			}()
		}
		writeEmpty(OS)
		wg.Wait()

		var syscallArchs []ArchData
		unsupported := make(map[string]int)
		for _, job := range jobs {
			if !job.OK {
				fmt.Printf("compilation of %v/%v target failed:\n", job.Target.OS, job.Target.Arch)
				for _, msg := range job.Errors {
					fmt.Print(msg)
				}
				os.Exit(1)
			}
			syscallArchs = append(syscallArchs, job.ArchData)
			for u := range job.Unsupported {
				unsupported[u]++
			}
		}
		data.OSes = append(data.OSes, OSData{
			GOOS:  OS,
			Archs: syscallArchs,
		})

		for what, count := range unsupported {
			if count == len(jobs) {
				failf("%v is unsupported on all arches (typo?)", what)
			}
		}
	}

	attrs := reflect.TypeOf(prog.SyscallAttrs{})
	for i := 0; i < attrs.NumField(); i++ {
		data.CallAttrs = append(data.CallAttrs, prog.CppName(attrs.Field(i).Name))
	}

	writeExecutorSyscalls(data)
}

func generate(target *targets.Target, prg *compiler.Prog, consts map[string]uint64, out io.Writer) {
	tag := fmt.Sprintf("syz_target,syz_os_%v,syz_arch_%v", target.OS, target.Arch)
	if target.VMArch != "" {
		tag += fmt.Sprintf(" syz_target,syz_os_%v,syz_arch_%v", target.OS, target.VMArch)
	}
	fmt.Fprintf(out, "// AUTOGENERATED FILE\n")
	fmt.Fprintf(out, "// +build !codeanalysis\n")
	fmt.Fprintf(out, "// +build !syz_target %v\n\n", tag)
	fmt.Fprintf(out, "package gen\n\n")
	fmt.Fprintf(out, "import . \"github.com/google/syzkaller/prog\"\n")
	fmt.Fprintf(out, "import . \"github.com/google/syzkaller/sys/%v\"\n\n", target.OS)

	fmt.Fprintf(out, "func init() {\n")
	fmt.Fprintf(out, "\tRegisterTarget(&Target{"+
		"OS: %q, Arch: %q, Revision: revision_%v, PtrSize: %v, "+
		"PageSize: %v, NumPages: %v, DataOffset: %v, LittleEndian: %v, Syscalls: syscalls_%v, "+
		"Resources: resources_%v, Consts: consts_%v}, "+
		"types_%v, InitTarget)\n}\n\n",
		target.OS, target.Arch, target.Arch, target.PtrSize,
		target.PageSize, target.NumPages, target.DataOffset,
		target.LittleEndian, target.Arch, target.Arch, target.Arch, target.Arch)

	fmt.Fprintf(out, "var resources_%v = ", target.Arch)
	serializer.Write(out, prg.Resources)
	fmt.Fprintf(out, "\n\n")

	fmt.Fprintf(out, "var syscalls_%v = ", target.Arch)
	serializer.Write(out, prg.Syscalls)
	fmt.Fprintf(out, "\n\n")

	fmt.Fprintf(out, "var types_%v = ", target.Arch)
	serializer.Write(out, prg.Types)
	fmt.Fprintf(out, "\n\n")

	constArr := make([]prog.ConstValue, 0, len(consts))
	for name, val := range consts {
		constArr = append(constArr, prog.ConstValue{Name: name, Value: val})
	}
	sort.Slice(constArr, func(i, j int) bool {
		return constArr[i].Name < constArr[j].Name
	})
	fmt.Fprintf(out, "var consts_%v = ", target.Arch)
	serializer.Write(out, constArr)
	fmt.Fprintf(out, "\n\n")
}

func writeEmpty(OS string) {
	const data = `// AUTOGENERATED FILE
// This file is needed if OS is completely excluded by build tags.
package gen
`
	writeSource(filepath.Join(*outDir, "sys", OS, "gen", "empty.go"), []byte(data))
}

func generateExecutorSyscalls(target *targets.Target, syscalls []*prog.Syscall, rev string) ArchData {
	data := ArchData{
		Revision:   rev,
		GOARCH:     target.Arch,
		PageSize:   target.PageSize,
		NumPages:   target.NumPages,
		DataOffset: target.DataOffset,
	}
	if target.ExecutorUsesForkServer {
		data.ForkServer = 1
	}
	if target.ExecutorUsesShmem {
		data.Shmem = 1
	}
	for _, c := range syscalls {
		var attrVals []uint64
		attrs := reflect.ValueOf(c.Attrs)
		last := -1
		for i := 0; i < attrs.NumField(); i++ {
			attr := attrs.Field(i)
			val := uint64(0)
			switch attr.Type().Kind() {
			case reflect.Bool:
				if attr.Bool() {
					val = 1
				}
			case reflect.Uint64:
				val = attr.Uint()
			default:
				panic("unsupported syscall attribute type")
			}
			attrVals = append(attrVals, val)
			if val != 0 {
				last = i
			}
		}
		data.Calls = append(data.Calls, newSyscallData(target, c, attrVals[:last+1]))
	}
	sort.Slice(data.Calls, func(i, j int) bool {
		return data.Calls[i].Name < data.Calls[j].Name
	})
	return data
}

func newSyscallData(target *targets.Target, sc *prog.Syscall, attrs []uint64) SyscallData {
	callName, patchCallName := target.SyscallTrampolines[sc.Name]
	if !patchCallName {
		callName = sc.CallName
	}
	return SyscallData{
		Name:     sc.Name,
		CallName: callName,
		NR:       int32(sc.NR),
		NeedCall: (!target.SyscallNumbers || strings.HasPrefix(sc.CallName, "syz_") || patchCallName) && !sc.Attrs.Disabled,
		Attrs:    attrs,
	}
}

func writeExecutorSyscalls(data *ExecutorData) {
	osutil.MkdirAll(filepath.Join(*outDir, "executor"))
	sort.Slice(data.OSes, func(i, j int) bool {
		return data.OSes[i].GOOS < data.OSes[j].GOOS
	})
	buf := new(bytes.Buffer)
	if err := defsTempl.Execute(buf, data); err != nil {
		failf("failed to execute defs template: %v", err)
	}
	writeFile(filepath.Join(*outDir, "executor", "defs.h"), buf.Bytes())
	buf.Reset()
	if err := syscallsTempl.Execute(buf, data); err != nil {
		failf("failed to execute syscalls template: %v", err)
	}
	writeFile(filepath.Join(*outDir, "executor", "syscalls.h"), buf.Bytes())
}

func writeSource(file string, data []byte) {
	if oldSrc, err := ioutil.ReadFile(file); err == nil && bytes.Equal(data, oldSrc) {
		return
	}
	writeFile(file, data)
}

func writeFile(file string, data []byte) {
	outf, err := os.Create(file)
	if err != nil {
		failf("failed to create output file: %v", err)
	}
	defer outf.Close()
	outf.Write(data)
}

func failf(msg string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, msg+"\n", args...)
	os.Exit(1)
}

var defsTempl = template.Must(template.New("").Parse(`// AUTOGENERATED FILE

struct call_attrs_t { {{range $attr := $.CallAttrs}}
	uint64_t {{$attr}};{{end}}
};
{{range $os := $.OSes}}
#if GOOS_{{$os.GOOS}}
#define GOOS "{{$os.GOOS}}"
{{range $arch := $os.Archs}}
#if GOARCH_{{$arch.GOARCH}}
#define GOARCH "{{.GOARCH}}"
#define SYZ_REVISION "{{.Revision}}"
#define SYZ_EXECUTOR_USES_FORK_SERVER {{.ForkServer}}
#define SYZ_EXECUTOR_USES_SHMEM {{.Shmem}}
#define SYZ_PAGE_SIZE {{.PageSize}}
#define SYZ_NUM_PAGES {{.NumPages}}
#define SYZ_DATA_OFFSET {{.DataOffset}}
#endif
{{end}}
#endif
{{end}}
`))

// nolint: lll
var syscallsTempl = template.Must(template.New("").Parse(`// AUTOGENERATED FILE
// clang-format off
{{range $os := $.OSes}}
#if GOOS_{{$os.GOOS}}
{{range $arch := $os.Archs}}
#if GOARCH_{{$arch.GOARCH}}
const call_t syscalls[] = {
{{range $c := $arch.Calls}}    {"{{$c.Name}}", {{$c.NR}}{{if or $c.Attrs $c.NeedCall}}, { {{- range $attr := $c.Attrs}}{{$attr}}, {{end}}}{{end}}{{if $c.NeedCall}}, (syscall_t){{$c.CallName}}{{end}}},
{{end}}};
#endif
{{end}}
#endif
{{end}}
`))
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package lease

import "io"

// A sentinel error used when a lease has been revoked. See notes on particular
// methods.
type RevokedError struct {
}

func (re *RevokedError) Error() string {
	return "Lease revoked"
}

// A read-only wrapper around a file that may be revoked, when e.g. there is
// temporary disk space pressure. A read lease may also be upgraded to a write
// lease, if it is still valid.
//
// All methods are safe for concurrent access.
type ReadLease interface {
	// Reads for an expired lease will return an error of type *RevokedError.
	io.ReaderAt

	// Attempt to upgrade the lease to a read/write lease, returning an error of
	// type *RevokedError if the lease has been revoked. After upgrading, it is
	// as if the lease has been revoked.
	Upgrade() (rwl ReadWriteLease, err error)

	// Cause the lease to be revoked and any associated resources to be cleaned
	// up, if it has not already been revoked.
	Revoke()
}

// A read-write wrapper around a file. Unlike a read lease, this cannot be
// revoked.
//
// All methods are safe for concurrent access.
type ReadWriteLease interface {
	// Methods with semantics matching *os.File.
	io.ReaderAt
	io.WriterAt
	Seek(offset int64, whence int) (ret int64, err error)
	Truncate(size int64) error

	// Downgrade to a read lease, releasing any resources pinned by this lease to
	// the pool that may be revoked, as with any read lease. After downgrading,
	// this lease must not be used again.
	Downgrade() (rl ReadLease)
}
<commit_msg>Upgrade need not return an error.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package lease

import "io"

// A sentinel error used when a lease has been revoked. See notes on particular
// methods.
type RevokedError struct {
}

func (re *RevokedError) Error() string {
	return "Lease revoked"
}

// A read-only wrapper around a file that may be revoked, when e.g. there is
// temporary disk space pressure. A read lease may also be upgraded to a write
// lease, if it is still valid.
//
// All methods are safe for concurrent access.
type ReadLease interface {
	// Reads for an expired lease will return an error of type *RevokedError.
	io.ReaderAt

	// Attempt to upgrade the lease to a read/write lease, returning nil if the
	// lease has been revoked. After upgrading, it is as if the lease has been
	// revoked.
	Upgrade() (rwl ReadWriteLease)

	// Cause the lease to be revoked and any associated resources to be cleaned
	// up, if it has not already been revoked.
	Revoke()
}

// A read-write wrapper around a file. Unlike a read lease, this cannot be
// revoked.
//
// All methods are safe for concurrent access.
type ReadWriteLease interface {
	// Methods with semantics matching *os.File.
	io.ReaderAt
	io.WriterAt
	Seek(offset int64, whence int) (ret int64, err error)
	Truncate(size int64) error

	// Downgrade to a read lease, releasing any resources pinned by this lease to
	// the pool that may be revoked, as with any read lease. After downgrading,
	// this lease must not be used again.
	Downgrade() (rl ReadLease)
}
<|endoftext|>


<commit_before>package golog

import (
	"fmt"
)

const (
	INFO = iota
	WARNING
	ERROR
	FATAL
)

var levelStrings []string = []string{"I", "W", "E", "F"}

type LevelLogger struct {
	// TODO Can we get away with just a Logger?
	FailLogger
}

func (l *LevelLogger) logCommon(level int, closure func() string) {
	// TODO Add prefix, timestamp, etc.
	l.Logc(level, closure)
}

func (l *LevelLogger) Info(vals ...interface{}) {
	l.logCommon(INFO, func() string { return fmt.Sprint(vals...) })
}

func (l *LevelLogger) Infof(f string, args ...interface{}) {
	l.logCommon(INFO, func() string { return fmt.Sprintf(f, args...) })
}

func (l *LevelLogger) Infoc(closure func() string) {
	l.logCommon(INFO, closure)
}

func (l *LevelLogger) Warning(vals ...interface{}) {
	l.logCommon(WARNING, func() string { return fmt.Sprint(vals...) })
}

func (l *LevelLogger) Warningf(f string, args ...interface{}) {
	l.logCommon(WARNING, func() string { return fmt.Sprintf(f, args...) })
}

func (l *LevelLogger) Warningc(closure func() string) {
	l.logCommon(WARNING, closure)
}

func (l *LevelLogger) Error(vals ...interface{}) {
	l.logCommon(ERROR, func() string { return fmt.Sprint(vals...) })
}

func (l *LevelLogger) Errorf(f string, args ...interface{}) {
	l.logCommon(ERROR, func() string { return fmt.Sprintf(f, args...) })
}

func (l *LevelLogger) Errorc(closure func() string) {
	l.logCommon(ERROR, closure)
}

func (l *LevelLogger) Fatal(vals ...interface{}) {
	l.logCommon(FATAL, func() string { return fmt.Sprint(vals...) })
	l.FailNow()
}

func (l *LevelLogger) Fatalf(f string, args ...interface{}) {
	l.logCommon(FATAL, func() string { return fmt.Sprintf(f, args...) })
	l.FailNow()
}

func (l *LevelLogger) Fatalc(closure func() string) {
	l.logCommon(FATAL, closure)
	l.FailNow()
}
<commit_msg>Add metatdata to closure before logging<commit_after>package golog

import (
	"bytes"
	"fmt"
	"time"
)

const (
	INFO = iota
	WARNING
	ERROR
	FATAL
)

var levelStrings []string = []string{"I", "W", "E", "F"}

type LevelLogger struct {
	// TODO Can we get away with just a Logger?
	FailLogger
}

// Formats the message with metadata. The format is:
// LEVEL HH:MM:SS:NANOSC LOC] MESSAGE
func makeLogClosure(level int, ns int64, location string, msg func() string) func() string {
	return func() string {
		var buf bytes.Buffer
		buf.WriteString(levelStrings[level])
		t := time.NanosecondsToLocalTime(ns)
		buf.WriteString(t.Format(" 15:04:05.000000 "))
		// TODO Write file and line?
		buf.WriteString(location)
		buf.WriteString("] ")
		buf.WriteString(msg())
		return buf.String()
	}
}

func (l *LevelLogger) logCommon(level int, closure func() string) {
	ns := time.Nanoseconds()
	// TODO Get location?
	l.Logc(level, makeLogClosure(level, ns, "", closure))
}

func (l *LevelLogger) Info(vals ...interface{}) {
	l.logCommon(INFO, func() string { return fmt.Sprint(vals...) })
}

func (l *LevelLogger) Infof(f string, args ...interface{}) {
	l.logCommon(INFO, func() string { return fmt.Sprintf(f, args...) })
}

func (l *LevelLogger) Infoc(closure func() string) {
	l.logCommon(INFO, closure)
}

func (l *LevelLogger) Warning(vals ...interface{}) {
	l.logCommon(WARNING, func() string { return fmt.Sprint(vals...) })
}

func (l *LevelLogger) Warningf(f string, args ...interface{}) {
	l.logCommon(WARNING, func() string { return fmt.Sprintf(f, args...) })
}

func (l *LevelLogger) Warningc(closure func() string) {
	l.logCommon(WARNING, closure)
}

func (l *LevelLogger) Error(vals ...interface{}) {
	l.logCommon(ERROR, func() string { return fmt.Sprint(vals...) })
}

func (l *LevelLogger) Errorf(f string, args ...interface{}) {
	l.logCommon(ERROR, func() string { return fmt.Sprintf(f, args...) })
}

func (l *LevelLogger) Errorc(closure func() string) {
	l.logCommon(ERROR, closure)
}

func (l *LevelLogger) Fatal(vals ...interface{}) {
	l.logCommon(FATAL, func() string { return fmt.Sprint(vals...) })
	l.FailNow()
}

func (l *LevelLogger) Fatalf(f string, args ...interface{}) {
	l.logCommon(FATAL, func() string { return fmt.Sprintf(f, args...) })
	l.FailNow()
}

func (l *LevelLogger) Fatalc(closure func() string) {
	l.logCommon(FATAL, closure)
	l.FailNow()
}
<|endoftext|>


<commit_before>package golog

import (
	"fmt"
)

type LevelLogger struct {
	Logger
}

func (l *LevelLogger) logCommon(level int, closure func() string) {
	// TODO: Add prefix, timestamp, etc.
	l.Logc(level, closure)
}

func (l *LevelLogger) Info(vals ...interface{}) {
	l.logCommon(INFO, func() string { return fmt.Sprint(vals...) })
}

func (l *LevelLogger) Infof(f string, args ...interface{}) {
	l.logCommon(INFO, func() string { return fmt.Sprintf(f, args...) })
}

func (l *LevelLogger) Infoc(closure func() string) {
	l.logCommon(INFO, closure)
}
<commit_msg>LevelLogger uses FailLogger<commit_after>package golog

import (
	"fmt"
)

type LevelLogger struct {
	// TODO Can we get away with just a Logger?
	FailLogger
}

func (l *LevelLogger) logCommon(level int, closure func() string) {
	// TODO Add prefix, timestamp, etc.
	l.Logc(level, closure)
}

func (l *LevelLogger) Info(vals ...interface{}) {
	l.logCommon(INFO, func() string { return fmt.Sprint(vals...) })
}

func (l *LevelLogger) Infof(f string, args ...interface{}) {
	l.logCommon(INFO, func() string { return fmt.Sprintf(f, args...) })
}

func (l *LevelLogger) Infoc(closure func() string) {
	l.logCommon(INFO, closure)
}
<|endoftext|>


<commit_before>package arm

import (
	"fmt"
	sysnum "github.com/lunixbochs/ghostrace/ghost/sys/num"
	uc "github.com/unicorn-engine/unicorn/bindings/go/unicorn"
	"os"

	"github.com/lunixbochs/usercorn/go/kernel/common"
	"github.com/lunixbochs/usercorn/go/kernel/linux"
	"github.com/lunixbochs/usercorn/go/models"
)

var LinuxRegs = []int{uc.ARM_REG_R0, uc.ARM_REG_R1, uc.ARM_REG_R2, uc.ARM_REG_R3, uc.ARM_REG_R4, uc.ARM_REG_R5, uc.ARM_REG_R6}

type ArmLinuxKernel struct {
	*linux.LinuxKernel
}

func (k *ArmLinuxKernel) SetTls(addr uint64) {}

func LinuxKernels(u models.Usercorn) []interface{} {
	kernel := &ArmLinuxKernel{linux.DefaultKernel()}
	kernel.UsercornInit(kernel, u)
	return []interface{}{kernel}
}

func LinuxInit(u models.Usercorn, args, env []string) error {
	if err := enterUsermode(u); err != nil {
		return err
	}
	u.MemMap(0xffff0000, 0x10000)
	_, err := u.HookAdd(uc.HOOK_CODE, func(_ uc.Unicorn, addr uint64, size uint32) {
		fmt.Fprintf(os.Stderr, "kernel trap (unimplemented): 0x%x\n", addr)
		u.Stop()
	}, 0xffff0000, 0xffffffff)
	if err != nil {
		return err
	}
	return linux.StackInit(u, args, env)
}

func LinuxSyscall(u models.Usercorn) {
	// TODO: handle errors or something
	num, _ := u.RegRead(uc.ARM_REG_R7)
	// TODO: EABI has a different syscall base (OABI is 0x900000)
	// TODO: does the generator handle this? it needs to.
	if num > 0x900000 {
		num -= 0x900000
	}
	name, _ := sysnum.Linux_arm[int(num)]
	ret, _ := u.Syscall(int(num), name, common.RegArgs(u, LinuxRegs))
	u.RegWrite(uc.ARM_REG_R0, ret)
}

func LinuxInterrupt(u models.Usercorn, intno uint32) {
	if intno == 2 {
		LinuxSyscall(u)
		return
	}
	panic(fmt.Sprintf("unhandled ARM interrupt: %d", intno))
}

func init() {
	Arch.RegisterOS(&models.OS{
		Name:      "linux",
		Kernels:   LinuxKernels,
		Init:      LinuxInit,
		Interrupt: LinuxInterrupt,
	})
}
<commit_msg>kinda support arm tls trap<commit_after>package arm

import (
	"fmt"
	sysnum "github.com/lunixbochs/ghostrace/ghost/sys/num"
	uc "github.com/unicorn-engine/unicorn/bindings/go/unicorn"

	"github.com/lunixbochs/usercorn/go/kernel/common"
	"github.com/lunixbochs/usercorn/go/kernel/linux"
	"github.com/lunixbochs/usercorn/go/models"
)

var LinuxRegs = []int{uc.ARM_REG_R0, uc.ARM_REG_R1, uc.ARM_REG_R2, uc.ARM_REG_R3, uc.ARM_REG_R4, uc.ARM_REG_R5, uc.ARM_REG_R6}

type ArmLinuxKernel struct {
	*linux.LinuxKernel
	tls uint64
}

func (k *ArmLinuxKernel) SetTls(addr uint64) {
	k.tls = addr
}

func setupTraps(u models.Usercorn, kernel *ArmLinuxKernel) error {
	// handle arm kernel traps
	// https://www.kernel.org/doc/Documentation/arm/kernel_user_helpers.txt
	if err := u.MemMap(0xffff0000, 0x10000); err != nil {
		return err
	}
	for addr := 0; addr < 0x10000; addr += 4 {
		// write "bx lr" to all kernel trap addresses so they will return
		bxlr := []byte{0x1e, 0xff, 0x2f, 0xe1}
		if err := u.MemWrite(uint64(0xffff0000+addr), bxlr); err != nil {
			return err
		}
	}
	_, err := u.HookAdd(uc.HOOK_CODE, func(_ uc.Unicorn, addr uint64, size uint32) {
		switch addr {
		case 0xffff0fe0:
			// __kuser_get_tls
			u.RegWrite(uc.ARM_REG_R0, kernel.tls)
		default:
			panic(fmt.Sprintf("unsupported kernel trap: 0x%x\n", addr))
		}
	}, 0xffff0000, 0xffffffff)
	if err != nil {
		return err
	}
	return nil
}

func LinuxKernels(u models.Usercorn) []interface{} {
	kernel := &ArmLinuxKernel{LinuxKernel: linux.DefaultKernel()}
	kernel.UsercornInit(kernel, u)
	// TODO: LinuxInit needs to have a copy of the kernel
	// honestly init should be part of the kernel?
	if err := setupTraps(u, kernel); err != nil {
		panic(err)
	}
	return []interface{}{kernel}
}

func LinuxInit(u models.Usercorn, args, env []string) error {
	if err := enterUsermode(u); err != nil {
		return err
	}
	return linux.StackInit(u, args, env)
}

func LinuxSyscall(u models.Usercorn) {
	// TODO: handle errors or something
	num, _ := u.RegRead(uc.ARM_REG_R7)
	// TODO: EABI has a different syscall base (OABI is 0x900000)
	// TODO: does the generator handle this? it needs to.
	if num > 0x900000 {
		num -= 0x900000
	}
	name, _ := sysnum.Linux_arm[int(num)]
	ret, _ := u.Syscall(int(num), name, common.RegArgs(u, LinuxRegs))
	u.RegWrite(uc.ARM_REG_R0, ret)
}

func LinuxInterrupt(u models.Usercorn, intno uint32) {
	if intno == 2 {
		LinuxSyscall(u)
		return
	}
	panic(fmt.Sprintf("unhandled ARM interrupt: %d", intno))
}

func init() {
	Arch.RegisterOS(&models.OS{
		Name:      "linux",
		Kernels:   LinuxKernels,
		Init:      LinuxInit,
		Interrupt: LinuxInterrupt,
	})
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mysql

// GTIDSet represents the set of transactions received or applied by a server.
// In some flavors, a single GTID is enough to specify the set of all
// transactions that came before it, but in others a more complex structure is
// required.
//
// GTIDSet is wrapped by replication.Position, which is a concrete struct.
// When sending a GTIDSet over RPCs, encode/decode it as a string.
// Most code outside of this package should use replication.Position rather
// than GTIDSet.
type GTIDSet interface {
	// String returns the canonical printed form of the set as expected by a
	// particular flavor of MySQL.
	String() string

	// Flavor returns the key under which the corresponding parser function is
	// registered in the transactionSetParsers map.
	Flavor() string

	// ContainsGTID returns true if the set contains the specified transaction.
	ContainsGTID(GTID) bool

	// Contains returns true if the set is a superset of another set. All implementations should return false if
	// other GTIDSet is nil.
	Contains(GTIDSet) bool

	// Equal returns true if the set is equal to another set.
	Equal(GTIDSet) bool

	// AddGTID returns a new GTIDSet that is expanded to contain the given GTID.
	AddGTID(GTID) GTIDSet

	// Union returns a union of the receiver GTIDSet and the supplied GTIDSet.
	Union(GTIDSet) GTIDSet
}

// gtidSetParsers maps flavor names to parser functions. It is used by
// ParsePosition().
var gtidSetParsers = make(map[string]func(string) (GTIDSet, error))
<commit_msg>Comment clarity.<commit_after>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mysql

// GTIDSet represents the set of transactions received or applied by a server.
// In some flavors, a single GTID is enough to specify the set of all
// transactions that came before it, but in others a more complex structure is
// required.
//
// GTIDSet is wrapped by replication.Position, which is a concrete struct.
// When sending a GTIDSet over RPCs, encode/decode it as a string.
// Most code outside of this package should use replication.Position rather
// than GTIDSet.
type GTIDSet interface {
	// String returns the canonical printed form of the set as expected by a
	// particular flavor of MySQL.
	String() string

	// Flavor returns the key under which the corresponding parser function is
	// registered in the transactionSetParsers map.
	Flavor() string

	// ContainsGTID returns true if the set contains the specified transaction.
	ContainsGTID(GTID) bool

	// Contains returns true if the set is a superset of another set. All implementations should return false if
	// other GTIDSet is not the right concrete type for that flavor.
	Contains(GTIDSet) bool

	// Equal returns true if the set is equal to another set.
	Equal(GTIDSet) bool

	// AddGTID returns a new GTIDSet that is expanded to contain the given GTID.
	AddGTID(GTID) GTIDSet

	// Union returns a union of the receiver GTIDSet and the supplied GTIDSet.
	Union(GTIDSet) GTIDSet
}

// gtidSetParsers maps flavor names to parser functions. It is used by
// ParsePosition().
var gtidSetParsers = make(map[string]func(string) (GTIDSet, error))
<|endoftext|>