<commit_before>package main

import (
	"fmt"
	"os"
	"strings"

	"github.com/codegangsta/cli"

	"github.com/y-uuki/grabeni/aws"
	. "github.com/y-uuki/grabeni/log"
)

var Commands = []cli.Command{
	commandStatus,
	commandList,
	commandGrab,
	commandAttach,
	commandDetach,
}

var commandStatus = cli.Command{
	Name:  "status",
	Usage: "Show ENI status",
	Description: `
    Show the information of the ENI identified with <eni-id>.
`,
	Action: doStatus,
}

var commandList = cli.Command{
	Name:  "list",
	Usage: "Show all ENIs",
	Description: `
    List the ENIs owned by your account.
`,
	Action: doList,
}

var commandGrab = cli.Command{
	Name:  "grab",
	Usage: "Detach and attach ENI",
	Description: `
    Detach the ENI identified with <eni-id> whether the eni has already attached or not. And then attach the ENI to EC2 instance identified with --instanceid option.
`,
	Action: doGrab,
	Flags: []cli.Flag{
		cli.IntFlag{Name: "d, deviceindex", Value: 1, Usage: "device index number"},
		cli.StringFlag{Name: "i, instanceid", Usage: "attach-targeted instance id"},
	},
}

var commandAttach = cli.Command{
	Name:  "attach",
	Usage: "Attach ENI",
	Description: `
    Just attach the ENI identified with <eni-id> to EC2 instance identified with --instanceid option.
`,
	Action: doAttach,
	Flags: []cli.Flag{
		cli.IntFlag{Name: "d, deviceindex", Value: 1, Usage: "device index number"},
		cli.StringFlag{Name: "i, instanceid", Usage: "attach-targeted instance id"},
	},
}

var commandDetach = cli.Command{
	Name:  "detach",
	Usage: "Detach ENI",
	Description: `
    Just detach the ENI identified with <eni-id>.
`,
	Action: doDetach,
	Flags: []cli.Flag{
		cli.StringFlag{Name: "i, instanceid", Usage: "detach-targeted instance id"},
	},
}

type commandDoc struct {
	Parent    string
	Arguments string
}

var commandDocs = map[string]commandDoc{
	"status": {"", "<eni-id>"},
	"list":  {"", ""},
	"grab": {"", "[--deviceindex | -d <device index> (default: 1)] [--instanceid | -i <instance id>] <eni-id>"},
	"attach": {"", "[--deviceindex | -d <device index> (default: 1)] [--instanceid | -i <instance id>] <eni-id>"},
	"detach":  {"", "[--instanceid | -i <instance id>] <eni-id>"},
}

// Makes template conditionals to generate per-command documents.
func mkCommandsTemplate(genTemplate func(commandDoc) string) string {
	template := "{{if false}}"
	for _, command := range append(Commands) {
		template = template + fmt.Sprintf("{{else if (eq .Name %q)}}%s", command.Name, genTemplate(commandDocs[command.Name]))
	}
	return template + "{{end}}"
}

func init() {
	argsTemplate := mkCommandsTemplate(func(doc commandDoc) string { return doc.Arguments })
	parentTemplate := mkCommandsTemplate(func(doc commandDoc) string { return string(strings.TrimLeft(doc.Parent+" ", " ")) })

	cli.CommandHelpTemplate = `NAME:
    {{.Name}} - {{.Usage}}

USAGE:
    grabeni ` + parentTemplate + `{{.Name}} ` + argsTemplate + `
{{if (len .Description)}}
DESCRIPTION: {{.Description}}
{{end}}{{if (len .Flags)}}
OPTIONS:
    {{range .Flags}}{{.}}
    {{end}}
{{end}}`
}

func awsCli(c *cli.Context) *aws.Client {
	client, err := aws.NewClient(
		c.GlobalString("region"),
		c.GlobalString("accesskey"),
		c.GlobalString("secretkey"),
	)
	DieIf(err)
	return client
}

func fetchInstanceIDIfEmpty(c *cli.Context) string {
	if instanceID := c.String("instanceid"); instanceID != "" {
		return c.String(instanceID)
	}

	instanceID, err := aws.GetInstanceID()
	if err != nil {
		cli.ShowCommandHelp(c, c.Command.Name)
		DieIf(err)
	}

	return instanceID
}

func doStatus(c *cli.Context) {
	if len(c.Args()) < 1 {
		cli.ShowCommandHelp(c, "status")
		os.Exit(1)
	}

	eniID := c.Args()[0]
	if eniID == "" {
		cli.ShowCommandHelp(c, "status")
		os.Exit(1)
	}

	eni, err := awsCli(c).DescribeENIByID(eniID)
	DieIf(err)
	if eni == nil {
		os.Exit(0)
	}

	fmt.Println(StatusHeader())
	fmt.Println(StatusRow(eni))
}

func doList(c *cli.Context) {
	enis, err := awsCli(c).DescribeENIs()
	DieIf(err)
	if enis == nil {
		os.Exit(0)
	}

	fmt.Println(StatusHeader())
	for _, eni := range enis {
		fmt.Println(StatusRow(eni))
	}
}

func doGrab(c *cli.Context) {
	if len(c.Args()) < 1 {
		cli.ShowCommandHelp(c, "grab")
		os.Exit(1)
	}

	eniID := c.Args()[0]
	if eniID == "" {
		cli.ShowCommandHelp(c, "grab")
		os.Exit(1)
	}

	instanceID := fetchInstanceIDIfEmpty(c)
	deviceIndex := c.Int("deviceindex")

	err, ok := awsCli(c).GrabENI(eniID, instanceID, deviceIndex)
	DieIf(err)
	if !ok {
		Logf("attached", "eni %s already attached to %s", eniID, instanceID)
		os.Exit(0)
	}

	Logf("grabbed", "eni %s attached to instance %s", eniID, instanceID)
}

func doAttach(c *cli.Context) {
	if len(c.Args()) < 1 {
		cli.ShowCommandHelp(c, "attach")
		os.Exit(1)
	}

	eniID := c.Args()[0]
	if eniID == "" {
		cli.ShowCommandHelp(c, "attach")
		os.Exit(1)
	}

	instanceID := fetchInstanceIDIfEmpty(c)
	deviceIndex := c.Int("deviceindex")

	err := awsCli(c).AttachENI(eniID, instanceID, deviceIndex)
	DieIf(err)

	Logf("attached", "eni %s attached to instance %s", eniID, instanceID)
}

func doDetach(c *cli.Context) {
	if len(c.Args()) < 1 {
		cli.ShowCommandHelp(c, "detach")
		os.Exit(1)
	}

	eniID := c.Args()[0]
	if eniID == "" {
		cli.ShowCommandHelp(c, "detach")
		os.Exit(1)
	}

	instanceID := fetchInstanceIDIfEmpty(c)

	err := awsCli(c).DetachENI(eniID, instanceID)
	DieIf(err)

	Logf("detached", "eni %s detached from instance %s", eniID, instanceID)
}
<commit_msg>Fix github username<commit_after>package main

import (
	"fmt"
	"os"
	"strings"

	"github.com/codegangsta/cli"

	"github.com/yuuki1/grabeni/aws"
	. "github.com/yuuki1/grabeni/log"
)

var Commands = []cli.Command{
	commandStatus,
	commandList,
	commandGrab,
	commandAttach,
	commandDetach,
}

var commandStatus = cli.Command{
	Name:  "status",
	Usage: "Show ENI status",
	Description: `
    Show the information of the ENI identified with <eni-id>.
`,
	Action: doStatus,
}

var commandList = cli.Command{
	Name:  "list",
	Usage: "Show all ENIs",
	Description: `
    List the ENIs owned by your account.
`,
	Action: doList,
}

var commandGrab = cli.Command{
	Name:  "grab",
	Usage: "Detach and attach ENI",
	Description: `
    Detach the ENI identified with <eni-id> whether the eni has already attached or not. And then attach the ENI to EC2 instance identified with --instanceid option.
`,
	Action: doGrab,
	Flags: []cli.Flag{
		cli.IntFlag{Name: "d, deviceindex", Value: 1, Usage: "device index number"},
		cli.StringFlag{Name: "i, instanceid", Usage: "attach-targeted instance id"},
	},
}

var commandAttach = cli.Command{
	Name:  "attach",
	Usage: "Attach ENI",
	Description: `
    Just attach the ENI identified with <eni-id> to EC2 instance identified with --instanceid option.
`,
	Action: doAttach,
	Flags: []cli.Flag{
		cli.IntFlag{Name: "d, deviceindex", Value: 1, Usage: "device index number"},
		cli.StringFlag{Name: "i, instanceid", Usage: "attach-targeted instance id"},
	},
}

var commandDetach = cli.Command{
	Name:  "detach",
	Usage: "Detach ENI",
	Description: `
    Just detach the ENI identified with <eni-id>.
`,
	Action: doDetach,
	Flags: []cli.Flag{
		cli.StringFlag{Name: "i, instanceid", Usage: "detach-targeted instance id"},
	},
}

type commandDoc struct {
	Parent    string
	Arguments string
}

var commandDocs = map[string]commandDoc{
	"status": {"", "<eni-id>"},
	"list":  {"", ""},
	"grab": {"", "[--deviceindex | -d <device index> (default: 1)] [--instanceid | -i <instance id>] <eni-id>"},
	"attach": {"", "[--deviceindex | -d <device index> (default: 1)] [--instanceid | -i <instance id>] <eni-id>"},
	"detach":  {"", "[--instanceid | -i <instance id>] <eni-id>"},
}

// Makes template conditionals to generate per-command documents.
func mkCommandsTemplate(genTemplate func(commandDoc) string) string {
	template := "{{if false}}"
	for _, command := range append(Commands) {
		template = template + fmt.Sprintf("{{else if (eq .Name %q)}}%s", command.Name, genTemplate(commandDocs[command.Name]))
	}
	return template + "{{end}}"
}

func init() {
	argsTemplate := mkCommandsTemplate(func(doc commandDoc) string { return doc.Arguments })
	parentTemplate := mkCommandsTemplate(func(doc commandDoc) string { return string(strings.TrimLeft(doc.Parent+" ", " ")) })

	cli.CommandHelpTemplate = `NAME:
    {{.Name}} - {{.Usage}}

USAGE:
    grabeni ` + parentTemplate + `{{.Name}} ` + argsTemplate + `
{{if (len .Description)}}
DESCRIPTION: {{.Description}}
{{end}}{{if (len .Flags)}}
OPTIONS:
    {{range .Flags}}{{.}}
    {{end}}
{{end}}`
}

func awsCli(c *cli.Context) *aws.Client {
	client, err := aws.NewClient(
		c.GlobalString("region"),
		c.GlobalString("accesskey"),
		c.GlobalString("secretkey"),
	)
	DieIf(err)
	return client
}

func fetchInstanceIDIfEmpty(c *cli.Context) string {
	if instanceID := c.String("instanceid"); instanceID != "" {
		return c.String(instanceID)
	}

	instanceID, err := aws.GetInstanceID()
	if err != nil {
		cli.ShowCommandHelp(c, c.Command.Name)
		DieIf(err)
	}

	return instanceID
}

func doStatus(c *cli.Context) {
	if len(c.Args()) < 1 {
		cli.ShowCommandHelp(c, "status")
		os.Exit(1)
	}

	eniID := c.Args()[0]
	if eniID == "" {
		cli.ShowCommandHelp(c, "status")
		os.Exit(1)
	}

	eni, err := awsCli(c).DescribeENIByID(eniID)
	DieIf(err)
	if eni == nil {
		os.Exit(0)
	}

	fmt.Println(StatusHeader())
	fmt.Println(StatusRow(eni))
}

func doList(c *cli.Context) {
	enis, err := awsCli(c).DescribeENIs()
	DieIf(err)
	if enis == nil {
		os.Exit(0)
	}

	fmt.Println(StatusHeader())
	for _, eni := range enis {
		fmt.Println(StatusRow(eni))
	}
}

func doGrab(c *cli.Context) {
	if len(c.Args()) < 1 {
		cli.ShowCommandHelp(c, "grab")
		os.Exit(1)
	}

	eniID := c.Args()[0]
	if eniID == "" {
		cli.ShowCommandHelp(c, "grab")
		os.Exit(1)
	}

	instanceID := fetchInstanceIDIfEmpty(c)
	deviceIndex := c.Int("deviceindex")

	err, ok := awsCli(c).GrabENI(eniID, instanceID, deviceIndex)
	DieIf(err)
	if !ok {
		Logf("attached", "eni %s already attached to %s", eniID, instanceID)
		os.Exit(0)
	}

	Logf("grabbed", "eni %s attached to instance %s", eniID, instanceID)
}

func doAttach(c *cli.Context) {
	if len(c.Args()) < 1 {
		cli.ShowCommandHelp(c, "attach")
		os.Exit(1)
	}

	eniID := c.Args()[0]
	if eniID == "" {
		cli.ShowCommandHelp(c, "attach")
		os.Exit(1)
	}

	instanceID := fetchInstanceIDIfEmpty(c)
	deviceIndex := c.Int("deviceindex")

	err := awsCli(c).AttachENI(eniID, instanceID, deviceIndex)
	DieIf(err)

	Logf("attached", "eni %s attached to instance %s", eniID, instanceID)
}

func doDetach(c *cli.Context) {
	if len(c.Args()) < 1 {
		cli.ShowCommandHelp(c, "detach")
		os.Exit(1)
	}

	eniID := c.Args()[0]
	if eniID == "" {
		cli.ShowCommandHelp(c, "detach")
		os.Exit(1)
	}

	instanceID := fetchInstanceIDIfEmpty(c)

	err := awsCli(c).DetachENI(eniID, instanceID)
	DieIf(err)

	Logf("detached", "eni %s detached from instance %s", eniID, instanceID)
}
<|endoftext|>


<commit_before>package ipc

import (
	"fmt"
	log "github.com/GameGophers/nsq-logger"
	nsq "github.com/bitly/go-nsq"
	"github.com/boltdb/bolt"
	"gopkg.in/vmihailenco/msgpack.v2"
	"os"
	"strings"
	"time"
)

const (
	DEFAULT_NSQLOOKUPD   = "127.0.0.1:4160"
	ENV_NSQLOOKUPD       = "NSQLOOKUPD_HOST"
	TOPIC                = "REDOLOG"
	CHANNEL              = "ARCH"
	SERVICE              = "[ARCH]"
	REDO_TIME_FORMAT     = "REDO-2006-01-02T15:04:05.RDO"
	REDO_ROTATE_INTERVAL = 24 * time.Hour
	BOLTDB_BUCKET        = "REDOLOG"
)

type Archiver struct {
	pending chan []byte
}

func (arch *Archiver) init() {
	arch.pending = make(chan []byte)
	cfg := nsq.NewConfig()
	consumer, err := nsq.NewConsumer(TOPIC, CHANNEL, cfg)
	if err != nil {
		log.Critical(err)
		os.Exit(-1)
	}

	// message process
	consumer.AddHandler(nsq.HandlerFunc(func(msg *nsq.Message) error {
		return nil
	}))

	// read environtment variable
	addresses := []string{DEFAULT_NSQLOOKUPD}
	if env := os.Getenv(ENV_NSQLOOKUPD); env != "" {
		addresses = strings.Split(env, ";")
	}

	// connect to nsqlookupd
	log.Trace("connect to nsqlookupds ip:", addresses)
	if err := consumer.ConnectToNSQLookupds(addresses); err != nil {
		log.Critical(err)
		return
	}
	log.Info("nsqlookupd connected")

	go arch.archive_task()
}

func (arch *Archiver) archive_task() {
	timer := time.After(REDO_ROTATE_INTERVAL)
	db, err := bolt.Open(time.Now().Format(REDO_TIME_FORMAT), 0600, nil)
	if err != nil {
		log.Critical(err)
		os.Exit(-1)
	}
	for {
		select {
		case msg := <-arch.pending:
			var record map[string]interface{}
			err := msgpack.Unmarshal(msg, &record)
			if err != nil {
				log.Error(err)
				continue
			}

			db.Update(func(tx *bolt.Tx) error {
				b := tx.Bucket([]byte(BOLTDB_BUCKET))
				err := b.Put([]byte(fmt.Sprint(record["TS"])), msg)
				return err
			})
		case <-timer:
			db.Close()
			db, err = bolt.Open(time.Now().Format(REDO_TIME_FORMAT), 0600, nil)
			if err != nil {
				log.Critical(err)
				os.Exit(-1)
			}
			timer = time.After(REDO_ROTATE_INTERVAL)
		}
	}
}
<commit_msg>update<commit_after>package ipc

import (
	"fmt"
	log "github.com/GameGophers/nsq-logger"
	nsq "github.com/bitly/go-nsq"
	"github.com/boltdb/bolt"
	"gopkg.in/vmihailenco/msgpack.v2"
	"os"
	"strings"
	"time"
)

const (
	DEFAULT_NSQLOOKUPD   = "127.0.0.1:4160"
	ENV_NSQLOOKUPD       = "NSQLOOKUPD_HOST"
	TOPIC                = "REDOLOG"
	CHANNEL              = "ARCH"
	SERVICE              = "[ARCH]"
	REDO_TIME_FORMAT     = "REDO-2006-01-02T15:04:05.RDO"
	REDO_ROTATE_INTERVAL = 24 * time.Hour
	BOLTDB_BUCKET        = "REDOLOG"
	DATA_DIRECTORY       = "/data"
)

type Archiver struct {
	pending chan []byte
}

func (arch *Archiver) init() {
	arch.pending = make(chan []byte)
	cfg := nsq.NewConfig()
	consumer, err := nsq.NewConsumer(TOPIC, CHANNEL, cfg)
	if err != nil {
		log.Critical(err)
		os.Exit(-1)
	}

	// message process
	consumer.AddHandler(nsq.HandlerFunc(func(msg *nsq.Message) error {
		return nil
	}))

	// read environtment variable
	addresses := []string{DEFAULT_NSQLOOKUPD}
	if env := os.Getenv(ENV_NSQLOOKUPD); env != "" {
		addresses = strings.Split(env, ";")
	}

	// connect to nsqlookupd
	log.Trace("connect to nsqlookupds ip:", addresses)
	if err := consumer.ConnectToNSQLookupds(addresses); err != nil {
		log.Critical(err)
		return
	}
	log.Info("nsqlookupd connected")

	go arch.archive_task()
}

func (arch *Archiver) archive_task() {
	timer := time.After(REDO_ROTATE_INTERVAL)
	db, err := bolt.Open(DATA_DIRECTORY+"/"+time.Now().Format(REDO_TIME_FORMAT), 0600, nil)
	if err != nil {
		log.Critical(err)
		os.Exit(-1)
	}
	for {
		select {
		case msg := <-arch.pending:
			var record map[string]interface{}
			err := msgpack.Unmarshal(msg, &record)
			if err != nil {
				log.Error(err)
				continue
			}

			db.Update(func(tx *bolt.Tx) error {
				b := tx.Bucket([]byte(BOLTDB_BUCKET))
				err := b.Put([]byte(fmt.Sprint(record["TS"])), msg)
				return err
			})
		case <-timer:
			db.Close()
			db, err = bolt.Open(DATA_DIRECTORY+"/"+time.Now().Format(REDO_TIME_FORMAT), 0600, nil)
			if err != nil {
				log.Critical(err)
				os.Exit(-1)
			}
			timer = time.After(REDO_ROTATE_INTERVAL)
		}
	}
}
<|endoftext|>


<commit_before>package controller

import (
	"github.com/MG-RAST/AWE/lib/conf"
	"github.com/MG-RAST/AWE/lib/core"
	e "github.com/MG-RAST/AWE/lib/errors"
	"github.com/MG-RAST/AWE/lib/request"
	"github.com/MG-RAST/AWE/lib/user"
	"github.com/MG-RAST/golib/goweb"
	"github.com/MG-RAST/golib/mgo"
	"github.com/MG-RAST/golib/mgo/bson"
	"net/http"
	"strconv"
	"strings"
)

type ClientGroupController struct{}

// OPTIONS: /clientgroup
func (cr *ClientGroupController) Options(cx *goweb.Context) {
	LogRequest(cx.Request)
	cx.RespondWithOK()
	return
}

// POST: /clientgroup/{name}
func (cr *ClientGroupController) CreateWithId(name string, cx *goweb.Context) {
	LogRequest(cx.Request)

	// Try to authenticate user.
	u, err := request.Authenticate(cx.Request)
	if err != nil && err.Error() != e.NoAuth {
		cx.RespondWithErrorMessage(err.Error(), http.StatusUnauthorized)
		return
	}

	// If no auth was provided and ANON_CG_WRITE is true, use the public user.
	// Otherwise if no auth was provided or user is not an admin, and ANON_CG_WRITE is false, throw an error.
	// Otherwise, proceed with creation of the clientgroup with the user.
	if u == nil && conf.ANON_CG_WRITE == true {
		u = &user.User{Uuid: "public"}
	} else if u == nil || !u.Admin {
		if conf.ANON_CG_WRITE == false {
			cx.RespondWithErrorMessage(e.UnAuth, http.StatusUnauthorized)
			return
		}
	}

	cg, err := core.CreateClientGroup(name, u)
	if err != nil {
		cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
		return
	}

	cx.RespondWithData(cg)
	return
}

// GET: /clientgroup/{id}
func (cr *ClientGroupController) Read(id string, cx *goweb.Context) {
	LogRequest(cx.Request)

	// Try to authenticate user.
	u, err := request.Authenticate(cx.Request)
	if err != nil && err.Error() != e.NoAuth {
		cx.RespondWithErrorMessage(err.Error(), http.StatusUnauthorized)
		return
	}

	// If no auth was provided and ANON_CG_READ is true, use the public user.
	// Otherwise if no auth was provided, throw an error.
	// Otherwise, proceed with retrieval of the clientgroup using the user.
	if u == nil {
		if conf.ANON_CG_READ == true {
			u = &user.User{Uuid: "public"}
		} else {
			cx.RespondWithErrorMessage(e.UnAuth, http.StatusUnauthorized)
			return
		}
	}

	// Load clientgroup by id
	cg, err := core.LoadClientGroup(id)

	if err != nil {
		if err == mgo.ErrNotFound {
			cx.RespondWithNotFound()
		} else {
			// In theory the db connection could be lost between
			// checking user and load but seems unlikely.
			cx.RespondWithErrorMessage("clientgroup id not found:"+id, http.StatusBadRequest)
		}
		return
	}

	// User must have read permissions on clientgroup or be clientgroup owner or be an admin or the clientgroup is publicly readable.
	// The other possibility is that public read of clientgroups is enabled and the clientgroup is publicly readable.
	rights := cg.Acl.Check(u.Uuid)
	public_rights := cg.Acl.Check("public")
	if (u.Uuid != "public" && (cg.Acl.Owner == u.Uuid || rights["read"] == true || u.Admin == true || public_rights["read"] == true)) ||
		(u.Uuid == "public" && conf.ANON_CG_READ == true && public_rights["read"] == true) {
		cx.RespondWithData(cg)
		return
	}

	cx.RespondWithErrorMessage(e.UnAuth, http.StatusUnauthorized)
	return
}

// GET: /clientgroup
func (cr *ClientGroupController) ReadMany(cx *goweb.Context) {
	LogRequest(cx.Request)

	// Try to authenticate user.
	u, err := request.Authenticate(cx.Request)
	if err != nil && err.Error() != e.NoAuth {
		cx.RespondWithErrorMessage(err.Error(), http.StatusUnauthorized)
		return
	}

	// If no auth was provided and ANON_CG_READ is true, use the public user.
	// Otherwise if no auth was provided, throw an error.
	// Otherwise, proceed with retrieval of the clientgroups using the user.
	if u == nil {
		if conf.ANON_CG_READ == true {
			u = &user.User{Uuid: "public"}
		} else {
			cx.RespondWithErrorMessage(e.UnAuth, http.StatusUnauthorized)
			return
		}
	}

	// Gather query params
	query := &Query{Li: cx.Request.URL.Query()}

	// Setup query and clientgroups objects
	q := bson.M{}
	cgs := core.ClientGroups{}

	// Add authorization checking to query if the user is not an admin
	if u.Admin == false {
		q["$or"] = []bson.M{bson.M{"acl.read": "public"}, bson.M{"acl.read": u.Uuid}, bson.M{"acl.owner": u.Uuid}}
	}

	limit := conf.DEFAULT_PAGE_SIZE
	offset := 0
	order := "last_modified"
	direction := "desc"
	if query.Has("limit") {
		limit, err = strconv.Atoi(query.Value("limit"))
		if err != nil {
			cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
			return
		}
	}
	if query.Has("offset") {
		offset, err = strconv.Atoi(query.Value("offset"))
		if err != nil {
			cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
			return
		}
	}
	if query.Has("order") {
		order = query.Value("order")
	}
	if query.Has("direction") {
		direction = query.Value("direction")
	}

	// Gather params to make db query. Do not include the following list.
	skip := map[string]int{
		"limit":     1,
		"offset":    1,
		"order":     1,
		"direction": 1,
	}

	for key, val := range query.All() {
		_, s := skip[key]
		if !s {
			queryvalues := strings.Split(val[0], ",")
			q[key] = bson.M{"$in": queryvalues}
		}
	}

	total, err := cgs.GetPaginated(q, limit, offset, order, direction)
	if err != nil {
		cx.RespondWithError(http.StatusBadRequest)
		return
	}

	cx.RespondWithPaginatedData(cgs, limit, offset, total)
	return
}

// DELETE: /clientgroup/{id}
func (cr *ClientGroupController) Delete(id string, cx *goweb.Context) {
	LogRequest(cx.Request)

	// Try to authenticate user.
	u, err := request.Authenticate(cx.Request)
	if err != nil && err.Error() != e.NoAuth {
		cx.RespondWithErrorMessage(err.Error(), http.StatusUnauthorized)
		return
	}

	// If no auth was provided and ANON_CG_DELETE is true, use the public user.
	// Otherwise if no auth was provided, throw an error.
	// Otherwise, proceed with deletion of the clientgroup using the user.
	if u == nil {
		if conf.ANON_CG_DELETE == true {
			u = &user.User{Uuid: "public"}
		} else {
			cx.RespondWithErrorMessage(e.UnAuth, http.StatusUnauthorized)
			return
		}
	}

	// Load clientgroup by id
	cg, err := core.LoadClientGroup(id)

	if err != nil {
		if err == mgo.ErrNotFound {
			cx.RespondWithNotFound()
		} else {
			// In theory the db connection could be lost between
			// checking user and load but seems unlikely.
			cx.RespondWithErrorMessage("clientgroup id not found:"+id, http.StatusBadRequest)
		}
		return
	}

	// User must have delete permissions on clientgroup or be clientgroup owner or be an admin or the clientgroup is publicly deletable.
	// The other possibility is that public deletion of clientgroups is enabled and the clientgroup is publicly deletable.
	rights := cg.Acl.Check(u.Uuid)
	public_rights := cg.Acl.Check("public")
	if (u.Uuid != "public" && (cg.Acl.Owner == u.Uuid || rights["delete"] == true || u.Admin == true || public_rights["delete"] == true)) ||
		(u.Uuid == "public" && conf.ANON_CG_DELETE == true && public_rights["delete"] == true) {
		err := core.DeleteClientGroup(id)
		if err != nil {
			cx.RespondWithErrorMessage("Could not delete clientgroup.", http.StatusInternalServerError)
			return
		}
		cx.RespondWithOK()
		return
	}

	cx.RespondWithErrorMessage(e.UnAuth, http.StatusUnauthorized)
	return
}
<commit_msg>Fixing resource name.<commit_after>package controller

import (
	"github.com/MG-RAST/AWE/lib/conf"
	"github.com/MG-RAST/AWE/lib/core"
	e "github.com/MG-RAST/AWE/lib/errors"
	"github.com/MG-RAST/AWE/lib/request"
	"github.com/MG-RAST/AWE/lib/user"
	"github.com/MG-RAST/golib/goweb"
	"github.com/MG-RAST/golib/mgo"
	"github.com/MG-RAST/golib/mgo/bson"
	"net/http"
	"strconv"
	"strings"
)

type ClientGroupController struct{}

// OPTIONS: /cgroup
func (cr *ClientGroupController) Options(cx *goweb.Context) {
	LogRequest(cx.Request)
	cx.RespondWithOK()
	return
}

// POST: /cgroup/{name}
func (cr *ClientGroupController) CreateWithId(name string, cx *goweb.Context) {
	LogRequest(cx.Request)

	// Try to authenticate user.
	u, err := request.Authenticate(cx.Request)
	if err != nil && err.Error() != e.NoAuth {
		cx.RespondWithErrorMessage(err.Error(), http.StatusUnauthorized)
		return
	}

	// If no auth was provided and ANON_CG_WRITE is true, use the public user.
	// Otherwise if no auth was provided or user is not an admin, and ANON_CG_WRITE is false, throw an error.
	// Otherwise, proceed with creation of the clientgroup with the user.
	if u == nil && conf.ANON_CG_WRITE == true {
		u = &user.User{Uuid: "public"}
	} else if u == nil || !u.Admin {
		if conf.ANON_CG_WRITE == false {
			cx.RespondWithErrorMessage(e.UnAuth, http.StatusUnauthorized)
			return
		}
	}

	cg, err := core.CreateClientGroup(name, u)
	if err != nil {
		cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
		return
	}

	cx.RespondWithData(cg)
	return
}

// GET: /cgroup/{id}
func (cr *ClientGroupController) Read(id string, cx *goweb.Context) {
	LogRequest(cx.Request)

	// Try to authenticate user.
	u, err := request.Authenticate(cx.Request)
	if err != nil && err.Error() != e.NoAuth {
		cx.RespondWithErrorMessage(err.Error(), http.StatusUnauthorized)
		return
	}

	// If no auth was provided and ANON_CG_READ is true, use the public user.
	// Otherwise if no auth was provided, throw an error.
	// Otherwise, proceed with retrieval of the clientgroup using the user.
	if u == nil {
		if conf.ANON_CG_READ == true {
			u = &user.User{Uuid: "public"}
		} else {
			cx.RespondWithErrorMessage(e.UnAuth, http.StatusUnauthorized)
			return
		}
	}

	// Load clientgroup by id
	cg, err := core.LoadClientGroup(id)

	if err != nil {
		if err == mgo.ErrNotFound {
			cx.RespondWithNotFound()
		} else {
			// In theory the db connection could be lost between
			// checking user and load but seems unlikely.
			cx.RespondWithErrorMessage("clientgroup id not found:"+id, http.StatusBadRequest)
		}
		return
	}

	// User must have read permissions on clientgroup or be clientgroup owner or be an admin or the clientgroup is publicly readable.
	// The other possibility is that public read of clientgroups is enabled and the clientgroup is publicly readable.
	rights := cg.Acl.Check(u.Uuid)
	public_rights := cg.Acl.Check("public")
	if (u.Uuid != "public" && (cg.Acl.Owner == u.Uuid || rights["read"] == true || u.Admin == true || public_rights["read"] == true)) ||
		(u.Uuid == "public" && conf.ANON_CG_READ == true && public_rights["read"] == true) {
		cx.RespondWithData(cg)
		return
	}

	cx.RespondWithErrorMessage(e.UnAuth, http.StatusUnauthorized)
	return
}

// GET: /cgroup
func (cr *ClientGroupController) ReadMany(cx *goweb.Context) {
	LogRequest(cx.Request)

	// Try to authenticate user.
	u, err := request.Authenticate(cx.Request)
	if err != nil && err.Error() != e.NoAuth {
		cx.RespondWithErrorMessage(err.Error(), http.StatusUnauthorized)
		return
	}

	// If no auth was provided and ANON_CG_READ is true, use the public user.
	// Otherwise if no auth was provided, throw an error.
	// Otherwise, proceed with retrieval of the clientgroups using the user.
	if u == nil {
		if conf.ANON_CG_READ == true {
			u = &user.User{Uuid: "public"}
		} else {
			cx.RespondWithErrorMessage(e.UnAuth, http.StatusUnauthorized)
			return
		}
	}

	// Gather query params
	query := &Query{Li: cx.Request.URL.Query()}

	// Setup query and clientgroups objects
	q := bson.M{}
	cgs := core.ClientGroups{}

	// Add authorization checking to query if the user is not an admin
	if u.Admin == false {
		q["$or"] = []bson.M{bson.M{"acl.read": "public"}, bson.M{"acl.read": u.Uuid}, bson.M{"acl.owner": u.Uuid}}
	}

	limit := conf.DEFAULT_PAGE_SIZE
	offset := 0
	order := "last_modified"
	direction := "desc"
	if query.Has("limit") {
		limit, err = strconv.Atoi(query.Value("limit"))
		if err != nil {
			cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
			return
		}
	}
	if query.Has("offset") {
		offset, err = strconv.Atoi(query.Value("offset"))
		if err != nil {
			cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
			return
		}
	}
	if query.Has("order") {
		order = query.Value("order")
	}
	if query.Has("direction") {
		direction = query.Value("direction")
	}

	// Gather params to make db query. Do not include the following list.
	skip := map[string]int{
		"limit":     1,
		"offset":    1,
		"order":     1,
		"direction": 1,
	}

	for key, val := range query.All() {
		_, s := skip[key]
		if !s {
			queryvalues := strings.Split(val[0], ",")
			q[key] = bson.M{"$in": queryvalues}
		}
	}

	total, err := cgs.GetPaginated(q, limit, offset, order, direction)
	if err != nil {
		cx.RespondWithError(http.StatusBadRequest)
		return
	}

	cx.RespondWithPaginatedData(cgs, limit, offset, total)
	return
}

// DELETE: /cgroup/{id}
func (cr *ClientGroupController) Delete(id string, cx *goweb.Context) {
	LogRequest(cx.Request)

	// Try to authenticate user.
	u, err := request.Authenticate(cx.Request)
	if err != nil && err.Error() != e.NoAuth {
		cx.RespondWithErrorMessage(err.Error(), http.StatusUnauthorized)
		return
	}

	// If no auth was provided and ANON_CG_DELETE is true, use the public user.
	// Otherwise if no auth was provided, throw an error.
	// Otherwise, proceed with deletion of the clientgroup using the user.
	if u == nil {
		if conf.ANON_CG_DELETE == true {
			u = &user.User{Uuid: "public"}
		} else {
			cx.RespondWithErrorMessage(e.UnAuth, http.StatusUnauthorized)
			return
		}
	}

	// Load clientgroup by id
	cg, err := core.LoadClientGroup(id)

	if err != nil {
		if err == mgo.ErrNotFound {
			cx.RespondWithNotFound()
		} else {
			// In theory the db connection could be lost between
			// checking user and load but seems unlikely.
			cx.RespondWithErrorMessage("clientgroup id not found:"+id, http.StatusBadRequest)
		}
		return
	}

	// User must have delete permissions on clientgroup or be clientgroup owner or be an admin or the clientgroup is publicly deletable.
	// The other possibility is that public deletion of clientgroups is enabled and the clientgroup is publicly deletable.
	rights := cg.Acl.Check(u.Uuid)
	public_rights := cg.Acl.Check("public")
	if (u.Uuid != "public" && (cg.Acl.Owner == u.Uuid || rights["delete"] == true || u.Admin == true || public_rights["delete"] == true)) ||
		(u.Uuid == "public" && conf.ANON_CG_DELETE == true && public_rights["delete"] == true) {
		err := core.DeleteClientGroup(id)
		if err != nil {
			cx.RespondWithErrorMessage("Could not delete clientgroup.", http.StatusInternalServerError)
			return
		}
		cx.RespondWithOK()
		return
	}

	cx.RespondWithErrorMessage(e.UnAuth, http.StatusUnauthorized)
	return
}
<|endoftext|>


<commit_before>// Package wildmatch solves inclusion problem for the wildcard path patterns.
package wildmatch

import (
	"strings"
)

// Wildcard respresents wildcard pattern.
// It is an alias to the string type to provide extra methods of this package.
type Wildcard string

// IsSubset verifies if current wildcard is a subset of a given one `superset`.
// Wildcard A is subset of B if any possible path that matches A also matches B.
func (w Wildcard) IsSubsetOf(superset Wildcard) bool {
	return IsSubsetOf(string(w), string(superset))
}

// Contains verifies if current wildcard is a superset of a given one `subset`.
// This operation is inversion of IsSubsetOf().
func (w Wildcard) Contains(subset Wildcard) bool {
	return IsSubsetOf(string(subset), string(w))
}

// IsSubsetOf verifies if `w` wildcard is a subset of `s`.
// I.e. checks if `s` is a superset of subset `w`.
func IsSubsetOf(w string, s string) bool {

	// shortcut for identical sets
	if s == w {
		return true
	}

	// only empty set is a subset of an empty set
	if len(s) == 0 {
		return len(w) == 0
	}

	// find nesting separators
	sp := strings.Index(s, "/")
	wp := strings.Index(w, "/")

	// check if this is a nested path
	if sp >= 0 {

		// if set is nested then tested wildcard must be nested too
		if wp < 0 {
			return false
		}

		// Special case for /**/ mask that matches any number of levels
		if s[:sp] == "**" &&
			IsSubsetOf(w[wp+1:], s) ||
			IsSubsetOf(w, s[sp+1:]) {
			return true
		}

		// check that current level names are subsets
		// and compare rest of the path to be subset also
		return (IsSubsetOf(w[:wp], s[:sp]) &&
			IsSubsetOf(w[wp+1:], s[sp+1:]))
	}

	// subset can't have more levels than set
	if wp >= 0 {
		return false
	}

	// we are comparing names on the same nesting level here
	// so let's do symbol by symbol comparison
	switch s[0] {
	case '?':
		// ? matches non empty character. '*' can't be a subset of '?'
		if len(w) == 0 || w[0] == '*' {
			return false
		}
		// any onther symbol matches '?', so let's skip to next
		return IsSubsetOf(w[1:], s[1:])
	case '*':
		// '*' matches 0 and any other number of symbols
		// so checking 0 and recursively subset without first letter
		return IsSubsetOf(w, s[1:]) ||
			(len(w) > 0 && IsSubsetOf(w[1:], s))
	default:
		// making sure next symbol in w exists and it's the same as in set
		if len(w) == 0 || w[0] != s[0] {
			return false
		}
	}

	// recursively check rest of the set and w
	return IsSubsetOf(w[1:], s[1:])
}

// IsSubsetOfAny verifies if current wildcard is a subset of any of the given sets.
// Wildcard A is subset of B if any possible path that matches A also matches B.
// If multiple subsets match then the smallest or first lexicographical set is returned
func (w Wildcard) IsSubsetOfAny(supersets ...Wildcard) (result Wildcard, found bool) {
	for _, superset := range supersets {
		if !w.IsSubsetOf(superset) {
			continue
		}
		found = true
		if result == "" || superset.IsSubsetOf(result) {
			result = superset
		}
	}
	return
}

// IsSubsetOfAny verifies if current wildcard `w` is a subset of any of the given sets.
// Wildcard A is subset of B if any possible path that matches A also matches B.
// If multiple subsets match then the smallest or first lexicographical set is returned
func IsSubsetOfAny(w string, sets ...string) (result string, found bool) {
	for _, superset := range sets {
		if !IsSubsetOf(w, superset) {
			continue
		}
		found = true
		if result == "" || IsSubsetOf(superset, result) {
			result = superset
		}
	}
	return
}

// NOTE it's hard to combine both IsSubsetOfAny() functions to one
// because of missing []string <-> []Wildcard conversion
<commit_msg>introduce IsSubsetOfAnyI function which returns matched index<commit_after>// Package wildmatch solves inclusion problem for the wildcard path patterns.
package wildmatch

import (
	"strings"
)

// Wildcard respresents wildcard pattern.
// It is an alias to the string type to provide extra methods of this package.
type Wildcard string

// IsSubset verifies if current wildcard is a subset of a given one `superset`.
// Wildcard A is subset of B if any possible path that matches A also matches B.
func (w Wildcard) IsSubsetOf(superset Wildcard) bool {
	return IsSubsetOf(string(w), string(superset))
}

// Contains verifies if current wildcard is a superset of a given one `subset`.
// This operation is inversion of IsSubsetOf().
func (w Wildcard) Contains(subset Wildcard) bool {
	return IsSubsetOf(string(subset), string(w))
}

// IsSubsetOf verifies if `w` wildcard is a subset of `s`.
// I.e. checks if `s` is a superset of subset `w`.
func IsSubsetOf(w string, s string) bool {

	// shortcut for identical sets
	if s == w {
		return true
	}

	// only empty set is a subset of an empty set
	if len(s) == 0 {
		return len(w) == 0
	}

	// find nesting separators
	sp := strings.Index(s, "/")
	wp := strings.Index(w, "/")

	// check if this is a nested path
	if sp >= 0 {

		// if set is nested then tested wildcard must be nested too
		if wp < 0 {
			return false
		}

		// Special case for /**/ mask that matches any number of levels
		if s[:sp] == "**" &&
			IsSubsetOf(w[wp+1:], s) ||
			IsSubsetOf(w, s[sp+1:]) {
			return true
		}

		// check that current level names are subsets
		// and compare rest of the path to be subset also
		return (IsSubsetOf(w[:wp], s[:sp]) &&
			IsSubsetOf(w[wp+1:], s[sp+1:]))
	}

	// subset can't have more levels than set
	if wp >= 0 {
		return false
	}

	// we are comparing names on the same nesting level here
	// so let's do symbol by symbol comparison
	switch s[0] {
	case '?':
		// ? matches non empty character. '*' can't be a subset of '?'
		if len(w) == 0 || w[0] == '*' {
			return false
		}
		// any onther symbol matches '?', so let's skip to next
		return IsSubsetOf(w[1:], s[1:])
	case '*':
		// '*' matches 0 and any other number of symbols
		// so checking 0 and recursively subset without first letter
		return IsSubsetOf(w, s[1:]) ||
			(len(w) > 0 && IsSubsetOf(w[1:], s))
	default:
		// making sure next symbol in w exists and it's the same as in set
		if len(w) == 0 || w[0] != s[0] {
			return false
		}
	}

	// recursively check rest of the set and w
	return IsSubsetOf(w[1:], s[1:])
}

// IsSubsetOfAny verifies if current wildcard is a subset of any of the given sets.
// Wildcard A is subset of B if any possible path that matches A also matches B.
// If multiple subsets match then the smallest or first lexicographical set is returned
func (w Wildcard) IsSubsetOfAny(supersets ...Wildcard) (result Wildcard, found bool) {
	for _, superset := range supersets {
		if !w.IsSubsetOf(superset) {
			continue
		}
		found = true
		if result == "" || superset.IsSubsetOf(result) {
			result = superset
		}
	}
	return
}

// IsSubsetOfAny verifies if current wildcard `w` is a subset of any of the given sets.
// Wildcard A is subset of B if any possible path that matches A also matches B.
// If multiple subsets match then the smallest or first lexicographical set is returned
func IsSubsetOfAny(w string, sets ...string) (result string, found bool) {
	if index := IsSubsetOfAnyI(w, sets...); index >= 0 {
		return sets[index], true
	}

	return "", false // not found
}

// IsSubsetOfAny verifies if current wildcard `w` is a subset of any of the given sets.
// Wildcard A is subset of B if any possible path that matches A also matches B.
// If multiple subsets match then the smallest or first lexicographical set is returned
// Return -1 if not found or superset index.
func IsSubsetOfAnyI(w string, sets ...string) (found int) {
	found = -1 // not found by default
	for i, superset := range sets {
		if !IsSubsetOf(w, superset) {
			continue
		}
		if found < 0 || IsSubsetOf(superset, sets[found]) {
			found = i
		}
	}
	return
}

// NOTE it's hard to combine both IsSubsetOfAny() functions to one
// because of missing []string <-> []Wildcard conversion
<|endoftext|>


<commit_before>package receiver

import (
	"context"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strconv"
	"strings"

	gw "github.com/cvmfs/gateway/internal/gateway"
	"github.com/pkg/errors"
)

// Error is returned by the various receiver commands in case of error
type Error string

func (e Error) Error() string {
	return string(e)
}

// receiverOp is used to identify the different operation performed
// by the cvmfs_receiver process
type receiverOp int32

// The different operations are defined as constants. The numbering
// must match (enum receiver::Request from "cvmfs.git/cvmfs/receiver/reactor.h")
const (
	receiverQuit receiverOp = iota
	receiverEcho
	receiverGenerateToken // Unused
	receiverGetTokenID    // Unused
	receiverCheckToken    // Unused
	receiverSubmitPayload
	receiverCommit
	receiverError // Unused
)

// Receiver contains the operations that "receiver" worker processes perform
type Receiver interface {
	Quit() error
	Echo() error
	SubmitPayload(leasePath string, payload io.Reader, digest string, headerSize int) error
	Commit(leasePath, oldRootHash, newRootHash string, tag gw.RepositoryTag) error
}

// NewReceiver is the factory method for Receiver types
func NewReceiver(ctx context.Context, execPath string, mock bool) (Receiver, error) {
	if mock {
		return NewMockReceiver(ctx)
	}

	return NewCvmfsReceiver(ctx, execPath)
}

// CvmfsReceiver spawns an external cvmfs_receiver worker process
type CvmfsReceiver struct {
	worker       *exec.Cmd
	workerCmdIn  io.WriteCloser
	workerCmdOut io.ReadCloser
	workerStderr io.ReadCloser
	workerStdout io.ReadCloser
	ctx          context.Context
}

// NewCvmfsReceiver will spawn an external cvmfs_receiver worker process and wait for a command
func NewCvmfsReceiver(ctx context.Context, execPath string) (*CvmfsReceiver, error) {
	if _, err := os.Stat(execPath); os.IsNotExist(err) {
		return nil, errors.Wrap(err, "worker process executable not found")
	}

	cmd := exec.Command(execPath, "-i", strconv.Itoa(3), "-o", strconv.Itoa(4))

	workerInRead, workerInWrite, err := os.Pipe()
	if err != nil {
		return nil, errors.Wrap(err, "could not create worker input pipe")
	}
	workerOutRead, workerOutWrite, err := os.Pipe()
	if err != nil {
		return nil, errors.Wrap(err, "could not create worker output pipe")
	}

	cmd.ExtraFiles = []*os.File{workerInRead, workerOutWrite}

	stderr, err := cmd.StderrPipe()
	if err != nil {
		return nil, errors.Wrap(err, "could not create worker stderr pipe")
	}
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, errors.Wrap(err, "could not create worker stdout pipe")
	}

	if err := cmd.Start(); err != nil {
		return nil, errors.Wrap(err, "could not start worker process")
	}

	gw.LogC(ctx, "receiver", gw.LogDebug).
		Str("command", "start").
		Msg("worker process ready")

	return &CvmfsReceiver{
		worker: cmd, workerCmdIn: workerInWrite, workerCmdOut: workerOutRead,
		workerStderr: stderr, workerStdout: stdout, ctx: ctx}, nil
}

// Quit command is sent to the worker
func (r *CvmfsReceiver) Quit() error {
	defer func() {
		r.workerCmdIn.Close()
		r.workerCmdOut.Close()
	}()

	if _, err := r.call(receiverQuit, []byte{}, nil); err != nil {
		return errors.Wrap(err, "worker 'quit' call failed")
	}

	var buf1 []byte
	if _, err := io.ReadFull(r.workerStderr, buf1); err != nil {
		return errors.Wrap(err, "could not retrieve worker stderr")
	}
	gw.LogC(r.ctx, "receiver", gw.LogDebug).
		Str("pipe", "stderr").
		Msg(string(buf1))

	var buf2 []byte
	if _, err := io.ReadFull(r.workerStdout, buf2); err != nil {
		return errors.Wrap(err, "could not retrieve worker stdout")
	}
	gw.LogC(r.ctx, "receiver", gw.LogDebug).
		Str("pipe", "stdout").
		Msg(string(buf2))

	if err := r.worker.Wait(); err != nil {
		return errors.Wrap(err, "waiting for worker process failed")
	}

	gw.LogC(r.ctx, "receiver", gw.LogDebug).
		Str("command", "quit").
		Msg("worker process has stopped")

	return nil
}

// Echo command is sent to the worker
func (r *CvmfsReceiver) Echo() error {
	rep, err := r.call(receiverEcho, []byte("Ping"), nil)
	if err != nil {
		return errors.Wrap(err, "worker 'echo' call failed")
	}
	reply := string(rep)

	if !strings.HasPrefix(reply, "PID: ") {
		return fmt.Errorf("invalid 'echo' reply received: %v", reply)
	}

	gw.LogC(r.ctx, "receiver", gw.LogDebug).
		Str("command", "echo").
		Msgf("reply: %v", reply)

	return nil
}

// SubmitPayload command is sent to the worker
func (r *CvmfsReceiver) SubmitPayload(leasePath string, payload io.Reader, digest string, headerSize int) error {
	req := map[string]interface{}{"path": leasePath, "digest": digest, "header_size": headerSize}
	buf, err := json.Marshal(&req)
	if err != nil {
		return errors.Wrap(err, "request encoding failed")
	}
	reply, err := r.call(receiverSubmitPayload, buf, payload)
	if err != nil {
		return errors.Wrap(err, "worker 'payload submission' call failed")
	}

	result := toReceiverError(reply)

	gw.LogC(r.ctx, "receiver", gw.LogDebug).
		Str("command", "submit payload").
		Msgf("result: %v", result)

	return result
}

// Commit command is sent to the worker
func (r *CvmfsReceiver) Commit(leasePath, oldRootHash, newRootHash string, tag gw.RepositoryTag) error {
	req := map[string]interface{}{
		"lease_path":      leasePath,
		"old_root_hash":   oldRootHash,
		"new_root_hash":   newRootHash,
		"tag_name":        tag.Name,
		"tag_channel":     tag.Channel,
		"tag_description": tag.Description,
	}
	buf, err := json.Marshal(&req)
	if err != nil {
		return errors.Wrap(err, "request encoding failed")
	}

	reply, err := r.call(receiverCommit, buf, nil)
	if err != nil {
		return errors.Wrap(err, "worker 'commit' call failed")
	}

	result := toReceiverError(reply)

	gw.LogC(r.ctx, "receiver", gw.LogDebug).
		Str("command", "commit").
		Msgf("result: %v", result)

	return result
}

func (r *CvmfsReceiver) call(reqID receiverOp, msg []byte, payload io.Reader) ([]byte, error) {
	if err := r.request(reqID, msg, payload); err != nil {
		return nil, err
	}
	return r.reply()
}

func (r *CvmfsReceiver) request(reqID receiverOp, msg []byte, payload io.Reader) error {
	buf := make([]byte, 8+len(msg))
	binary.LittleEndian.PutUint32(buf[:4], uint32(reqID))
	binary.LittleEndian.PutUint32(buf[4:8], uint32(len(msg)))
	copy(buf[8:], msg)

	if _, err := r.workerCmdIn.Write(buf); err != nil {
		return errors.Wrap(err, "could not write request")
	}
	if payload != nil {
		if _, err := io.Copy(r.workerCmdIn, payload); err != nil {
			return errors.Wrap(err, "could not write request payload")
		}
	}
	return nil
}

func (r *CvmfsReceiver) reply() ([]byte, error) {
	buf := make([]byte, 4)
	if _, err := io.ReadFull(r.workerCmdOut, buf); err != nil {
		return nil, errors.Wrap(err, "could not read reply size")
	}
	repSize := int32(binary.LittleEndian.Uint32(buf))

	reply := make([]byte, repSize)
	if _, err := io.ReadFull(r.workerCmdOut, reply); err != nil {
		return nil, errors.Wrap(err, "could not read reply body")
	}

	return reply, nil
}

func toReceiverError(reply []byte) error {
	res := make(map[string]string)
	if err := json.Unmarshal(reply, &res); err != nil {
		return errors.Wrap(err, "could not decode reply")
	}

	if status, ok := res["status"]; ok {
		if status == "ok" {
			return nil
		}

		if reason, ok := res["reason"]; ok {
			return Error(reason)
		}

		return fmt.Errorf("invalid reply")
	}

	return fmt.Errorf("invalid reply")
}
<commit_msg>cvmfs_receiver reply JSON: allow additional unknown fields<commit_after>package receiver

import (
	"context"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strconv"
	"strings"

	gw "github.com/cvmfs/gateway/internal/gateway"
	"github.com/pkg/errors"
)

// Error is returned by the various receiver commands in case of error
type Error string

func (e Error) Error() string {
	return string(e)
}

// receiverOp is used to identify the different operation performed
// by the cvmfs_receiver process
type receiverOp int32

// The different operations are defined as constants. The numbering
// must match (enum receiver::Request from "cvmfs.git/cvmfs/receiver/reactor.h")
const (
	receiverQuit receiverOp = iota
	receiverEcho
	receiverGenerateToken // Unused
	receiverGetTokenID    // Unused
	receiverCheckToken    // Unused
	receiverSubmitPayload
	receiverCommit
	receiverError // Unused
)

// Receiver contains the operations that "receiver" worker processes perform
type Receiver interface {
	Quit() error
	Echo() error
	SubmitPayload(leasePath string, payload io.Reader, digest string, headerSize int) error
	Commit(leasePath, oldRootHash, newRootHash string, tag gw.RepositoryTag) error
}

// NewReceiver is the factory method for Receiver types
func NewReceiver(ctx context.Context, execPath string, mock bool) (Receiver, error) {
	if mock {
		return NewMockReceiver(ctx)
	}

	return NewCvmfsReceiver(ctx, execPath)
}

// CvmfsReceiver spawns an external cvmfs_receiver worker process
type CvmfsReceiver struct {
	worker       *exec.Cmd
	workerCmdIn  io.WriteCloser
	workerCmdOut io.ReadCloser
	workerStderr io.ReadCloser
	workerStdout io.ReadCloser
	ctx          context.Context
}

type ReceiverReply struct {
	Status string `json:"status"`
	Reason string `json:"reason"`
}

// NewCvmfsReceiver will spawn an external cvmfs_receiver worker process and wait for a command
func NewCvmfsReceiver(ctx context.Context, execPath string) (*CvmfsReceiver, error) {
	if _, err := os.Stat(execPath); os.IsNotExist(err) {
		return nil, errors.Wrap(err, "worker process executable not found")
	}

	cmd := exec.Command(execPath, "-i", strconv.Itoa(3), "-o", strconv.Itoa(4))

	workerInRead, workerInWrite, err := os.Pipe()
	if err != nil {
		return nil, errors.Wrap(err, "could not create worker input pipe")
	}
	workerOutRead, workerOutWrite, err := os.Pipe()
	if err != nil {
		return nil, errors.Wrap(err, "could not create worker output pipe")
	}

	cmd.ExtraFiles = []*os.File{workerInRead, workerOutWrite}

	stderr, err := cmd.StderrPipe()
	if err != nil {
		return nil, errors.Wrap(err, "could not create worker stderr pipe")
	}
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, errors.Wrap(err, "could not create worker stdout pipe")
	}

	if err := cmd.Start(); err != nil {
		return nil, errors.Wrap(err, "could not start worker process")
	}

	gw.LogC(ctx, "receiver", gw.LogDebug).
		Str("command", "start").
		Msg("worker process ready")

	return &CvmfsReceiver{
		worker: cmd, workerCmdIn: workerInWrite, workerCmdOut: workerOutRead,
		workerStderr: stderr, workerStdout: stdout, ctx: ctx}, nil
}

// Quit command is sent to the worker
func (r *CvmfsReceiver) Quit() error {
	defer func() {
		r.workerCmdIn.Close()
		r.workerCmdOut.Close()
	}()

	if _, err := r.call(receiverQuit, []byte{}, nil); err != nil {
		return errors.Wrap(err, "worker 'quit' call failed")
	}

	var buf1 []byte
	if _, err := io.ReadFull(r.workerStderr, buf1); err != nil {
		return errors.Wrap(err, "could not retrieve worker stderr")
	}
	gw.LogC(r.ctx, "receiver", gw.LogDebug).
		Str("pipe", "stderr").
		Msg(string(buf1))

	var buf2 []byte
	if _, err := io.ReadFull(r.workerStdout, buf2); err != nil {
		return errors.Wrap(err, "could not retrieve worker stdout")
	}
	gw.LogC(r.ctx, "receiver", gw.LogDebug).
		Str("pipe", "stdout").
		Msg(string(buf2))

	if err := r.worker.Wait(); err != nil {
		return errors.Wrap(err, "waiting for worker process failed")
	}

	gw.LogC(r.ctx, "receiver", gw.LogDebug).
		Str("command", "quit").
		Msg("worker process has stopped")

	return nil
}

// Echo command is sent to the worker
func (r *CvmfsReceiver) Echo() error {
	rep, err := r.call(receiverEcho, []byte("Ping"), nil)
	if err != nil {
		return errors.Wrap(err, "worker 'echo' call failed")
	}
	reply := string(rep)

	if !strings.HasPrefix(reply, "PID: ") {
		return fmt.Errorf("invalid 'echo' reply received: %v", reply)
	}

	gw.LogC(r.ctx, "receiver", gw.LogDebug).
		Str("command", "echo").
		Msgf("reply: %v", reply)

	return nil
}

// SubmitPayload command is sent to the worker
func (r *CvmfsReceiver) SubmitPayload(leasePath string, payload io.Reader, digest string, headerSize int) error {
	req := map[string]interface{}{"path": leasePath, "digest": digest, "header_size": headerSize}
	buf, err := json.Marshal(&req)
	if err != nil {
		return errors.Wrap(err, "request encoding failed")
	}
	reply, err := r.call(receiverSubmitPayload, buf, payload)
	if err != nil {
		return errors.Wrap(err, "worker 'payload submission' call failed")
	}

	result := toReceiverError(reply)

	gw.LogC(r.ctx, "receiver", gw.LogDebug).
		Str("command", "submit payload").
		Msgf("result: %v", result)

	return result
}

// Commit command is sent to the worker
func (r *CvmfsReceiver) Commit(leasePath, oldRootHash, newRootHash string, tag gw.RepositoryTag) error {
	req := map[string]interface{}{
		"lease_path":      leasePath,
		"old_root_hash":   oldRootHash,
		"new_root_hash":   newRootHash,
		"tag_name":        tag.Name,
		"tag_channel":     tag.Channel,
		"tag_description": tag.Description,
	}
	buf, err := json.Marshal(&req)
	if err != nil {
		return errors.Wrap(err, "request encoding failed")
	}

	reply, err := r.call(receiverCommit, buf, nil)
	if err != nil {
		return errors.Wrap(err, "worker 'commit' call failed")
	}

	result := toReceiverError(reply)

	gw.LogC(r.ctx, "receiver", gw.LogDebug).
		Str("command", "commit").
		Msgf("result: %v", result)

	return result
}

func (r *CvmfsReceiver) call(reqID receiverOp, msg []byte, payload io.Reader) ([]byte, error) {
	if err := r.request(reqID, msg, payload); err != nil {
		return nil, err
	}
	return r.reply()
}

func (r *CvmfsReceiver) request(reqID receiverOp, msg []byte, payload io.Reader) error {
	buf := make([]byte, 8+len(msg))
	binary.LittleEndian.PutUint32(buf[:4], uint32(reqID))
	binary.LittleEndian.PutUint32(buf[4:8], uint32(len(msg)))
	copy(buf[8:], msg)

	if _, err := r.workerCmdIn.Write(buf); err != nil {
		return errors.Wrap(err, "could not write request")
	}
	if payload != nil {
		if _, err := io.Copy(r.workerCmdIn, payload); err != nil {
			return errors.Wrap(err, "could not write request payload")
		}
	}
	return nil
}

func (r *CvmfsReceiver) reply() ([]byte, error) {
	buf := make([]byte, 4)
	if _, err := io.ReadFull(r.workerCmdOut, buf); err != nil {
		return nil, errors.Wrap(err, "could not read reply size")
	}
	repSize := int32(binary.LittleEndian.Uint32(buf))

	reply := make([]byte, repSize)
	if _, err := io.ReadFull(r.workerCmdOut, reply); err != nil {
		return nil, errors.Wrap(err, "could not read reply body")
	}

	return reply, nil
}

func toReceiverError(reply []byte) error {
	res := ReceiverReply{}
	if err := json.Unmarshal(reply, &res); err != nil {
		return errors.Wrap(err, "could not decode reply")
	}

	if res.Status == "ok" {
		return nil
	}

	if res.Reason != "" {
		return Error(res.Reason)
	}

	return fmt.Errorf("invalid reply")
}
<|endoftext|>


<commit_before>package revel

import (
	"compress/gzip"
	"compress/zlib"
	"net/http"
	"strings"
)

var compressionTypes = [...]string{
	"gzip",
	"deflate",
}

var compressedMimes = [...]string{
	"text/plain",
	"text/html",
	"text/xml",
	"text/css",
	"application/xml",
	"application/xhtml+xml",
	"application/rss+xml",
	"application/javascript",
	"application/x-javascript",
}

type WriteFlusher interface {
	Write([]byte) (int, error)
	Flush() error
}

type CompressResponseWriter struct {
	http.ResponseWriter
	compressWriter  WriteFlusher
	compressionType string
	headersWritten  bool
}

func CompressFilter(c *Controller, fc []Filter) {
	writer := CompressResponseWriter{c.Response.Out, nil, "", false}
	writer.DetectCompressionType(c.Request, c.Response)
	c.Response.Out = &writer

	fc[0](c, fc[1:])
}

func (c *CompressResponseWriter) prepareHeaders() {
	mime := c.Header().Get("Content-Type")
	shouldEncode := false

	for _, mimeType := range compressedMimes {
		if strings.HasPrefix(mime, mimeType) {
			shouldEncode = true
			c.Header().Set("Content-Encoding", c.compressionType)
			c.Header().Del("Content-Length")
			break
		}
	}

	if !shouldEncode {
		c.compressWriter = nil
		c.compressionType = ""
	}
}

func (c *CompressResponseWriter) WriteHeader(status int) {
	c.headersWritten = true
	c.prepareHeaders()
	c.ResponseWriter.WriteHeader(status)
}

func (c *CompressResponseWriter) Write(b []byte) (int, error) {
	if !c.headersWritten {
		c.prepareHeaders()
		c.headersWritten = true
	}

	if c.compressionType != "" {
		defer c.compressWriter.Flush()
		return c.compressWriter.Write(b)
	} else {
		return c.ResponseWriter.Write(b)
	}
}

func (c *CompressResponseWriter) DetectCompressionType(req *Request, resp *Response) {
	if Config.BoolDefault("results.compressed", false) {
		acceptedEncodings := strings.Split(req.Request.Header.Get("Accept-Encoding"), ",")

		for i := 0; i < len(acceptedEncodings) && c.compressionType == ""; i++ {
			switch strings.TrimSpace(acceptedEncodings[i]) {
			case "gzip":
				c.compressionType = "gzip"
				c.compressWriter = gzip.NewWriter(resp.Out)
			case "deflate":
				c.compressionType = "deflate"
				c.compressWriter = zlib.NewWriter(resp.Out)
			}
		}
	}
}
<commit_msg>Renamed variable to be more descriptive.<commit_after>package revel

import (
	"compress/gzip"
	"compress/zlib"
	"net/http"
	"strings"
)

var compressionTypes = [...]string{
	"gzip",
	"deflate",
}

var compressableMimes = [...]string{
	"text/plain",
	"text/html",
	"text/xml",
	"text/css",
	"application/xml",
	"application/xhtml+xml",
	"application/rss+xml",
	"application/javascript",
	"application/x-javascript",
}

type WriteFlusher interface {
	Write([]byte) (int, error)
	Flush() error
}

type CompressResponseWriter struct {
	http.ResponseWriter
	compressWriter  WriteFlusher
	compressionType string
	headersWritten  bool
}

func CompressFilter(c *Controller, fc []Filter) {
	writer := CompressResponseWriter{c.Response.Out, nil, "", false}
	writer.DetectCompressionType(c.Request, c.Response)
	c.Response.Out = &writer

	fc[0](c, fc[1:])
}

func (c *CompressResponseWriter) prepareHeaders() {
	mime := c.Header().Get("Content-Type")
	shouldEncode := false

	for _, mimeType := range compressableMimes {
		if strings.HasPrefix(mime, mimeType) {
			shouldEncode = true
			c.Header().Set("Content-Encoding", c.compressionType)
			c.Header().Del("Content-Length")
			break
		}
	}

	if !shouldEncode {
		c.compressWriter = nil
		c.compressionType = ""
	}
}

func (c *CompressResponseWriter) WriteHeader(status int) {
	c.headersWritten = true
	c.prepareHeaders()
	c.ResponseWriter.WriteHeader(status)
}

func (c *CompressResponseWriter) Write(b []byte) (int, error) {
	if !c.headersWritten {
		c.prepareHeaders()
		c.headersWritten = true
	}

	if c.compressionType != "" {
		defer c.compressWriter.Flush()
		return c.compressWriter.Write(b)
	} else {
		return c.ResponseWriter.Write(b)
	}
}

func (c *CompressResponseWriter) DetectCompressionType(req *Request, resp *Response) {
	if Config.BoolDefault("results.compressed", false) {
		acceptedEncodings := strings.Split(req.Request.Header.Get("Accept-Encoding"), ",")

		for i := 0; i < len(acceptedEncodings) && c.compressionType == ""; i++ {
			switch strings.TrimSpace(acceptedEncodings[i]) {
			case "gzip":
				c.compressionType = "gzip"
				c.compressWriter = gzip.NewWriter(resp.Out)
			case "deflate":
				c.compressionType = "deflate"
				c.compressWriter = zlib.NewWriter(resp.Out)
			}
		}
	}
}
<|endoftext|>


<commit_before>package cli

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/mgutz/ansi"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"text/template"
	"time"
)

func FindParentPaths(fileName string) []string {
	cwd, _ := os.Getwd()

	paths := make([]string, 0)

	// special case if homedir is not in current path then check there anyway
	homedir := os.Getenv("HOME")
	if !strings.HasPrefix(cwd, homedir) {
		file := fmt.Sprintf("%s/%s", homedir, fileName)
		if _, err := os.Stat(file); err == nil {
			paths = append(paths, file)
		}
	}

	var dir string
	for _, part := range strings.Split(cwd, string(os.PathSeparator)) {
		if dir == "/" {
			dir = fmt.Sprintf("/%s", part)
		} else {
			dir = fmt.Sprintf("%s/%s", dir, part)
		}
		file := fmt.Sprintf("%s/%s", dir, fileName)
		if _, err := os.Stat(file); err == nil {
			paths = append(paths, file)
		}
	}
	return paths
}

func FindClosestParentPath(fileName string) (string, error) {
	paths := FindParentPaths(fileName)
	if len(paths) > 0 {
		return paths[len(paths)-1], nil
	}
	return "", errors.New(fmt.Sprintf("%s not found in parent directory hierarchy", fileName))
}

func readFile(file string) string {
	var bytes []byte
	var err error
	if bytes, err = ioutil.ReadFile(file); err != nil {
		log.Error("Failed to read file %s: %s", file, err)
		os.Exit(1)
	}
	return string(bytes)
}

func copyFile(src, dst string) (err error) {
	var s, d *os.File
	if s, err = os.Open(src); err == nil {
		defer s.Close()
		if d, err = os.Create(dst); err == nil {
			if _, err = io.Copy(d, s); err != nil {
				d.Close()
				return
			}
			return d.Close()
		}
	}
	return
}

func fuzzyAge(start string) (string, error) {
	if t, err := time.Parse("2006-01-02T15:04:05.000-0700", start); err != nil {
		return "", err
	} else {
		delta := time.Now().Sub(t)
		if delta.Minutes() < 2 {
			return "a minute", nil
		} else if dm := delta.Minutes(); dm < 45 {
			return fmt.Sprintf("%d minutes", int(dm)), nil
		} else if dm := delta.Minutes(); dm < 90 {
			return "an hour", nil
		} else if dh := delta.Hours(); dh < 24 {
			return fmt.Sprintf("%d hours", int(dh)), nil
		} else if dh := delta.Hours(); dh < 48 {
			return "a day", nil
		} else {
			return fmt.Sprintf("%d days", int(delta.Hours()/24)), nil
		}
	}
	return "unknown", nil
}

func runTemplate(templateContent string, data interface{}, out io.Writer) error {

	if out == nil {
		out = os.Stdout
	}

	funcs := map[string]interface{}{
		"toJson": func(content interface{}) (string, error) {
			if bytes, err := json.MarshalIndent(content, "", "    "); err != nil {
				return "", err
			} else {
				return string(bytes), nil
			}
		},
		"append": func(more string, content interface{}) (string, error) {
			switch value := content.(type) {
			case string:
				return string(append([]byte(content.(string)), []byte(more)...)), nil
			case []byte:
				return string(append(content.([]byte), []byte(more)...)), nil
			default:
				return "", errors.New(fmt.Sprintf("Unknown type: %s", value))
			}
		},
		"indent": func(spaces int, content string) string {
			indent := make([]byte, spaces+1, spaces+1)
			indent[0] = '\n'
			for i := 1; i < spaces+1; i += 1 {
				indent[i] = ' '
			}
			return strings.Replace(content, "\n", string(indent), -1)
		},
		"color": func(color string) string {
			return ansi.ColorCode(color)
		},
		"split": func(sep string, content string) []string {
			return strings.Split(content, sep)
		},
		"abbrev": func(max int, content string) string {
			if len(content) > max {
				var buffer bytes.Buffer
				buffer.WriteString(content[:max-3])
				buffer.WriteString("...")
				return buffer.String()
			}
			return content
		},
		"rep": func(count int, content string) string {
			var buffer bytes.Buffer
			for i := 0; i < count; i += 1 {
				buffer.WriteString(content)
			}
			return buffer.String()
		},
		"age": func(content string) (string, error) {
			return fuzzyAge(content)
		},
	}
	if tmpl, err := template.New("template").Funcs(funcs).Parse(templateContent); err != nil {
		log.Error("Failed to parse template: %s", err)
		return err
	} else {
		if err := tmpl.Execute(out, data); err != nil {
			log.Error("Failed to execute template: %s", err)
			return err
		}
	}
	return nil
}

func responseToJson(resp *http.Response, err error) (interface{}, error) {
	if err != nil {
		return nil, err
	}

	data := jsonDecode(resp.Body)
	if resp.StatusCode == 400 {
		if val, ok := data.(map[string]interface{})["errorMessages"]; ok {
			for _, errMsg := range val.([]interface{}) {
				log.Error("%s", errMsg)
			}
		}
	}

	return data, nil
}

func jsonDecode(io io.Reader) interface{} {
	content, err := ioutil.ReadAll(io)
	var data interface{}
	err = json.Unmarshal(content, &data)
	if err != nil {
		log.Error("JSON Parse Error: %s from %s", err, content)
	}
	return data
}

func jsonEncode(data interface{}) (string, error) {
	buffer := bytes.NewBuffer(make([]byte, 0))
	enc := json.NewEncoder(buffer)

	err := enc.Encode(data)
	if err != nil {
		log.Error("Failed to encode data %s: %s", data, err)
		return "", err
	}
	return buffer.String(), nil
}

func jsonWrite(file string, data interface{}) {
	fh, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	defer fh.Close()
	if err != nil {
		log.Error("Failed to open %s: %s", file, err)
		os.Exit(1)
	}
	enc := json.NewEncoder(fh)
	enc.Encode(data)
}

func promptYN(prompt string, yes bool) bool {
	reader := bufio.NewReader(os.Stdin)
	if !yes {
		prompt = fmt.Sprintf("%s [y/N]: ", prompt)
	} else {
		prompt = fmt.Sprintf("%s [Y/n]: ", prompt)
	}

	fmt.Printf("%s", prompt)
	text, _ := reader.ReadString('\n')
	ans := strings.ToLower(strings.TrimRight(text, "\n"))
	if ans == "" {
		return yes
	}
	if strings.HasPrefix(ans, "y") {
		return true
	}
	return false
}

func yamlFixup(data interface{}) (interface{}, error) {
	switch d := data.(type) {
	case map[interface{}]interface{}:
		// need to copy this map into a string map so json can encode it
		copy := make(map[string]interface{})
		for key, val := range d {
			switch k := key.(type) {
			case string:
				if fixed, err := yamlFixup(val); err != nil {
					return nil, err
				} else if fixed != nil {
					copy[k] = fixed
				}
			default:
				err := fmt.Errorf("YAML: key %s is type '%T', require 'string'", key, k)
				log.Error("%s", err)
				return nil, err
			}
		}
		return copy, nil
	case map[string]interface{}:
		copy := make(map[string]interface{})
		for k, v := range d {
			if fixed, err := yamlFixup(v); err != nil {
				return nil, err
			} else if fixed != nil {
				copy[k] = fixed
			}
		}
		return copy, nil
	case []interface{}:
		copy := make([]interface{}, 0, len(d))
		for _, val := range d {
			if fixed, err := yamlFixup(val); err != nil {
				return nil, err
			} else if fixed != nil {
				copy = append(copy, fixed)
			}
		}
		return copy, nil
	case string:
		if d == "" {
			return nil, nil
		}
		return d, nil
	default:
		return d, nil
	}
}

func mkdir(dir string) error {
	if stat, err := os.Stat(dir); err != nil && !os.IsNotExist(err) {
		log.Error("Failed to stat %s: %s", dir, err)
		return err
	} else if err == nil && !stat.IsDir() {
		err := fmt.Errorf("%s exists and is not a directory!", dir)
		log.Error("%s", err)
		return err
	} else {
		// dir does not exist, so try to create it
		if err := os.MkdirAll(dir, 0755); err != nil {
			log.Error("Failed to mkdir -p %s: %s", dir, err)
			return err
		}
	}
	return nil
}
<commit_msg>add "comment" template filter to comment out multiline statements<commit_after>package cli

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/mgutz/ansi"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"text/template"
	"time"
)

func FindParentPaths(fileName string) []string {
	cwd, _ := os.Getwd()

	paths := make([]string, 0)

	// special case if homedir is not in current path then check there anyway
	homedir := os.Getenv("HOME")
	if !strings.HasPrefix(cwd, homedir) {
		file := fmt.Sprintf("%s/%s", homedir, fileName)
		if _, err := os.Stat(file); err == nil {
			paths = append(paths, file)
		}
	}

	var dir string
	for _, part := range strings.Split(cwd, string(os.PathSeparator)) {
		if dir == "/" {
			dir = fmt.Sprintf("/%s", part)
		} else {
			dir = fmt.Sprintf("%s/%s", dir, part)
		}
		file := fmt.Sprintf("%s/%s", dir, fileName)
		if _, err := os.Stat(file); err == nil {
			paths = append(paths, file)
		}
	}
	return paths
}

func FindClosestParentPath(fileName string) (string, error) {
	paths := FindParentPaths(fileName)
	if len(paths) > 0 {
		return paths[len(paths)-1], nil
	}
	return "", errors.New(fmt.Sprintf("%s not found in parent directory hierarchy", fileName))
}

func readFile(file string) string {
	var bytes []byte
	var err error
	if bytes, err = ioutil.ReadFile(file); err != nil {
		log.Error("Failed to read file %s: %s", file, err)
		os.Exit(1)
	}
	return string(bytes)
}

func copyFile(src, dst string) (err error) {
	var s, d *os.File
	if s, err = os.Open(src); err == nil {
		defer s.Close()
		if d, err = os.Create(dst); err == nil {
			if _, err = io.Copy(d, s); err != nil {
				d.Close()
				return
			}
			return d.Close()
		}
	}
	return
}

func fuzzyAge(start string) (string, error) {
	if t, err := time.Parse("2006-01-02T15:04:05.000-0700", start); err != nil {
		return "", err
	} else {
		delta := time.Now().Sub(t)
		if delta.Minutes() < 2 {
			return "a minute", nil
		} else if dm := delta.Minutes(); dm < 45 {
			return fmt.Sprintf("%d minutes", int(dm)), nil
		} else if dm := delta.Minutes(); dm < 90 {
			return "an hour", nil
		} else if dh := delta.Hours(); dh < 24 {
			return fmt.Sprintf("%d hours", int(dh)), nil
		} else if dh := delta.Hours(); dh < 48 {
			return "a day", nil
		} else {
			return fmt.Sprintf("%d days", int(delta.Hours()/24)), nil
		}
	}
	return "unknown", nil
}

func runTemplate(templateContent string, data interface{}, out io.Writer) error {

	if out == nil {
		out = os.Stdout
	}

	funcs := map[string]interface{}{
		"toJson": func(content interface{}) (string, error) {
			if bytes, err := json.MarshalIndent(content, "", "    "); err != nil {
				return "", err
			} else {
				return string(bytes), nil
			}
		},
		"append": func(more string, content interface{}) (string, error) {
			switch value := content.(type) {
			case string:
				return string(append([]byte(content.(string)), []byte(more)...)), nil
			case []byte:
				return string(append(content.([]byte), []byte(more)...)), nil
			default:
				return "", errors.New(fmt.Sprintf("Unknown type: %s", value))
			}
		},
		"indent": func(spaces int, content string) string {
			indent := make([]byte, spaces+1, spaces+1)
			indent[0] = '\n'
			for i := 1; i < spaces+1; i += 1 {
				indent[i] = ' '
			}
			return strings.Replace(content, "\n", string(indent), -1)
		},
		"comment": func(content string) string {
			return strings.Replace(content, "\n", "\n# ", -1)
		},
		"color": func(color string) string {
			return ansi.ColorCode(color)
		},
		"split": func(sep string, content string) []string {
			return strings.Split(content, sep)
		},
		"abbrev": func(max int, content string) string {
			if len(content) > max {
				var buffer bytes.Buffer
				buffer.WriteString(content[:max-3])
				buffer.WriteString("...")
				return buffer.String()
			}
			return content
		},
		"rep": func(count int, content string) string {
			var buffer bytes.Buffer
			for i := 0; i < count; i += 1 {
				buffer.WriteString(content)
			}
			return buffer.String()
		},
		"age": func(content string) (string, error) {
			return fuzzyAge(content)
		},
	}
	if tmpl, err := template.New("template").Funcs(funcs).Parse(templateContent); err != nil {
		log.Error("Failed to parse template: %s", err)
		return err
	} else {
		if err := tmpl.Execute(out, data); err != nil {
			log.Error("Failed to execute template: %s", err)
			return err
		}
	}
	return nil
}

func responseToJson(resp *http.Response, err error) (interface{}, error) {
	if err != nil {
		return nil, err
	}

	data := jsonDecode(resp.Body)
	if resp.StatusCode == 400 {
		if val, ok := data.(map[string]interface{})["errorMessages"]; ok {
			for _, errMsg := range val.([]interface{}) {
				log.Error("%s", errMsg)
			}
		}
	}

	return data, nil
}

func jsonDecode(io io.Reader) interface{} {
	content, err := ioutil.ReadAll(io)
	var data interface{}
	err = json.Unmarshal(content, &data)
	if err != nil {
		log.Error("JSON Parse Error: %s from %s", err, content)
	}
	return data
}

func jsonEncode(data interface{}) (string, error) {
	buffer := bytes.NewBuffer(make([]byte, 0))
	enc := json.NewEncoder(buffer)

	err := enc.Encode(data)
	if err != nil {
		log.Error("Failed to encode data %s: %s", data, err)
		return "", err
	}
	return buffer.String(), nil
}

func jsonWrite(file string, data interface{}) {
	fh, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	defer fh.Close()
	if err != nil {
		log.Error("Failed to open %s: %s", file, err)
		os.Exit(1)
	}
	enc := json.NewEncoder(fh)
	enc.Encode(data)
}

func promptYN(prompt string, yes bool) bool {
	reader := bufio.NewReader(os.Stdin)
	if !yes {
		prompt = fmt.Sprintf("%s [y/N]: ", prompt)
	} else {
		prompt = fmt.Sprintf("%s [Y/n]: ", prompt)
	}

	fmt.Printf("%s", prompt)
	text, _ := reader.ReadString('\n')
	ans := strings.ToLower(strings.TrimRight(text, "\n"))
	if ans == "" {
		return yes
	}
	if strings.HasPrefix(ans, "y") {
		return true
	}
	return false
}

func yamlFixup(data interface{}) (interface{}, error) {
	switch d := data.(type) {
	case map[interface{}]interface{}:
		// need to copy this map into a string map so json can encode it
		copy := make(map[string]interface{})
		for key, val := range d {
			switch k := key.(type) {
			case string:
				if fixed, err := yamlFixup(val); err != nil {
					return nil, err
				} else if fixed != nil {
					copy[k] = fixed
				}
			default:
				err := fmt.Errorf("YAML: key %s is type '%T', require 'string'", key, k)
				log.Error("%s", err)
				return nil, err
			}
		}
		return copy, nil
	case map[string]interface{}:
		copy := make(map[string]interface{})
		for k, v := range d {
			if fixed, err := yamlFixup(v); err != nil {
				return nil, err
			} else if fixed != nil {
				copy[k] = fixed
			}
		}
		return copy, nil
	case []interface{}:
		copy := make([]interface{}, 0, len(d))
		for _, val := range d {
			if fixed, err := yamlFixup(val); err != nil {
				return nil, err
			} else if fixed != nil {
				copy = append(copy, fixed)
			}
		}
		return copy, nil
	case string:
		if d == "" {
			return nil, nil
		}
		return d, nil
	default:
		return d, nil
	}
}

func mkdir(dir string) error {
	if stat, err := os.Stat(dir); err != nil && !os.IsNotExist(err) {
		log.Error("Failed to stat %s: %s", dir, err)
		return err
	} else if err == nil && !stat.IsDir() {
		err := fmt.Errorf("%s exists and is not a directory!", dir)
		log.Error("%s", err)
		return err
	} else {
		// dir does not exist, so try to create it
		if err := os.MkdirAll(dir, 0755); err != nil {
			log.Error("Failed to mkdir -p %s: %s", dir, err)
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 The Serulian Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package typeconstructor

import (
	"fmt"

	"github.com/serulian/compiler/compilercommon"
	"github.com/serulian/compiler/compilergraph"
	"github.com/serulian/compiler/graphs/typegraph"
	"github.com/serulian/compiler/webidl"
)

// GLOBAL_CONTEXT_ANNOTATION is the annotation that marks an interface as being the global context
// (e.g. Window) in WebIDL.
const GLOBAL_CONTEXT_ANNOTATION = "GlobalContext"

// CONSTRUCTOR_ANNOTATION is an annotation that describes support for a constructor on a WebIDL
// type. This translates to being able to do "new Type(...)" in ECMAScript.
const CONSTRUCTOR_ANNOTATION = "Constructor"

// NATIVE_OPERATOR_ANNOTATION is an annotation that marks an declaration as supporting the
// specified operator natively (i.e. not a custom defined operator).
const NATIVE_OPERATOR_ANNOTATION = "NativeOperator"

// GetConstructor returns a TypeGraph constructor for the given IRG.
func GetConstructor(irg *webidl.WebIRG) *irgTypeConstructor {
	return &irgTypeConstructor{
		irg: irg,
	}
}

// irgTypeConstructor defines a type for populating a type graph from the IRG.
type irgTypeConstructor struct {
	irg *webidl.WebIRG // The IRG being transformed.
}

func (itc *irgTypeConstructor) DefineModules(builder typegraph.GetModuleBuilder) {
	for _, module := range itc.irg.GetModules() {
		builder().
			Name(module.Name()).
			Path(string(module.InputSource())).
			SourceNode(module.Node()).
			Define()
	}
}

func (itc *irgTypeConstructor) DefineTypes(builder typegraph.GetTypeBuilder) {
	for _, module := range itc.irg.GetModules() {
		for _, declaration := range module.Declarations() {
			if declaration.HasAnnotation(GLOBAL_CONTEXT_ANNOTATION) {
				continue
			}

			typeBuilder := builder(module.Node())
			typeBuilder.Name(declaration.Name()).
				SourceNode(declaration.GraphNode).
				TypeKind(typegraph.ExternalInternalType).
				Define()
		}
	}
}

func (itc *irgTypeConstructor) DefineDependencies(annotator *typegraph.Annotator, graph *typegraph.TypeGraph) {

}

func (itc *irgTypeConstructor) DefineMembers(builder typegraph.GetMemberBuilder, reporter typegraph.IssueReporter, graph *typegraph.TypeGraph) {
	for _, declaration := range itc.irg.Declarations() {
		// GlobalContext members get defined under their module, not their declaration.
		var parentNode = declaration.GraphNode
		if declaration.HasAnnotation(GLOBAL_CONTEXT_ANNOTATION) {
			parentNode = declaration.Module().GraphNode
		}

		// If the declaration has one (or more) constructors, add then as a "new".
		if declaration.HasAnnotation(CONSTRUCTOR_ANNOTATION) {
			// For each constructor defined, create the intersection of their parameters.
			var parameters = make([]typegraph.TypeReference, 0)
			for constructorIndex, constructor := range declaration.GetAnnotations(CONSTRUCTOR_ANNOTATION) {
				for index, parameter := range constructor.Parameters() {
					parameterType, err := itc.ResolveType(parameter.DeclaredType(), graph)
					if err != nil {
						reporter.ReportError(parameter.GraphNode, "%v", err)
						continue
					}

					var resolvedParameterType = parameterType
					if parameter.IsOptional() {
						resolvedParameterType = resolvedParameterType.AsNullable()
					}

					if index >= len(parameters) {
						// If this is not the first constructor, then this parameter is implicitly optional
						// and therefore nullable.
						if constructorIndex > 0 {
							resolvedParameterType = resolvedParameterType.AsNullable()
						}

						parameters = append(parameters, resolvedParameterType)
					} else {
						parameters[index] = parameters[index].Intersect(resolvedParameterType)
					}
				}
			}

			// Define the construction function for the type.
			typeDecl, _ := graph.GetTypeForSourceNode(declaration.GraphNode)
			var constructorFunction = graph.FunctionTypeReference(typeDecl.GetTypeReference())
			for _, parameterType := range parameters {
				constructorFunction = constructorFunction.WithParameter(parameterType)
			}

			// Declare a "new" member which returns an instance of this type.
			builder(parentNode, false).
				Name("new").
				InitialDefine().
				Exported(true).
				Static(true).
				ReadOnly(true).
				Synchronous(true).
				MemberKind(uint64(webidl.ConstructorMember)).
				MemberType(constructorFunction).
				Define()
		}

		// Add support for any native operators.
		if declaration.HasAnnotation(GLOBAL_CONTEXT_ANNOTATION) && declaration.HasAnnotation(NATIVE_OPERATOR_ANNOTATION) {
			reporter.ReportError(declaration.GraphNode, "[NativeOperator] not supported on declarations marked with [GlobalContext]")
			return
		}

		for _, nativeOp := range declaration.GetAnnotations(NATIVE_OPERATOR_ANNOTATION) {
			opName, hasOpName := nativeOp.Value()
			if !hasOpName {
				reporter.ReportError(nativeOp.GraphNode, "Missing operator name on [NativeOperator] annotation")
				continue
			}

			typeDecl, _ := graph.GetTypeForSourceNode(declaration.GraphNode)

			var operatorType = graph.FunctionTypeReference(typeDecl.GetTypeReference())
			operatorType = operatorType.WithParameter(typeDecl.GetTypeReference())
			operatorType = operatorType.WithParameter(typeDecl.GetTypeReference())

			builder(parentNode, true).
				Name(opName).
				SourceNode(nativeOp.GraphNode).
				InitialDefine().
				Native(true).
				Exported(true).
				Static(true).
				ReadOnly(true).
				MemberType(operatorType).
				MemberKind(uint64(webidl.OperatorMember)).
				Define()
		}

		// Add the declared members.
		for _, member := range declaration.Members() {
			ibuilder := builder(parentNode, false).
				Name(member.Name()).
				SourceNode(member.GraphNode).
				InitialDefine()

			declaredType, err := itc.ResolveType(member.DeclaredType(), graph)
			if err != nil {
				reporter.ReportError(member.GraphNode, "%v", err)
				continue
			}

			var memberType = declaredType
			var isReadonly = member.IsReadonly()

			switch member.Kind() {
			case webidl.FunctionMember:
				isReadonly = true
				memberType = graph.FunctionTypeReference(memberType)

				// Add the parameter types.
				var markOptional = false
				for _, parameter := range member.Parameters() {
					if parameter.IsOptional() {
						markOptional = true
					}

					parameterType, err := itc.ResolveType(parameter.DeclaredType(), graph)
					if err != nil {
						reporter.ReportError(member.GraphNode, "%v", err)
						continue
					}

					// All optional parameters get marked as nullable, which means we can skip
					// passing them on function calls.
					if markOptional {
						memberType = memberType.WithParameter(parameterType.AsNullable())
					} else {
						memberType = memberType.WithParameter(parameterType)
					}
				}

			case webidl.AttributeMember:
				if len(member.Parameters()) > 0 {
					reporter.ReportError(member.GraphNode, "Attributes cannot have parameters")
				}

			default:
				panic("Unknown WebIDL member kind")
			}

			ibuilder.Exported(true).
				Static(member.IsStatic()).
				Synchronous(true).
				ReadOnly(isReadonly).
				MemberKind(uint64(member.Kind())).
				MemberType(memberType).
				Define()
		}
	}
}

func (itc *irgTypeConstructor) Validate(reporter typegraph.IssueReporter, graph *typegraph.TypeGraph) {
	seen := map[string]bool{}

	for _, module := range itc.irg.GetModules() {
		for _, declaration := range module.Declarations() {
			if _, ok := seen[declaration.Name()]; ok {
				reporter.ReportError(declaration.GraphNode, "'%s' is already declared in WebIDL", declaration.Name())
			}
			seen[declaration.Name()] = true
		}
	}
}

func (itc *irgTypeConstructor) GetLocation(sourceNodeId compilergraph.GraphNodeId) (compilercommon.SourceAndLocation, bool) {
	layerNode, found := itc.irg.TryGetNode(sourceNodeId)
	if !found {
		return compilercommon.SourceAndLocation{}, false
	}

	return itc.irg.NodeLocation(layerNode), true
}

// ResolveType attempts to resolve the given type string.
func (itc *irgTypeConstructor) ResolveType(typeString string, graph *typegraph.TypeGraph) (typegraph.TypeReference, error) {
	if typeString == "any" {
		return graph.AnyTypeReference(), nil
	}

	if typeString == "void" {
		return graph.VoidTypeReference(), nil
	}

	declaration, hasDeclaration := itc.irg.FindDeclaration(typeString)
	if !hasDeclaration {
		return graph.AnyTypeReference(), fmt.Errorf("Could not find WebIDL type %v", typeString)
	}

	typeDecl, hasType := graph.GetTypeForSourceNode(declaration.GraphNode)
	if !hasType {
		panic("Type not found for WebIDL type declaration")
	}

	return typeDecl.GetTypeReference(), nil
}
<commit_msg>Fix handling of the one unary operator in WebIDL type constructor<commit_after>// Copyright 2015 The Serulian Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package typeconstructor

import (
	"fmt"
	"strings"

	"github.com/serulian/compiler/compilercommon"
	"github.com/serulian/compiler/compilergraph"
	"github.com/serulian/compiler/graphs/typegraph"
	"github.com/serulian/compiler/webidl"
)

// GLOBAL_CONTEXT_ANNOTATION is the annotation that marks an interface as being the global context
// (e.g. Window) in WebIDL.
const GLOBAL_CONTEXT_ANNOTATION = "GlobalContext"

// CONSTRUCTOR_ANNOTATION is an annotation that describes support for a constructor on a WebIDL
// type. This translates to being able to do "new Type(...)" in ECMAScript.
const CONSTRUCTOR_ANNOTATION = "Constructor"

// NATIVE_OPERATOR_ANNOTATION is an annotation that marks an declaration as supporting the
// specified operator natively (i.e. not a custom defined operator).
const NATIVE_OPERATOR_ANNOTATION = "NativeOperator"

// GetConstructor returns a TypeGraph constructor for the given IRG.
func GetConstructor(irg *webidl.WebIRG) *irgTypeConstructor {
	return &irgTypeConstructor{
		irg: irg,
	}
}

// irgTypeConstructor defines a type for populating a type graph from the IRG.
type irgTypeConstructor struct {
	irg *webidl.WebIRG // The IRG being transformed.
}

func (itc *irgTypeConstructor) DefineModules(builder typegraph.GetModuleBuilder) {
	for _, module := range itc.irg.GetModules() {
		builder().
			Name(module.Name()).
			Path(string(module.InputSource())).
			SourceNode(module.Node()).
			Define()
	}
}

func (itc *irgTypeConstructor) DefineTypes(builder typegraph.GetTypeBuilder) {
	for _, module := range itc.irg.GetModules() {
		for _, declaration := range module.Declarations() {
			if declaration.HasAnnotation(GLOBAL_CONTEXT_ANNOTATION) {
				continue
			}

			typeBuilder := builder(module.Node())
			typeBuilder.Name(declaration.Name()).
				SourceNode(declaration.GraphNode).
				TypeKind(typegraph.ExternalInternalType).
				Define()
		}
	}
}

func (itc *irgTypeConstructor) DefineDependencies(annotator *typegraph.Annotator, graph *typegraph.TypeGraph) {

}

func (itc *irgTypeConstructor) DefineMembers(builder typegraph.GetMemberBuilder, reporter typegraph.IssueReporter, graph *typegraph.TypeGraph) {
	for _, declaration := range itc.irg.Declarations() {
		// GlobalContext members get defined under their module, not their declaration.
		var parentNode = declaration.GraphNode
		if declaration.HasAnnotation(GLOBAL_CONTEXT_ANNOTATION) {
			parentNode = declaration.Module().GraphNode
		}

		// If the declaration has one (or more) constructors, add then as a "new".
		if declaration.HasAnnotation(CONSTRUCTOR_ANNOTATION) {
			// For each constructor defined, create the intersection of their parameters.
			var parameters = make([]typegraph.TypeReference, 0)
			for constructorIndex, constructor := range declaration.GetAnnotations(CONSTRUCTOR_ANNOTATION) {
				for index, parameter := range constructor.Parameters() {
					parameterType, err := itc.ResolveType(parameter.DeclaredType(), graph)
					if err != nil {
						reporter.ReportError(parameter.GraphNode, "%v", err)
						continue
					}

					var resolvedParameterType = parameterType
					if parameter.IsOptional() {
						resolvedParameterType = resolvedParameterType.AsNullable()
					}

					if index >= len(parameters) {
						// If this is not the first constructor, then this parameter is implicitly optional
						// and therefore nullable.
						if constructorIndex > 0 {
							resolvedParameterType = resolvedParameterType.AsNullable()
						}

						parameters = append(parameters, resolvedParameterType)
					} else {
						parameters[index] = parameters[index].Intersect(resolvedParameterType)
					}
				}
			}

			// Define the construction function for the type.
			typeDecl, _ := graph.GetTypeForSourceNode(declaration.GraphNode)
			var constructorFunction = graph.FunctionTypeReference(typeDecl.GetTypeReference())
			for _, parameterType := range parameters {
				constructorFunction = constructorFunction.WithParameter(parameterType)
			}

			// Declare a "new" member which returns an instance of this type.
			builder(parentNode, false).
				Name("new").
				InitialDefine().
				Exported(true).
				Static(true).
				ReadOnly(true).
				Synchronous(true).
				MemberKind(uint64(webidl.ConstructorMember)).
				MemberType(constructorFunction).
				Define()
		}

		// Add support for any native operators.
		if declaration.HasAnnotation(GLOBAL_CONTEXT_ANNOTATION) && declaration.HasAnnotation(NATIVE_OPERATOR_ANNOTATION) {
			reporter.ReportError(declaration.GraphNode, "[NativeOperator] not supported on declarations marked with [GlobalContext]")
			return
		}

		for _, nativeOp := range declaration.GetAnnotations(NATIVE_OPERATOR_ANNOTATION) {
			opName, hasOpName := nativeOp.Value()
			if !hasOpName {
				reporter.ReportError(nativeOp.GraphNode, "Missing operator name on [NativeOperator] annotation")
				continue
			}

			typeDecl, _ := graph.GetTypeForSourceNode(declaration.GraphNode)

			var operatorType = graph.FunctionTypeReference(typeDecl.GetTypeReference())
			operatorType = operatorType.WithParameter(typeDecl.GetTypeReference())

			if strings.Lower(opName) != "not" {
				operatorType = operatorType.WithParameter(typeDecl.GetTypeReference())
			}

			builder(parentNode, true).
				Name(opName).
				SourceNode(nativeOp.GraphNode).
				InitialDefine().
				Native(true).
				Exported(true).
				Static(true).
				ReadOnly(true).
				MemberType(operatorType).
				MemberKind(uint64(webidl.OperatorMember)).
				Define()
		}

		// Add the declared members.
		for _, member := range declaration.Members() {
			ibuilder := builder(parentNode, false).
				Name(member.Name()).
				SourceNode(member.GraphNode).
				InitialDefine()

			declaredType, err := itc.ResolveType(member.DeclaredType(), graph)
			if err != nil {
				reporter.ReportError(member.GraphNode, "%v", err)
				continue
			}

			var memberType = declaredType
			var isReadonly = member.IsReadonly()

			switch member.Kind() {
			case webidl.FunctionMember:
				isReadonly = true
				memberType = graph.FunctionTypeReference(memberType)

				// Add the parameter types.
				var markOptional = false
				for _, parameter := range member.Parameters() {
					if parameter.IsOptional() {
						markOptional = true
					}

					parameterType, err := itc.ResolveType(parameter.DeclaredType(), graph)
					if err != nil {
						reporter.ReportError(member.GraphNode, "%v", err)
						continue
					}

					// All optional parameters get marked as nullable, which means we can skip
					// passing them on function calls.
					if markOptional {
						memberType = memberType.WithParameter(parameterType.AsNullable())
					} else {
						memberType = memberType.WithParameter(parameterType)
					}
				}

			case webidl.AttributeMember:
				if len(member.Parameters()) > 0 {
					reporter.ReportError(member.GraphNode, "Attributes cannot have parameters")
				}

			default:
				panic("Unknown WebIDL member kind")
			}

			ibuilder.Exported(true).
				Static(member.IsStatic()).
				Synchronous(true).
				ReadOnly(isReadonly).
				MemberKind(uint64(member.Kind())).
				MemberType(memberType).
				Define()
		}
	}
}

func (itc *irgTypeConstructor) Validate(reporter typegraph.IssueReporter, graph *typegraph.TypeGraph) {
	seen := map[string]bool{}

	for _, module := range itc.irg.GetModules() {
		for _, declaration := range module.Declarations() {
			if _, ok := seen[declaration.Name()]; ok {
				reporter.ReportError(declaration.GraphNode, "'%s' is already declared in WebIDL", declaration.Name())
			}
			seen[declaration.Name()] = true
		}
	}
}

func (itc *irgTypeConstructor) GetLocation(sourceNodeId compilergraph.GraphNodeId) (compilercommon.SourceAndLocation, bool) {
	layerNode, found := itc.irg.TryGetNode(sourceNodeId)
	if !found {
		return compilercommon.SourceAndLocation{}, false
	}

	return itc.irg.NodeLocation(layerNode), true
}

// ResolveType attempts to resolve the given type string.
func (itc *irgTypeConstructor) ResolveType(typeString string, graph *typegraph.TypeGraph) (typegraph.TypeReference, error) {
	if typeString == "any" {
		return graph.AnyTypeReference(), nil
	}

	if typeString == "void" {
		return graph.VoidTypeReference(), nil
	}

	declaration, hasDeclaration := itc.irg.FindDeclaration(typeString)
	if !hasDeclaration {
		return graph.AnyTypeReference(), fmt.Errorf("Could not find WebIDL type %v", typeString)
	}

	typeDecl, hasType := graph.GetTypeForSourceNode(declaration.GraphNode)
	if !hasType {
		panic("Type not found for WebIDL type declaration")
	}

	return typeDecl.GetTypeReference(), nil
}
<|endoftext|>