<commit_before>package main

import (
    "archive/zip"
    "flag"
    "io"
    "log"
    "net/http"
    "os"
    "path/filepath"
)

func main() {
    rawUrl := flag.String("url", "", "download zip file")
    flag.Parse()
    log.Println("Downloading ", *rawUrl)

    Download(*rawUrl)
    destPath := Unzip("/tmp/downloaded.zip", "/")
    log.Println("Unzipped ", *destPath)
    os.Symlink(destPath, "/data")

    http.Handle("/", http.FileServer(http.Dir("/data")))

    log.Println("Serving /data on HTTP port 8080")
    log.Fatal(http.ListenAndServe(":8080", nil))
}

func Download(rawUrl string) {
    os.MkdirAll("/tmp", 0755)
    fileName := "/tmp/downloaded.zip"

    file, err := os.Create(fileName)
    if err != nil {
     log.Println(err)
     panic(err)
    }
    defer file.Close()

    check := http.Client{
        CheckRedirect: func(r *http.Request, via []*http.Request) error {
            r.URL.Opaque = r.URL.Path
            return nil
        },
    }

    resp, err := check.Get(rawUrl)
    if err != nil {
        log.Println(err)
        panic(err)
    }
    defer resp.Body.Close()

    _, err = io.Copy(file, resp.Body)
    if err != nil {
        panic(err)
    }
}

func Unzip(src, dest string) *string {
    r, err := zip.OpenReader(src)
    if err != nil {
        panic(err)
    }
    defer func() {
        if err := r.Close(); err != nil {
            panic(err)
        }
    }()

    // Closure to address file descriptors issue with all the deferred .Close() methods
    extractAndWriteFile := func(f *zip.File) error {
        rc, err := f.Open()
        if err != nil {
            return err
        }
        defer func() {
            if err := rc.Close(); err != nil {
                panic(err)
            }
        }()

        path := filepath.Join(dest, f.Name)

        if f.FileInfo().IsDir() {
            os.MkdirAll(path, f.Mode())
        } else {
            f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
            if err != nil {
                return err
            }
            defer func() {
                if err := f.Close(); err != nil {
                    panic(err)
                }
            }()

            _, err = io.Copy(f, rc)
            if err != nil {
                return err
            }
        }
        return nil
    }

    for _, f := range r.File {
        err := extractAndWriteFile(f)
        if err != nil {
            panic(err)
        }
    }

    result := filepath.Join(dest, r.File[0].Name)
    return result
}
<commit_msg>fix result 2<commit_after>package main

import (
    "archive/zip"
    "flag"
    "io"
    "log"
    "net/http"
    "os"
    "path/filepath"
)

func main() {
    rawUrl := flag.String("url", "", "download zip file")
    flag.Parse()
    log.Println("Downloading ", *rawUrl)

    Download(*rawUrl)
    destPath := Unzip("/tmp/downloaded.zip", "/")
    log.Println("Unzipped ", *destPath)
    os.Symlink(*destPath, "/data")

    http.Handle("/", http.FileServer(http.Dir("/data")))

    log.Println("Serving /data on HTTP port 8080")
    log.Fatal(http.ListenAndServe(":8080", nil))
}

func Download(rawUrl string) {
    os.MkdirAll("/tmp", 0755)
    fileName := "/tmp/downloaded.zip"

    file, err := os.Create(fileName)
    if err != nil {
     log.Println(err)
     panic(err)
    }
    defer file.Close()

    check := http.Client{
        CheckRedirect: func(r *http.Request, via []*http.Request) error {
            r.URL.Opaque = r.URL.Path
            return nil
        },
    }

    resp, err := check.Get(rawUrl)
    if err != nil {
        log.Println(err)
        panic(err)
    }
    defer resp.Body.Close()

    _, err = io.Copy(file, resp.Body)
    if err != nil {
        panic(err)
    }
}

func Unzip(src, dest string) *string {
    r, err := zip.OpenReader(src)
    if err != nil {
        panic(err)
    }
    defer func() {
        if err := r.Close(); err != nil {
            panic(err)
        }
    }()

    // Closure to address file descriptors issue with all the deferred .Close() methods
    extractAndWriteFile := func(f *zip.File) error {
        rc, err := f.Open()
        if err != nil {
            return err
        }
        defer func() {
            if err := rc.Close(); err != nil {
                panic(err)
            }
        }()

        path := filepath.Join(dest, f.Name)

        if f.FileInfo().IsDir() {
            os.MkdirAll(path, f.Mode())
        } else {
            f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
            if err != nil {
                return err
            }
            defer func() {
                if err := f.Close(); err != nil {
                    panic(err)
                }
            }()

            _, err = io.Copy(f, rc)
            if err != nil {
                return err
            }
        }
        return nil
    }

    for _, f := range r.File {
        err := extractAndWriteFile(f)
        if err != nil {
            panic(err)
        }
    }

    result := filepath.Join(dest, r.File[0].Name)
    return *result
}
<|endoftext|>


<commit_before>package tarantool

import (
	"github.com/tinylib/msgp/msgp"
)

type Upsert struct {
	Space interface{}
	Tuple []interface{}
	Set   []Operator
}

var _ Query = (*Upsert)(nil)

func (q *Upsert) GetCommandID() uint {
	return UpsertCommand
}

func (q *Upsert) packMsg(data *packData, b []byte) (o []byte, err error) {
	o = b
	o = msgp.AppendMapHeader(o, 3)

	if o, err = data.packSpace(q.Space, o); err != nil {
		return o, err
	}

	o = msgp.AppendUint(o, KeyTuple)
	if o, err = msgp.AppendIntf(o, q.Tuple); err != nil {
		return o, err
	}

	o = msgp.AppendUint(o, KeyDefTuple)
	o = msgp.AppendArrayHeader(o, uint32(len(q.Set)))
	for _, op := range q.Set {
		if o, err = msgp.AppendIntf(o, op.AsTuple()); err != nil {
			return o, err
		}
	}

	return o, nil
}

// MarshalMsg implements msgp.Marshaler
func (q *Upsert) MarshalMsg(b []byte) ([]byte, error) {
	return q.packMsg(defaultPackData, b)
}

// UnmarshalMsg implements msgp.Unmarshaler
func (q *Upsert) UnmarshalMsg(data []byte) ([]byte, error) {
	return msgp.Skip(data)
}
<commit_msg>Implement unmarshalling of Upsert queries<commit_after>package tarantool

import (
	"errors"

	"github.com/tinylib/msgp/msgp"
)

type Upsert struct {
	Space interface{}
	Tuple []interface{}
	Set   []Operator
}

var _ Query = (*Upsert)(nil)

func (q *Upsert) GetCommandID() uint {
	return UpsertCommand
}

func (q *Upsert) packMsg(data *packData, b []byte) (o []byte, err error) {
	o = b
	o = msgp.AppendMapHeader(o, 3)

	if o, err = data.packSpace(q.Space, o); err != nil {
		return o, err
	}

	o = msgp.AppendUint(o, KeyTuple)
	if o, err = msgp.AppendIntf(o, q.Tuple); err != nil {
		return o, err
	}

	o = msgp.AppendUint(o, KeyDefTuple)
	o = msgp.AppendArrayHeader(o, uint32(len(q.Set)))
	for _, op := range q.Set {
		if o, err = marshalOperator(op, o); err != nil {
			return o, err
		}
	}

	return o, nil
}

// MarshalMsg implements msgp.Marshaler
func (q *Upsert) MarshalMsg(b []byte) ([]byte, error) {
	return q.packMsg(defaultPackData, b)
}

// UnmarshalMsg implements msgp.Unmarshaler
func (q *Upsert) UnmarshalMsg(data []byte) (buf []byte, err error) {
	var i uint32
	var k uint
	var t interface{}

	q.Space = nil

	buf = data
	if i, buf, err = msgp.ReadMapHeaderBytes(buf); err != nil {
		return
	}

	for ; i > 0; i-- {
		if k, buf, err = msgp.ReadUintBytes(buf); err != nil {
			return
		}

		switch k {
		case KeySpaceNo:
			if q.Space, buf, err = msgp.ReadUintBytes(buf); err != nil {
				return
			}
		case KeyTuple:
			t, buf, err = msgp.ReadIntfBytes(buf)
			if err != nil {
				return
			}

			if q.Tuple = t.([]interface{}); q.Tuple == nil {
				return buf, errors.New("Interface type is not []interface{}")
			}
		case KeyDefTuple:
			var len uint32
			if len, buf, err = msgp.ReadArrayHeaderBytes(buf); err != nil {
				return
			}

			q.Set = make([]Operator, len)
			for j := uint32(0); j < len; j++ {
				if q.Set[j], buf, err = unmarshalOperator(buf); err != nil {
					return
				}
			}
		}
	}

	if q.Space == nil {
		return buf, errors.New("upsert.Unpack: no space specified")
	}

	return
}
<|endoftext|>


<commit_before>package app

import (
	"cf"
	"cf/api"
	"cf/commands"
	"cf/configuration"
	"cf/requirements"
	"cf/terminal"
	"fmt"
	"github.com/codegangsta/cli"
	"os"
)

func New() (app *cli.App, err error) {
	cli.AppHelpTemplate = `NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   [environment variables] {{.Name}} [global options] command [command options] [arguments...]

VERSION:
   {{.Version}}

COMMANDS:
   {{range .Commands}}{{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Description}}
   {{end}}
GLOBAL OPTIONS:
   {{range .Flags}}{{.}}
   {{end}}
ENVIRONMENT VARIABLES:
   CF_TRACE=true - will output HTTP requests and responses during command
   HTTP_PROXY=http://proxy.example.com:8080 - set to your proxy
`

	cli.CommandHelpTemplate = `NAME:
   {{.Name}} - {{.Description}}

USAGE:
   {{.Usage}}{{with .Flags}}

OPTIONS:
   {{range .}}{{.}}
   {{end}}{{else}}
{{end}}`

	termUI := new(terminal.TerminalUI)
	configRepo := configuration.NewConfigurationDiskRepository()
	config, err := configRepo.Get()
	if err != nil {
		termUI.Failed(fmt.Sprintf(
			"Error loading config. Please reset target (%s) and log in (%s).",
			terminal.CommandColor("cf target"),
			terminal.CommandColor("cf login"),
		))
		configRepo.Delete()
		os.Exit(1)
		return
	}

	repoLocator := api.NewRepositoryLocator(config)
	cmdFactory := commands.NewFactory(termUI, repoLocator)
	reqFactory := requirements.NewFactory(termUI, repoLocator)
	cmdRunner := commands.NewRunner(reqFactory)

	app = cli.NewApp()
	app.Name = "cf"
	app.Usage = "A command line tool to interact with Cloud Foundry"
	app.Version = cf.Version
	app.Commands = []cli.Command{
		{
			Name:        "api",
			Description: "Set or view target api endpoint",
			Usage:       "cf api [URL]",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewApi()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "apps",
			ShortName:   "a",
			Description: "List all applications in the currently targeted space",
			Usage:       "cf apps",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewApps()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "bind-service",
			ShortName:   "bs",
			Description: "Bind a service instance to an application",
			Usage:       "cf bind-service --app APP --service SERVICE",
			Flags: []cli.Flag{
				cli.StringFlag{"app", "", "Which app"},
				cli.StringFlag{"service", "", "Which service instance to bind to the app"},
			},
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewBindService()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "create-org",
			ShortName:   "co",
			Description: "Create organization",
			Usage:       "cf create-org ORG",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewCreateOrganization()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "create-service",
			ShortName:   "cs",
			Description: "Create service instance",
			Usage: "cf create-service --offering [OFFERING] --plan [PLAN] --name SERVICE\n" +
				"   cf create-service --offering user-provided --name SERVICE --parameters \"<comma separated parameter names>\"",
			Flags: []cli.Flag{
				cli.StringFlag{"name", "", "name of the service instance"},
				cli.StringFlag{"offering", "", "name of the service offering to use"},
				cli.StringFlag{"plan", "", "name of the service plan to use"},
				cli.StringFlag{"parameters", "", "list of comma separated parameter names to use for user-provided services (eg. \"n1,n2\")"},
			},
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewCreateService()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "create-space",
			Description: "Create a space",
			Usage:       "cf create-space SPACE",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewCreateSpace()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "delete",
			ShortName:   "d",
			Description: "Delete an application",
			Usage:       "cf delete -f APP",
			Flags: []cli.Flag{
				cli.BoolFlag{"f", "Force deletion without confirmation"},
			},
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewDelete()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "delete-org",
			Description: "Delete an org",
			Usage:       "cf delete-org ORG",
			Flags: []cli.Flag{
				cli.BoolFlag{"f", "force deletion without confirmation"},
			},
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewDeleteOrg()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "delete-service",
			ShortName:   "ds",
			Description: "Delete a service instance",
			Usage:       "cf delete-service SERVICE",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewDeleteService()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "delete-space",
			Description: "Delete a space",
			Usage:       "cf delete-space SPACE",
			Flags: []cli.Flag{
				cli.BoolFlag{"f", "Force deletion without confirmation"},
			},
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewDeleteSpace()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "files",
			Description: "Print out a list of files in a directory or the contents of a specific file",
			Usage:       "cf files APP [PATH]",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewFiles()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "login",
			ShortName:   "l",
			Description: "Log user in",
			Usage:       "cf login [USERNAME] [PASSWORD]\n\n" + terminal.WarningColor("WARNING:\nProviding your password as a command line option is highly discouraged.\nYour password may be visible to others and may be recorded in your shell history."),
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewLogin()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "logout",
			ShortName:   "lo",
			Description: "Log user out",
			Usage:       "cf logout",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewLogout()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "logs",
			Description: "Show recent logs for CF applications",
			Usage:       "cf logs APP",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewLogs()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "orgs",
			ShortName:   "o",
			Description: "List all organizations",
			Usage:       "cf orgs",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewListOrganizations()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "passwd",
			ShortName:   "pw",
			Description: "Change user password",
			Usage:       "cf passwd",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewPassword()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "push",
			ShortName:   "p",
			Description: "Push an app",
			Usage: "cf push --name APP [--domain DOMAIN] [--host HOST] [--instances NUM]\n" +
				"                                [--memory MEMORY] [--buildpack URL] [--no-[re]start] [--path PATH]\n" +
				"                                [--stack STACK]",
			Flags: []cli.Flag{
				cli.StringFlag{"name", "", "name of the application"},
				cli.StringFlag{"domain", "", "domain (for example: cfapps.io)"},
				cli.StringFlag{"host", "", "hostname (for example: my-subdomain)"},
				cli.IntFlag{"instances", 1, "number of instances"},
				cli.StringFlag{"memory", "128", "memory limit (for example: 256, 1G, 1024M)"},
				cli.StringFlag{"buildpack", "", "custom buildpack URL (for example: https://github.com/heroku/heroku-buildpack-play.git)"},
				cli.BoolFlag{"no-start", "do not start an application after pushing"},
				cli.BoolFlag{"no-restart", "do not restart an application after pushing"},
				cli.StringFlag{"path", "", "path of application directory or zip file"},
				cli.StringFlag{"stack", "", "stack to use"},
			},
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewPush()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "rename-org",
			Description: "Rename an organization",
			Usage:       "cf rename-space ORG NEW_ORG",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewRenameOrg()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "rename-space",
			Description: "Rename a space",
			Usage:       "cf rename-space SPACE NEW_SPACE",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewRenameSpace()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "routes",
			ShortName:   "r",
			Description: "List all routes",
			Usage:       "cf routes",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewRoutes()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "services",
			ShortName:   "s",
			Description: "List all services in the currently targeted space",
			Usage:       "cf services [--marketplace]",
			Flags: []cli.Flag{
				cli.BoolFlag{"marketplace", "List available offerings on the marketplace"},
			},
			Action: func(c *cli.Context) {
				var cmd commands.Command
				cmd = cmdFactory.NewServices()

				if c.Bool("marketplace") {
					cmd = cmdFactory.NewMarketplaceServices()
				}

				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "set-env",
			ShortName:   "se",
			Description: "Set an env variable for an app",
			Usage:       "cf set-env APP NAME VALUE",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewSetEnv()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "spaces",
			Description: "List all spaces in an org",
			Usage:       "cf spaces",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewSpaces()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "stacks",
			Description: "List all stacks",
			Usage:       "cf stacks",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewStacks()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "start",
			ShortName:   "st",
			Description: "Start an app",
			Usage:       "cf start APP",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewStart()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "stop",
			ShortName:   "sp",
			Description: "Stop an app",
			Usage:       "cf stop APP",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewStop()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "target",
			ShortName:   "t",
			Description: "Set or view the targeted org or space",
			Usage:       "cf target [-o ORG] [-s SPACE]",
			Flags: []cli.Flag{
				cli.StringFlag{"o", "", "organization"},
				cli.StringFlag{"s", "", "space"},
			},
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewTarget()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "unbind-service",
			ShortName:   "us",
			Description: "Unbind a service instance from an application",
			Usage:       "cf unbind-service --app APP --service SERVICE",
			Flags: []cli.Flag{
				cli.StringFlag{"app", "", "name of the application"},
				cli.StringFlag{"service", "", "name of the service instance to unbind from the application"},
			},
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewUnbindService()
				cmdRunner.Run(cmd, c)
			},
		},
	}
	return
}
<commit_msg>Updated login command help text<commit_after>package app

import (
	"cf"
	"cf/api"
	"cf/commands"
	"cf/configuration"
	"cf/requirements"
	"cf/terminal"
	"fmt"
	"github.com/codegangsta/cli"
	"os"
)

func New() (app *cli.App, err error) {
	cli.AppHelpTemplate = `NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   [environment variables] {{.Name}} [global options] command [command options] [arguments...]

VERSION:
   {{.Version}}

COMMANDS:
   {{range .Commands}}{{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Description}}
   {{end}}
GLOBAL OPTIONS:
   {{range .Flags}}{{.}}
   {{end}}
ENVIRONMENT VARIABLES:
   CF_TRACE=true - will output HTTP requests and responses during command
   HTTP_PROXY=http://proxy.example.com:8080 - set to your proxy
`

	cli.CommandHelpTemplate = `NAME:
   {{.Name}} - {{.Description}}

USAGE:
   {{.Usage}}{{with .Flags}}

OPTIONS:
   {{range .}}{{.}}
   {{end}}{{else}}
{{end}}`

	termUI := new(terminal.TerminalUI)
	configRepo := configuration.NewConfigurationDiskRepository()
	config, err := configRepo.Get()
	if err != nil {
		termUI.Failed(fmt.Sprintf(
			"Error loading config. Please reset target (%s) and log in (%s).",
			terminal.CommandColor("cf target"),
			terminal.CommandColor("cf login"),
		))
		configRepo.Delete()
		os.Exit(1)
		return
	}

	repoLocator := api.NewRepositoryLocator(config)
	cmdFactory := commands.NewFactory(termUI, repoLocator)
	reqFactory := requirements.NewFactory(termUI, repoLocator)
	cmdRunner := commands.NewRunner(reqFactory)

	app = cli.NewApp()
	app.Name = "cf"
	app.Usage = "A command line tool to interact with Cloud Foundry"
	app.Version = cf.Version
	app.Commands = []cli.Command{
		{
			Name:        "api",
			Description: "Set or view target api endpoint",
			Usage:       "cf api [URL]",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewApi()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "apps",
			ShortName:   "a",
			Description: "List all applications in the currently targeted space",
			Usage:       "cf apps",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewApps()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "bind-service",
			ShortName:   "bs",
			Description: "Bind a service instance to an application",
			Usage:       "cf bind-service --app APP --service SERVICE",
			Flags: []cli.Flag{
				cli.StringFlag{"app", "", "Which app"},
				cli.StringFlag{"service", "", "Which service instance to bind to the app"},
			},
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewBindService()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "create-org",
			ShortName:   "co",
			Description: "Create organization",
			Usage:       "cf create-org ORG",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewCreateOrganization()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "create-service",
			ShortName:   "cs",
			Description: "Create service instance",
			Usage: "cf create-service --offering [OFFERING] --plan [PLAN] --name SERVICE\n" +
				"   cf create-service --offering user-provided --name SERVICE --parameters \"<comma separated parameter names>\"",
			Flags: []cli.Flag{
				cli.StringFlag{"name", "", "name of the service instance"},
				cli.StringFlag{"offering", "", "name of the service offering to use"},
				cli.StringFlag{"plan", "", "name of the service plan to use"},
				cli.StringFlag{"parameters", "", "list of comma separated parameter names to use for user-provided services (eg. \"n1,n2\")"},
			},
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewCreateService()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "create-space",
			Description: "Create a space",
			Usage:       "cf create-space SPACE",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewCreateSpace()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "delete",
			ShortName:   "d",
			Description: "Delete an application",
			Usage:       "cf delete -f APP",
			Flags: []cli.Flag{
				cli.BoolFlag{"f", "Force deletion without confirmation"},
			},
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewDelete()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "delete-org",
			Description: "Delete an org",
			Usage:       "cf delete-org ORG",
			Flags: []cli.Flag{
				cli.BoolFlag{"f", "force deletion without confirmation"},
			},
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewDeleteOrg()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "delete-service",
			ShortName:   "ds",
			Description: "Delete a service instance",
			Usage:       "cf delete-service SERVICE",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewDeleteService()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "delete-space",
			Description: "Delete a space",
			Usage:       "cf delete-space SPACE",
			Flags: []cli.Flag{
				cli.BoolFlag{"f", "Force deletion without confirmation"},
			},
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewDeleteSpace()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "files",
			Description: "Print out a list of files in a directory or the contents of a specific file",
			Usage:       "cf files APP [PATH]",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewFiles()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "login",
			ShortName:   "l",
			Description: "Log user in",
			Usage:       "cf login [USERNAME] [PASSWORD]\n\n" +
					terminal.WarningColor("WARNING:\n   Providing your password as a command line option is highly discouraged.\n   Your password may be visible to others and may be recorded in your shell history.\n\n") +
					"EXAMPLE:\n" +
					"   cf login (omit username and password to login interactively -- cf will prompt for both)\n" +
					"   cf login name@example.com pa55woRD (specify username and password to login non-interactively)\n" +
					"   cf login name@example.com \"my password\" (use quotes for passwords with a space)",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewLogin()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "logout",
			ShortName:   "lo",
			Description: "Log user out",
			Usage:       "cf logout",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewLogout()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "logs",
			Description: "Show recent logs for CF applications",
			Usage:       "cf logs APP",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewLogs()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "orgs",
			ShortName:   "o",
			Description: "List all organizations",
			Usage:       "cf orgs",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewListOrganizations()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "passwd",
			ShortName:   "pw",
			Description: "Change user password",
			Usage:       "cf passwd",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewPassword()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "push",
			ShortName:   "p",
			Description: "Push an app",
			Usage: "cf push --name APP [--domain DOMAIN] [--host HOST] [--instances NUM]\n" +
				"                                [--memory MEMORY] [--buildpack URL] [--no-[re]start] [--path PATH]\n" +
				"                                [--stack STACK]",
			Flags: []cli.Flag{
				cli.StringFlag{"name", "", "name of the application"},
				cli.StringFlag{"domain", "", "domain (for example: cfapps.io)"},
				cli.StringFlag{"host", "", "hostname (for example: my-subdomain)"},
				cli.IntFlag{"instances", 1, "number of instances"},
				cli.StringFlag{"memory", "128", "memory limit (for example: 256, 1G, 1024M)"},
				cli.StringFlag{"buildpack", "", "custom buildpack URL (for example: https://github.com/heroku/heroku-buildpack-play.git)"},
				cli.BoolFlag{"no-start", "do not start an application after pushing"},
				cli.BoolFlag{"no-restart", "do not restart an application after pushing"},
				cli.StringFlag{"path", "", "path of application directory or zip file"},
				cli.StringFlag{"stack", "", "stack to use"},
			},
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewPush()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "rename-org",
			Description: "Rename an organization",
			Usage:       "cf rename-space ORG NEW_ORG",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewRenameOrg()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "rename-space",
			Description: "Rename a space",
			Usage:       "cf rename-space SPACE NEW_SPACE",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewRenameSpace()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "routes",
			ShortName:   "r",
			Description: "List all routes",
			Usage:       "cf routes",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewRoutes()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "services",
			ShortName:   "s",
			Description: "List all services in the currently targeted space",
			Usage:       "cf services [--marketplace]",
			Flags: []cli.Flag{
				cli.BoolFlag{"marketplace", "List available offerings on the marketplace"},
			},
			Action: func(c *cli.Context) {
				var cmd commands.Command
				cmd = cmdFactory.NewServices()

				if c.Bool("marketplace") {
					cmd = cmdFactory.NewMarketplaceServices()
				}

				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "set-env",
			ShortName:   "se",
			Description: "Set an env variable for an app",
			Usage:       "cf set-env APP NAME VALUE",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewSetEnv()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "spaces",
			Description: "List all spaces in an org",
			Usage:       "cf spaces",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewSpaces()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "stacks",
			Description: "List all stacks",
			Usage:       "cf stacks",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewStacks()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "start",
			ShortName:   "st",
			Description: "Start an app",
			Usage:       "cf start APP",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewStart()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "stop",
			ShortName:   "sp",
			Description: "Stop an app",
			Usage:       "cf stop APP",
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewStop()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "target",
			ShortName:   "t",
			Description: "Set or view the targeted org or space",
			Usage:       "cf target [-o ORG] [-s SPACE]",
			Flags: []cli.Flag{
				cli.StringFlag{"o", "", "organization"},
				cli.StringFlag{"s", "", "space"},
			},
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewTarget()
				cmdRunner.Run(cmd, c)
			},
		},
		{
			Name:        "unbind-service",
			ShortName:   "us",
			Description: "Unbind a service instance from an application",
			Usage:       "cf unbind-service --app APP --service SERVICE",
			Flags: []cli.Flag{
				cli.StringFlag{"app", "", "name of the application"},
				cli.StringFlag{"service", "", "name of the service instance to unbind from the application"},
			},
			Action: func(c *cli.Context) {
				cmd := cmdFactory.NewUnbindService()
				cmdRunner.Run(cmd, c)
			},
		},
	}
	return
}
<|endoftext|>


<commit_before>package app

import (
	"cf/api"
	"cf/commands"
	"cf/configuration"
	"cf/terminal"
	"github.com/codegangsta/cli"
)

func New() (app *cli.App) {
	cli.AppHelpTemplate = `NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   [environment variables] {{.Name}} [global options] command [command options] [arguments...]

VERSION:
   {{.Version}}

COMMANDS:
   {{range .Commands}}{{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Description}}
   {{end}}
GLOBAL OPTIONS:
   {{range .Flags}}{{.}}
   {{end}}
ENVIRONMENT VARIABLES:
   CF_TRACE=true - will output HTTP requests and responses during command
`

	cli.CommandHelpTemplate = `NAME:
   {{.Name}} - {{.Description}}

USAGE:
   {{.Usage}}{{with .Flags}}

OPTIONS:
   {{range .}}{{.}}
   {{end}}{{else}}
{{end}}`

	termUI := new(terminal.TerminalUI)
	config := configuration.Get()

	organizationRepo := new(api.CloudControllerOrganizationRepository)
	spaceRepo := new(api.CloudControllerSpaceRepository)
	appRepo := new(api.CloudControllerApplicationRepository)
	domainRepo := new(api.CloudControllerDomainRepository)
	routeRepo := new(api.CloudControllerRouteRepository)

	app = cli.NewApp()
	app.Name = "cf"
	app.Usage = "A command line tool to interact with Cloud Foundry"
	app.Version = "1.0.0.alpha"
	app.Commands = []cli.Command{
		{
			Name:        "target",
			ShortName:   "t",
			Description: "Set or view the target",
			Usage:       "cf target <target> --o <organization> --s <space>",
			Flags: []cli.Flag{
				cli.StringFlag{"o", "", "organization"},
				cli.StringFlag{"s", "", "space"},
			},
			Action: func(c *cli.Context) {
				cmd := commands.NewTarget(termUI, config, organizationRepo, spaceRepo)
				cmd.Run(c)
			},
		},
		{
			Name:        "login",
			ShortName:   "l",
			Description: "Log user in",
			Usage:       "cf login [username]",
			Action: func(c *cli.Context) {
				authenticator := new(api.UAAAuthenticator)

				cmd := commands.NewLogin(termUI, config, organizationRepo, spaceRepo, authenticator)
				cmd.Run(c)
			},
		},
		{
			Name:        "set-env",
			ShortName:   "se",
			Description: "Set an environment variable for an application",
			Usage:       "cf set-env <application> <variable> <value>",
			Action: func(c *cli.Context) {
				cmd := commands.NewSetEnv(termUI, config, appRepo)
				cmd.Run(c)
			},
		},
		{
			Name:        "logout",
			ShortName:   "lo",
			Description: "Log user out",
			Usage:       "cf logout",
			Action: func(c *cli.Context) {
				cmd := commands.NewLogout(termUI, config)
				cmd.Run(c)
			},
		},
		{
			Name:        "push",
			ShortName:   "p",
			Description: "Push an application",
			Usage:       "cf push --name <application> [--domain <domain>] [--host <hostname>] [--instances <num>] [--memory <memory>] [--buildpack <url>]",
			Flags: []cli.Flag{
				cli.StringFlag{"name", "", "name of the application"},
				cli.StringFlag{"domain", "", "domain (for example: cfapps.io)"},
				cli.StringFlag{"host", "", "hostname (for example: my-subdomain)"},
				cli.IntFlag{"instances", 1, "number of instances"},
				cli.StringFlag{"memory", "128", "memory limit (for example: 256, 1G, 1024M)"},
				cli.StringFlag{"buildpack", "", "Custom buildpack URL (for example: https://github.com/heroku/heroku-buildpack-play.git)"},
			},
			Action: func(c *cli.Context) {
				cmd := commands.NewPush(termUI, config, appRepo, domainRepo, routeRepo)
				cmd.Run(c)
			},
		},
		{
			Name:        "apps",
			ShortName:   "a",
			Description: "List all applications in the currently selected space",
			Usage:       "cf apps",
			Action: func(c *cli.Context) {
				cmd := commands.NewApps(termUI, config, appRepo)
				cmd.Run(c)
			},
		},
		{
			Name:        "delete",
			ShortName:   "d",
			Description: "Delete an application",
			Usage:       "cf delete <application>",
			Action: func(c *cli.Context) {
				cmd := commands.NewDelete(termUI, config, appRepo)
				cmd.Run(c)
			},
		},
		{
			Name:        "start",
			ShortName:   "s",
			Description: "Start applications",
			Usage:       "cf start <application>",
			Action: func(c *cli.Context) {
				cmd := commands.NewStart(termUI, config, appRepo)
				cmd.Run(c)
			},
		},
		{
			Name:        "stop",
			ShortName:   "st",
			Description: "Stop applications",
			Usage:       "cf stop <application>",
			Action: func(c *cli.Context) {
				cmd := commands.NewStop(termUI, config, appRepo)
				cmd.Run(c)
			},
		},
	}
	return
}
<commit_msg>make command text more consistent<commit_after>package app

import (
	"cf/api"
	"cf/commands"
	"cf/configuration"
	"cf/terminal"
	"github.com/codegangsta/cli"
)

func New() (app *cli.App) {
	cli.AppHelpTemplate = `NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   [environment variables] {{.Name}} [global options] command [command options] [arguments...]

VERSION:
   {{.Version}}

COMMANDS:
   {{range .Commands}}{{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Description}}
   {{end}}
GLOBAL OPTIONS:
   {{range .Flags}}{{.}}
   {{end}}
ENVIRONMENT VARIABLES:
   CF_TRACE=true - will output HTTP requests and responses during command
`

	cli.CommandHelpTemplate = `NAME:
   {{.Name}} - {{.Description}}

USAGE:
   {{.Usage}}{{with .Flags}}

OPTIONS:
   {{range .}}{{.}}
   {{end}}{{else}}
{{end}}`

	termUI := new(terminal.TerminalUI)
	config := configuration.Get()

	organizationRepo := new(api.CloudControllerOrganizationRepository)
	spaceRepo := new(api.CloudControllerSpaceRepository)
	appRepo := new(api.CloudControllerApplicationRepository)
	domainRepo := new(api.CloudControllerDomainRepository)
	routeRepo := new(api.CloudControllerRouteRepository)

	app = cli.NewApp()
	app.Name = "cf"
	app.Usage = "A command line tool to interact with Cloud Foundry"
	app.Version = "1.0.0.alpha"
	app.Commands = []cli.Command{
		{
			Name:        "target",
			ShortName:   "t",
			Description: "Set or view the target",
			Usage:       "cf target <target> --o <organization> --s <space>",
			Flags: []cli.Flag{
				cli.StringFlag{"o", "", "organization"},
				cli.StringFlag{"s", "", "space"},
			},
			Action: func(c *cli.Context) {
				cmd := commands.NewTarget(termUI, config, organizationRepo, spaceRepo)
				cmd.Run(c)
			},
		},
		{
			Name:        "login",
			ShortName:   "l",
			Description: "Log user in",
			Usage:       "cf login [username]",
			Action: func(c *cli.Context) {
				authenticator := new(api.UAAAuthenticator)

				cmd := commands.NewLogin(termUI, config, organizationRepo, spaceRepo, authenticator)
				cmd.Run(c)
			},
		},
		{
			Name:        "set-env",
			ShortName:   "se",
			Description: "Set an environment variable for an application",
			Usage:       "cf set-env <application> <variable> <value>",
			Action: func(c *cli.Context) {
				cmd := commands.NewSetEnv(termUI, config, appRepo)
				cmd.Run(c)
			},
		},
		{
			Name:        "logout",
			ShortName:   "lo",
			Description: "Log user out",
			Usage:       "cf logout",
			Action: func(c *cli.Context) {
				cmd := commands.NewLogout(termUI, config)
				cmd.Run(c)
			},
		},
		{
			Name:        "push",
			ShortName:   "p",
			Description: "Push an application",
			Usage:       "cf push --name <application> [--domain <domain>] [--host <hostname>] [--instances <num>] [--memory <memory>] [--buildpack <url>]",
			Flags: []cli.Flag{
				cli.StringFlag{"name", "", "name of the application"},
				cli.StringFlag{"domain", "", "domain (for example: cfapps.io)"},
				cli.StringFlag{"host", "", "hostname (for example: my-subdomain)"},
				cli.IntFlag{"instances", 1, "number of instances"},
				cli.StringFlag{"memory", "128", "memory limit (for example: 256, 1G, 1024M)"},
				cli.StringFlag{"buildpack", "", "custom buildpack URL (for example: https://github.com/heroku/heroku-buildpack-play.git)"},
			},
			Action: func(c *cli.Context) {
				cmd := commands.NewPush(termUI, config, appRepo, domainRepo, routeRepo)
				cmd.Run(c)
			},
		},
		{
			Name:        "apps",
			ShortName:   "a",
			Description: "List all applications in the currently selected space",
			Usage:       "cf apps",
			Action: func(c *cli.Context) {
				cmd := commands.NewApps(termUI, config, appRepo)
				cmd.Run(c)
			},
		},
		{
			Name:        "delete",
			ShortName:   "d",
			Description: "Delete an application",
			Usage:       "cf delete <application>",
			Action: func(c *cli.Context) {
				cmd := commands.NewDelete(termUI, config, appRepo)
				cmd.Run(c)
			},
		},
		{
			Name:        "start",
			ShortName:   "s",
			Description: "Start applications",
			Usage:       "cf start <application>",
			Action: func(c *cli.Context) {
				cmd := commands.NewStart(termUI, config, appRepo)
				cmd.Run(c)
			},
		},
		{
			Name:        "stop",
			ShortName:   "st",
			Description: "Stop applications",
			Usage:       "cf stop <application>",
			Action: func(c *cli.Context) {
				cmd := commands.NewStop(termUI, config, appRepo)
				cmd.Run(c)
			},
		},
	}
	return
}
<|endoftext|>


<commit_before>package parser

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"reflect"
	"strconv"
	"strings"
	"time"

	"cloud.google.com/go/bigquery"

	"github.com/m-lab/etl/metrics"
	"github.com/m-lab/etl/schema"
	"github.com/prometheus/client_golang/prometheus"
)

// TODO(JosephMarques) See if there is a better way of determining
// where to send the request (there almost certainly is)
var BaseURL = "https://annotator-dot-" +
	os.Getenv("GCLOUD_PROJECT") +
	".appspot.com/annotate?"

// AddMetaDataPTConnSpec takes a pointer to a
// MLabConnectionSpecification struct and a timestamp. With these, it
// will fetch the appropriate metadata and add it to the hop struct
// referenced by the pointer.
func AddMetaDataPTConnSpec(spec *schema.MLabConnectionSpecification, timestamp time.Time) {
	// Time the response
	timerStart := time.Now()
	defer func(tStart time.Time) {
		metrics.AnnotationTimeSummary.
			With(prometheus.Labels{"test_type": "PT"}).
			Observe(float64(time.Since(tStart).Nanoseconds()))
	}(timerStart)
	if spec.Server_ip != "" {
		GetAndInsertGeolocationIPStruct(&spec.Server_geolocation, spec.Server_ip, timestamp)
	}
	if spec.Client_ip != "" {
		GetAndInsertGeolocationIPStruct(&spec.Client_geolocation, spec.Client_ip, timestamp)
	}
}

// AddMetaDataPTHop takes a pointer to a ParisTracerouteHop and a
// timestamp. With these, it will fetch the appropriate metadata and
// add it to the hop struct referenced by the pointer.
func AddMetaDataPTHop(hop *schema.ParisTracerouteHop, timestamp time.Time) {
	// Time the response
	timerStart := time.Now()
	defer func(tStart time.Time) {
		metrics.AnnotationTimeSummary.
			With(prometheus.Labels{"test_type": "PT-HOP"}).
			Observe(float64(time.Since(tStart).Nanoseconds()))
	}(timerStart)
	if hop.Src_ip != "" {
		GetAndInsertGeolocationIPStruct(&hop.Src_geolocation, hop.Src_ip, timestamp)
	}
	if hop.Dest_ip != "" {
		GetAndInsertGeolocationIPStruct(&hop.Dest_geolocation, hop.Dest_ip, timestamp)
	}
}

// GetAndInsertGeolocationIPStruct takes a NON-NIL pointer to a
// pre-allocated schema.GeolocationIP struct, an IP address, and a
// timestamp. It will connect to the annotation service, get the
// metadata, and insert the metadata into the reigion pointed to by
// the schema.GeolocationIP pointer.
func GetAndInsertGeolocationIPStruct(geo *schema.GeolocationIP, ip string, timestamp time.Time) {
	url := BaseURL + "ip_addr=" + url.QueryEscape(ip) +
		"&since_epoch=" + strconv.FormatInt(timestamp.Unix(), 10)
	annotationData := GetMetaData(url)
	if annotationData != nil && annotationData.Geo != nil {
		*geo = *annotationData.Geo
	}
}

// AddMetaDataNDTConnSpec takes a connection spec and a timestamp and
// annotates the connection spec with metadata associated with each IP
// Address. It will either sucessfully add the metadata or fail
// silently and make no changes.
func AddMetaDataNDTConnSpec(spec schema.Web100ValueMap, timestamp time.Time) {
	// Time the response
	timerStart := time.Now()
	defer func(tStart time.Time) {
		metrics.AnnotationTimeSummary.
			With(prometheus.Labels{"test_type": "NDT"}).
			Observe(float64(time.Since(tStart).Nanoseconds()))
	}(timerStart)

	GetAndInsertMetaIntoNDTConnSpec("client", spec, timestamp)
	GetAndInsertMetaIntoNDTConnSpec("server", spec, timestamp)
}

// GetAndInsertNDT takes a timestamp, an NDT connection spec, and a
// string indicating whether it should get the metadata for the client
// end or the server end of the connection. It will either insert the
// data into the connection spec or silently fail.
func GetAndInsertMetaIntoNDTConnSpec(side string, spec schema.Web100ValueMap, timestamp time.Time) {
	ip, ok := spec.GetString([]string{side + "_ip"})
	if ok {
		url := BaseURL + "ip_addr=" + url.QueryEscape(ip) +
			"&since_epoch=" + strconv.FormatInt(timestamp.Unix(), 10)
		annotationData := GetMetaData(url)
		if annotationData != nil && annotationData.Geo != nil {
			CopyStructToMap(annotationData.Geo, spec.Get(side+"_geolocation"))
		} else {
			metrics.AnnotationErrorCount.With(prometheus.
				Labels{"source": "Couldn't get metadata for the " + side + " side."}).Inc()
		}
	}
}

// CopyStructToMap takes a POINTER to an arbitrary struct and copies
// it's fields into a value map. It will also make fields entirely
// lower case, for convienece when working with exported structs. Also,
// NEVER pass in something that is not a pointer to a struct, as this
// will cause a panic.
func CopyStructToMap(sourceStruct interface{}, destinationMap map[string]bigquery.Value) {
	structToCopy := reflect.ValueOf(sourceStruct).Elem()
	typeOfStruct := structToCopy.Type()
	for i := 0; i < typeOfStruct.NumField(); i++ {
		destinationMap[strings.ToLower(typeOfStruct.Field(i).Name)] =
			structToCopy.Field(i).Interface()
	}

}

// GetMetaData combines the functionality of QueryAnnotationService and
// ParseJSONMetaDataResponse to query the annotator service and return
// the corresponding MetaData if it can, or a nil pointer if it
// encounters any error and cannot get the data for any reason
func GetMetaData(url string) *schema.MetaData {
	// Query the service and grab the response safely
	annotatorResponse, err := QueryAnnotationService(url)
	if err != nil {
		metrics.AnnotationErrorCount.
			With(prometheus.Labels{"source": "Error querying annotation service"}).Inc()
		log.Println(err)
		return nil
	}

	// Safely parse the JSON response and pass it back to the caller
	metaDataFromResponse, err := ParseJSONMetaDataResponse(annotatorResponse)
	if err != nil {
		metrics.AnnotationErrorCount.With(prometheus.
			Labels{"source": "Failed to parse JSON"}).Inc()
		log.Println(err)
		return nil
	}
	return metaDataFromResponse
}

// QueryAnnotationService will connect to the annotation service and
// copy the body of a valid response to a byte slice and return it to a
// user, returning an error if any occurs
func QueryAnnotationService(url string) ([]byte, error) {
	// Make the actual request
	resp, err := http.Get(url)

	// Catch http errors
	if err != nil {
		metrics.AnnotationErrorCount.
			With(prometheus.Labels{"source": "Request to Annotator failed"}).Inc()
		return nil, err
	}

	// Catch errors reported by the service
	if resp.StatusCode != http.StatusOK {
		metrics.AnnotationErrorCount.
			With(prometheus.Labels{"source": "Webserver gave non-ok response"}).Inc()
		return nil, errors.New("URL:" + url + " gave response code " + resp.Status)
	}
	defer resp.Body.Close()

	// Copy response into a byte slice
	return ioutil.ReadAll(resp.Body)
}

// ParseJSONMetaDataResponse takes a byte slice containing the test of
// the JSON from the annotator service and parses it into a MetaData
// struct, for easy manipulation. It returns a pointer to the struct on
// success and an error if an error occurs.
func ParseJSONMetaDataResponse(jsonBuffer []byte) (*schema.MetaData, error) {
	parsedJSON := &schema.MetaData{}
	err := json.Unmarshal(jsonBuffer, parsedJSON)
	if err != nil {
		return nil, err
	}
	return parsedJSON, nil
}
<commit_msg>Added the SS function, with no test.<commit_after>package parser

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"reflect"
	"strconv"
	"strings"
	"time"

	"cloud.google.com/go/bigquery"

	"github.com/m-lab/etl/metrics"
	"github.com/m-lab/etl/schema"
	"github.com/prometheus/client_golang/prometheus"
)

// TODO(JosephMarques) See if there is a better way of determining
// where to send the request (there almost certainly is)
var BaseURL = "https://annotator-dot-" +
	os.Getenv("GCLOUD_PROJECT") +
	".appspot.com/annotate?"

// AddMetaDataSSConnSpec takes a pointer to a
// Web100ConnectionSpecification struct and a timestamp. With these,
// it will fetch the appropriate metadata and add it to the hop struct
// referenced by the pointer.
func AddMetaDataSSConnSpec(spec *schema.Web100ConnectionSpecification, timestamp time.Time) {
	// Time the response
	timerStart := time.Now()
	defer func(tStart time.Time) {
		metrics.AnnotationTimeSummary.
			With(prometheus.Labels{"test_type": "SS"}).
			Observe(float64(time.Since(tStart).Nanoseconds()))
	}(timerStart)
	if spec.Local_ip != "" {
		GetAndInsertGeolocationIPStruct(&spec.Local_geolocation, spec.Local_ip, timestamp)
	}
	if spec.Remote_ip != "" {
		GetAndInsertGeolocationIPStruct(&spec.Remote_geolocation, spec.Remote_ip, timestamp)
	}
}

// AddMetaDataPTConnSpec takes a pointer to a
// MLabConnectionSpecification struct and a timestamp. With these, it
// will fetch the appropriate metadata and add it to the hop struct
// referenced by the pointer.
func AddMetaDataPTConnSpec(spec *schema.MLabConnectionSpecification, timestamp time.Time) {
	// Time the response
	timerStart := time.Now()
	defer func(tStart time.Time) {
		metrics.AnnotationTimeSummary.
			With(prometheus.Labels{"test_type": "PT"}).
			Observe(float64(time.Since(tStart).Nanoseconds()))
	}(timerStart)
	if spec.Server_ip != "" {
		GetAndInsertGeolocationIPStruct(&spec.Server_geolocation, spec.Server_ip, timestamp)
	}
	if spec.Client_ip != "" {
		GetAndInsertGeolocationIPStruct(&spec.Client_geolocation, spec.Client_ip, timestamp)
	}
}

// AddMetaDataPTHop takes a pointer to a ParisTracerouteHop and a
// timestamp. With these, it will fetch the appropriate metadata and
// add it to the hop struct referenced by the pointer.
func AddMetaDataPTHop(hop *schema.ParisTracerouteHop, timestamp time.Time) {
	// Time the response
	timerStart := time.Now()
	defer func(tStart time.Time) {
		metrics.AnnotationTimeSummary.
			With(prometheus.Labels{"test_type": "PT-HOP"}).
			Observe(float64(time.Since(tStart).Nanoseconds()))
	}(timerStart)
	if hop.Src_ip != "" {
		GetAndInsertGeolocationIPStruct(&hop.Src_geolocation, hop.Src_ip, timestamp)
	}
	if hop.Dest_ip != "" {
		GetAndInsertGeolocationIPStruct(&hop.Dest_geolocation, hop.Dest_ip, timestamp)
	}
}

// GetAndInsertGeolocationIPStruct takes a NON-NIL pointer to a
// pre-allocated schema.GeolocationIP struct, an IP address, and a
// timestamp. It will connect to the annotation service, get the
// metadata, and insert the metadata into the reigion pointed to by
// the schema.GeolocationIP pointer.
func GetAndInsertGeolocationIPStruct(geo *schema.GeolocationIP, ip string, timestamp time.Time) {
	url := BaseURL + "ip_addr=" + url.QueryEscape(ip) +
		"&since_epoch=" + strconv.FormatInt(timestamp.Unix(), 10)
	annotationData := GetMetaData(url)
	if annotationData != nil && annotationData.Geo != nil {
		*geo = *annotationData.Geo
	}
}

// AddMetaDataNDTConnSpec takes a connection spec and a timestamp and
// annotates the connection spec with metadata associated with each IP
// Address. It will either sucessfully add the metadata or fail
// silently and make no changes.
func AddMetaDataNDTConnSpec(spec schema.Web100ValueMap, timestamp time.Time) {
	// Time the response
	timerStart := time.Now()
	defer func(tStart time.Time) {
		metrics.AnnotationTimeSummary.
			With(prometheus.Labels{"test_type": "NDT"}).
			Observe(float64(time.Since(tStart).Nanoseconds()))
	}(timerStart)

	GetAndInsertMetaIntoNDTConnSpec("client", spec, timestamp)
	GetAndInsertMetaIntoNDTConnSpec("server", spec, timestamp)
}

// GetAndInsertNDT takes a timestamp, an NDT connection spec, and a
// string indicating whether it should get the metadata for the client
// end or the server end of the connection. It will either insert the
// data into the connection spec or silently fail.
func GetAndInsertMetaIntoNDTConnSpec(side string, spec schema.Web100ValueMap, timestamp time.Time) {
	ip, ok := spec.GetString([]string{side + "_ip"})
	if ok {
		url := BaseURL + "ip_addr=" + url.QueryEscape(ip) +
			"&since_epoch=" + strconv.FormatInt(timestamp.Unix(), 10)
		annotationData := GetMetaData(url)
		if annotationData != nil && annotationData.Geo != nil {
			CopyStructToMap(annotationData.Geo, spec.Get(side+"_geolocation"))
		} else {
			metrics.AnnotationErrorCount.With(prometheus.
				Labels{"source": "Couldn't get metadata for the " + side + " side."}).Inc()
		}
	}
}

// CopyStructToMap takes a POINTER to an arbitrary struct and copies
// it's fields into a value map. It will also make fields entirely
// lower case, for convienece when working with exported structs. Also,
// NEVER pass in something that is not a pointer to a struct, as this
// will cause a panic.
func CopyStructToMap(sourceStruct interface{}, destinationMap map[string]bigquery.Value) {
	structToCopy := reflect.ValueOf(sourceStruct).Elem()
	typeOfStruct := structToCopy.Type()
	for i := 0; i < typeOfStruct.NumField(); i++ {
		destinationMap[strings.ToLower(typeOfStruct.Field(i).Name)] =
			structToCopy.Field(i).Interface()
	}

}

// GetMetaData combines the functionality of QueryAnnotationService and
// ParseJSONMetaDataResponse to query the annotator service and return
// the corresponding MetaData if it can, or a nil pointer if it
// encounters any error and cannot get the data for any reason
func GetMetaData(url string) *schema.MetaData {
	// Query the service and grab the response safely
	annotatorResponse, err := QueryAnnotationService(url)
	if err != nil {
		metrics.AnnotationErrorCount.
			With(prometheus.Labels{"source": "Error querying annotation service"}).Inc()
		log.Println(err)
		return nil
	}

	// Safely parse the JSON response and pass it back to the caller
	metaDataFromResponse, err := ParseJSONMetaDataResponse(annotatorResponse)
	if err != nil {
		metrics.AnnotationErrorCount.With(prometheus.
			Labels{"source": "Failed to parse JSON"}).Inc()
		log.Println(err)
		return nil
	}
	return metaDataFromResponse
}

// QueryAnnotationService will connect to the annotation service and
// copy the body of a valid response to a byte slice and return it to a
// user, returning an error if any occurs
func QueryAnnotationService(url string) ([]byte, error) {
	// Make the actual request
	resp, err := http.Get(url)

	// Catch http errors
	if err != nil {
		metrics.AnnotationErrorCount.
			With(prometheus.Labels{"source": "Request to Annotator failed"}).Inc()
		return nil, err
	}

	// Catch errors reported by the service
	if resp.StatusCode != http.StatusOK {
		metrics.AnnotationErrorCount.
			With(prometheus.Labels{"source": "Webserver gave non-ok response"}).Inc()
		return nil, errors.New("URL:" + url + " gave response code " + resp.Status)
	}
	defer resp.Body.Close()

	// Copy response into a byte slice
	return ioutil.ReadAll(resp.Body)
}

// ParseJSONMetaDataResponse takes a byte slice containing the test of
// the JSON from the annotator service and parses it into a MetaData
// struct, for easy manipulation. It returns a pointer to the struct on
// success and an error if an error occurs.
func ParseJSONMetaDataResponse(jsonBuffer []byte) (*schema.MetaData, error) {
	parsedJSON := &schema.MetaData{}
	err := json.Unmarshal(jsonBuffer, parsedJSON)
	if err != nil {
		return nil, err
	}
	return parsedJSON, nil
}
<|endoftext|>


<commit_before>package dev

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/watermint/toolbox/infra/control/app_control"
	"github.com/watermint/toolbox/infra/control/app_control_launcher"
	"github.com/watermint/toolbox/infra/recipe/rc_exec"
	"github.com/watermint/toolbox/infra/recipe/rc_recipe"
	"github.com/watermint/toolbox/infra/recipe/rc_spec"
	"github.com/watermint/toolbox/infra/ui/app_msg"
	"github.com/watermint/toolbox/infra/ui/app_msg_container"
	"github.com/watermint/toolbox/quality/infra/qt_messages"
	"github.com/watermint/toolbox/recipe/dev/spec"
	"go.uber.org/zap"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
)

const (
	minimumSpecDocVersion = 59
)

type Preflight struct {
}

func (z *Preflight) Preset() {
}

func (z *Preflight) Test(c app_control.Control) error {
	return z.Exec(c)
}

func (z *Preflight) sortMessages(c app_control.Control, filename string) error {
	l := c.Log().With(zap.String("filename", filename))
	p := filepath.Join("resources", filename)
	content, err := ioutil.ReadFile(p)
	if err != nil {
		l.Info("SKIP: Unable to open resource file", zap.Error(err))
		return nil
	}
	messages := make(map[string]string)
	if err = json.Unmarshal(content, &messages); err != nil {
		l.Warn("Unable to unmarshal message file", zap.Error(err))
		return err
	}
	buf := &bytes.Buffer{}
	je := json.NewEncoder(buf)
	je.SetEscapeHTML(false)
	je.SetIndent("", "  ")
	if err = je.Encode(messages); err != nil {
		l.Warn("Unable to create sorted image", zap.Error(err))
		return err
	}
	if err := ioutil.WriteFile(p, buf.Bytes(), 0644); err != nil {
		l.Warn("Unable to update message", zap.Error(err))
		return err
	}
	return nil
}

func (z *Preflight) deleteOldGeneratedFiles(c app_control.Control, path string) error {
	l := c.Log()
	whiteList := func(name string) bool {
		nameLower := strings.ToLower(name)
		switch {
		case nameLower == "changes.md":
			return true
		case strings.HasPrefix(nameLower, "spec") && strings.HasSuffix(nameLower, ".json"):
			return true
		default:
			return false
		}
	}

	entries, err := ioutil.ReadDir(path)
	if err != nil {
		return err
	}

	for _, e := range entries {
		if !whiteList(e.Name()) {
			p := filepath.Join(path, e.Name())
			if c.IsTest() {
				continue
			}
			err := os.Remove(p)
			if err != nil {
				l.Error("Unable to remove file", zap.Error(err), zap.String("path", p))
				return nil
			}
		}
	}
	return nil
}

func (z *Preflight) cloneSpec(c app_control.Control, path string, release int) error {
	l := c.Log()
	if c.IsTest() {
		l.Debug("Skip for test")
		return nil
	}
	s, err := ioutil.ReadFile(filepath.Join(path, "spec.json"))
	if err != nil {
		l.Error("Unable to open current spec file", zap.Error(err))
		return err
	}
	err = ioutil.WriteFile(filepath.Join(path, fmt.Sprintf("spec_%d.json", release)), s, 0644)
	if err != nil {
		l.Error("Unable to create version spec file", zap.Error(err))
		return err
	}
	return nil
}

func (z *Preflight) Exec(c app_control.Control) error {
	l := c.Log()
	languages := []string{"en", "ja"}

	release := 0
	if !c.IsTest() {
		v, err := ioutil.ReadFile("version")
		if err != nil {
			l.Error("Unable to read version file", zap.Error(err))
			return err
		}
		release, err = strconv.Atoi(string(v))
		if err != nil {
			l.Error("Unable to parse version number", zap.Error(err))
			return err
		}
	}

	for _, lang := range languages {
		suffix := ""
		if lang != "en" {
			suffix = "_" + lang
		}
		path := fmt.Sprintf("doc/generated%s/", suffix)
		ll := l.With(zap.String("lang", lang), zap.String("suffix", suffix))

		ll.Info("Clean up generated document folder")
		if err := z.deleteOldGeneratedFiles(c, path); err != nil {
			return err
		}

		ll.Info("Generating README & command documents")
		err := rc_exec.Exec(c, &Doc{}, func(r rc_recipe.Recipe) {
			rr := r.(*Doc)
			rr.Badge = true
			rr.MarkdownReadme = true
			rr.Lang = lang
			rr.Filename = fmt.Sprintf("README%s.md", suffix)
			rr.CommandPath = path
		})
		if err != nil {
			l.Error("Failed to generate documents", zap.Error(err))
			return err
		}

		if !c.IsTest() {
			ll.Info("Generating Spec document")
			err = rc_exec.Exec(c, &spec.Doc{}, func(r rc_recipe.Recipe) {
				rr := r.(*spec.Doc)
				rr.Lang = lang
				rr.FilePath = filepath.Join(path, "spec.json")
			})
			if err != nil {
				l.Error("Failed to generate documents", zap.Error(err))
				return err
			}
		}

		ll.Info("Clone spec")
		if err := z.cloneSpec(c, path, release); err != nil {
			return err
		}

		ll.Info("Sorting message resources")
		if err := z.sortMessages(c, fmt.Sprintf("messages%s.json", suffix)); err != nil {
			return err
		}

		if !c.IsTest() && minimumSpecDocVersion < release {
			ll.Info("Generating release notes")
			err := rc_exec.Exec(c, &spec.Diff{}, func(r rc_recipe.Recipe) {
				rr := r.(*spec.Diff)
				rr.Lang = lang
				rr.Release1 = fmt.Sprintf("%d", release-1)
				rr.FilePath = filepath.Join(path, "changes.md")
			})
			if err != nil {
				l.Error("Failed to generate documents", zap.Error(err))
				return err
			}
		}
	}

	{
		cl := c.(app_control_launcher.ControlLauncher)
		cat := cl.Catalogue()
		l.Info("Verify recipes")
		for _, r := range cat.Recipes() {
			spec := rc_spec.New(r)
			for _, m := range spec.Messages() {
				l.Debug("message", zap.String("key", m.Key()), zap.String("text", c.UI().Text(m)))
			}
		}

		l.Info("Verify ingredients")
		for _, r := range cat.Ingredients() {
			spec := rc_spec.New(r)
			for _, m := range spec.Messages() {
				l.Debug("message", zap.String("key", m.Key()), zap.String("text", c.UI().Text(m)))
			}
		}

		l.Info("Verify message objects")
		for _, m := range cat.Messages() {
			msgs := app_msg.Messages(m)
			for _, msg := range msgs {
				l.Debug("message", zap.String("key", msg.Key()), zap.String("text", c.UI().Text(msg)))
			}
		}
	}

	l.Info("Verify message resources")
	qm := c.Messages().(app_msg_container.Quality)
	missing := qm.MissingKeys()
	if len(missing) > 0 {
		suggested := make([]string, 0)
		for _, k := range missing {
			l.Error("Key missing", zap.String("key", k))
			suggested = append(suggested, "\""+k+"\":\"\",")
		}
		sort.Strings(suggested)
		fmt.Println(strings.Join(suggested, "\n"))
	}

	return qt_messages.VerifyMessages(c)
}
<commit_msg>fix #310 : fix test<commit_after>package dev

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/watermint/toolbox/infra/control/app_control"
	"github.com/watermint/toolbox/infra/control/app_control_launcher"
	"github.com/watermint/toolbox/infra/recipe/rc_exec"
	"github.com/watermint/toolbox/infra/recipe/rc_recipe"
	"github.com/watermint/toolbox/infra/recipe/rc_spec"
	"github.com/watermint/toolbox/infra/ui/app_msg"
	"github.com/watermint/toolbox/infra/ui/app_msg_container"
	"github.com/watermint/toolbox/quality/infra/qt_messages"
	"github.com/watermint/toolbox/recipe/dev/spec"
	"go.uber.org/zap"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
)

const (
	minimumSpecDocVersion = 59
)

type Preflight struct {
}

func (z *Preflight) Preset() {
}

func (z *Preflight) Test(c app_control.Control) error {
	return z.Exec(c)
}

func (z *Preflight) sortMessages(c app_control.Control, filename string) error {
	l := c.Log().With(zap.String("filename", filename))
	p := filepath.Join("resources", filename)
	content, err := ioutil.ReadFile(p)
	if err != nil {
		l.Info("SKIP: Unable to open resource file", zap.Error(err))
		return nil
	}
	messages := make(map[string]string)
	if err = json.Unmarshal(content, &messages); err != nil {
		l.Warn("Unable to unmarshal message file", zap.Error(err))
		return err
	}
	buf := &bytes.Buffer{}
	je := json.NewEncoder(buf)
	je.SetEscapeHTML(false)
	je.SetIndent("", "  ")
	if err = je.Encode(messages); err != nil {
		l.Warn("Unable to create sorted image", zap.Error(err))
		return err
	}
	if err := ioutil.WriteFile(p, buf.Bytes(), 0644); err != nil {
		l.Warn("Unable to update message", zap.Error(err))
		return err
	}
	return nil
}

func (z *Preflight) deleteOldGeneratedFiles(c app_control.Control, path string) error {
	l := c.Log()
	whiteList := func(name string) bool {
		nameLower := strings.ToLower(name)
		switch {
		case nameLower == "changes.md":
			return true
		case strings.HasPrefix(nameLower, "spec") && strings.HasSuffix(nameLower, ".json"):
			return true
		default:
			return false
		}
	}

	entries, err := ioutil.ReadDir(path)
	if err != nil {
		return err
	}

	for _, e := range entries {
		if !whiteList(e.Name()) {
			p := filepath.Join(path, e.Name())
			if c.IsTest() {
				continue
			}
			err := os.Remove(p)
			if err != nil {
				l.Error("Unable to remove file", zap.Error(err), zap.String("path", p))
				return nil
			}
		}
	}
	return nil
}

func (z *Preflight) cloneSpec(c app_control.Control, path string, release int) error {
	l := c.Log()
	if c.IsTest() {
		l.Debug("Skip for test")
		return nil
	}
	s, err := ioutil.ReadFile(filepath.Join(path, "spec.json"))
	if err != nil {
		l.Error("Unable to open current spec file", zap.Error(err))
		return err
	}
	err = ioutil.WriteFile(filepath.Join(path, fmt.Sprintf("spec_%d.json", release)), s, 0644)
	if err != nil {
		l.Error("Unable to create version spec file", zap.Error(err))
		return err
	}
	return nil
}

func (z *Preflight) Exec(c app_control.Control) error {
	l := c.Log()
	languages := []string{"en", "ja"}

	release := 0
	if !c.IsTest() {
		v, err := ioutil.ReadFile("version")
		if err != nil {
			l.Error("Unable to read version file", zap.Error(err))
			return err
		}
		release, err = strconv.Atoi(string(v))
		if err != nil {
			l.Error("Unable to parse version number", zap.Error(err))
			return err
		}
	}

	for _, lang := range languages {
		suffix := ""
		if lang != "en" {
			suffix = "_" + lang
		}
		path := fmt.Sprintf("doc/generated%s/", suffix)
		ll := l.With(zap.String("lang", lang), zap.String("suffix", suffix))

		if !c.IsTest() {
			ll.Info("Clean up generated document folder")
			if err := z.deleteOldGeneratedFiles(c, path); err != nil {
				return err
			}

			ll.Info("Generating README & command documents")
			err := rc_exec.Exec(c, &Doc{}, func(r rc_recipe.Recipe) {
				rr := r.(*Doc)
				rr.Badge = true
				rr.MarkdownReadme = true
				rr.Lang = lang
				rr.Filename = fmt.Sprintf("README%s.md", suffix)
				rr.CommandPath = path
			})
			if err != nil {
				l.Error("Failed to generate documents", zap.Error(err))
				return err
			}

			ll.Info("Generating Spec document")
			err = rc_exec.Exec(c, &spec.Doc{}, func(r rc_recipe.Recipe) {
				rr := r.(*spec.Doc)
				rr.Lang = lang
				rr.FilePath = filepath.Join(path, "spec.json")
			})
			if err != nil {
				l.Error("Failed to generate documents", zap.Error(err))
				return err
			}
		}

		ll.Info("Clone spec")
		if err := z.cloneSpec(c, path, release); err != nil {
			return err
		}

		ll.Info("Sorting message resources")
		if err := z.sortMessages(c, fmt.Sprintf("messages%s.json", suffix)); err != nil {
			return err
		}

		if !c.IsTest() && minimumSpecDocVersion < release {
			ll.Info("Generating release notes")
			err := rc_exec.Exec(c, &spec.Diff{}, func(r rc_recipe.Recipe) {
				rr := r.(*spec.Diff)
				rr.Lang = lang
				rr.Release1 = fmt.Sprintf("%d", release-1)
				rr.FilePath = filepath.Join(path, "changes.md")
			})
			if err != nil {
				l.Error("Failed to generate documents", zap.Error(err))
				return err
			}
		}
	}

	{
		cl := c.(app_control_launcher.ControlLauncher)
		cat := cl.Catalogue()
		l.Info("Verify recipes")
		for _, r := range cat.Recipes() {
			spec := rc_spec.New(r)
			for _, m := range spec.Messages() {
				l.Debug("message", zap.String("key", m.Key()), zap.String("text", c.UI().Text(m)))
			}
		}

		l.Info("Verify ingredients")
		for _, r := range cat.Ingredients() {
			spec := rc_spec.New(r)
			for _, m := range spec.Messages() {
				l.Debug("message", zap.String("key", m.Key()), zap.String("text", c.UI().Text(m)))
			}
		}

		l.Info("Verify message objects")
		for _, m := range cat.Messages() {
			msgs := app_msg.Messages(m)
			for _, msg := range msgs {
				l.Debug("message", zap.String("key", msg.Key()), zap.String("text", c.UI().Text(msg)))
			}
		}
	}

	l.Info("Verify message resources")
	qm := c.Messages().(app_msg_container.Quality)
	missing := qm.MissingKeys()
	if len(missing) > 0 {
		suggested := make([]string, 0)
		for _, k := range missing {
			l.Error("Key missing", zap.String("key", k))
			suggested = append(suggested, "\""+k+"\":\"\",")
		}
		sort.Strings(suggested)
		fmt.Println(strings.Join(suggested, "\n"))
	}

	return qt_messages.VerifyMessages(c)
}
<|endoftext|>


<commit_before>package libbroadcast

import (
	"log"
)

var globalBroadcastManager *BroadcastManager

type BroadcastManager struct {
	broadcasters map[string]*Broadcaster
}

func init() {
	globalBroadcastManager = &BroadcastManager{
		broadcasters: make(map[string]*Broadcaster),
	}
}

func Global() *BroadcastManager {
	return globalBroadcastManager
}

func (broadcastManager *BroadcastManager) CreateBroadcaster(name string) *Broadcaster {
	// don't allow duplicated names to be created (since that will be a bad situation)
	if b, ok := broadcastManager.broadcasters[name]; ok {
		return b
	}

	broadcaster := newBroadcaster()
	broadcastManager.broadcasters[name] = broadcaster
	return broadcaster
}

func (broadcastManager *BroadcastManager) GetBroadcaster(name string) *Broadcaster {
	return broadcastManager.CreateBroadcaster(name)
}

func (broadcastManager *BroadcastManager) Destroy(name string) {
	delete(broadcastManager.broadcasters, name)
}
<commit_msg>Remove log import<commit_after>package libbroadcast

var globalBroadcastManager *BroadcastManager

type BroadcastManager struct {
	broadcasters map[string]*Broadcaster
}

func init() {
	globalBroadcastManager = &BroadcastManager{
		broadcasters: make(map[string]*Broadcaster),
	}
}

func Global() *BroadcastManager {
	return globalBroadcastManager
}

func (broadcastManager *BroadcastManager) CreateBroadcaster(name string) *Broadcaster {
	// don't allow duplicated names to be created (since that will be a bad situation)
	if b, ok := broadcastManager.broadcasters[name]; ok {
		return b
	}

	broadcaster := newBroadcaster()
	broadcastManager.broadcasters[name] = broadcaster
	return broadcaster
}

func (broadcastManager *BroadcastManager) GetBroadcaster(name string) *Broadcaster {
	return broadcastManager.CreateBroadcaster(name)
}

func (broadcastManager *BroadcastManager) Destroy(name string) {
	delete(broadcastManager.broadcasters, name)
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*

Gc is the generic label for the family of Go compilers
that function as part of the (modified) Plan 9 tool chain.  The C compiler
documentation at

	http://plan9.bell-labs.com/sys/doc/comp.pdf     (Tools overview)
	http://plan9.bell-labs.com/sys/doc/compiler.pdf (C compiler architecture)

gives the overall design of the tool chain.  Aside from a few adapted pieces,
such as the optimizer, the Go compilers are wholly new programs.

The compiler reads in a set of Go files, typically suffixed ".go".  They
must all be part of one package.  The output is a single intermediate file
representing the "binary assembly" of the compiled package, ready as input
for the linker (6l, etc.).

The generated files contain type information about the symbols exported by
the package and about types used by symbols imported by the package from
other packages. It is therefore not necessary when compiling client C of
package P to read the files of P's dependencies, only the compiled output
of P.

Usage:
	6g [flags] file...
The specified files must be Go source files and all part of the same package.
Substitute 6g with 8g or 5g where appropriate.

Flags:
	-o file
		output file, default 6.out for 6g, etc.
	-e
		normally the compiler quits after 10 errors; -e prints all errors
	-L
		show entire file path when printing line numbers in errors
	-I dir1 -I dir2
		add dir1 and dir2 to the list of paths to check for imported packages
	-N
		disable optimization
	-S
		write assembly language text to standard output
	-V
		print the compiler version

There are also a number of debugging flags; run the command with no arguments
to get a usage message.

*/
package documentation
<commit_msg>gc: doc nit<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*

Gc is the generic label for the family of Go compilers
that function as part of the (modified) Plan 9 tool chain.  The C compiler
documentation at

	http://plan9.bell-labs.com/sys/doc/comp.pdf     (Tools overview)
	http://plan9.bell-labs.com/sys/doc/compiler.pdf (C compiler architecture)

gives the overall design of the tool chain.  Aside from a few adapted pieces,
such as the optimizer, the Go compilers are wholly new programs.

The compiler reads in a set of Go files, typically suffixed ".go".  They
must all be part of one package.  The output is a single intermediate file
representing the "binary assembly" of the compiled package, ready as input
for the linker (6l, etc.).

The generated files contain type information about the symbols exported by
the package and about types used by symbols imported by the package from
other packages. It is therefore not necessary when compiling client C of
package P to read the files of P's dependencies, only the compiled output
of P.

Usage:
	6g [flags] file...
The specified files must be Go source files and all part of the same package.
Substitute 6g with 8g or 5g where appropriate.

Flags:
	-o file
		output file, default file.6 for 6g, etc.
	-e
		normally the compiler quits after 10 errors; -e prints all errors
	-L
		show entire file path when printing line numbers in errors
	-I dir1 -I dir2
		add dir1 and dir2 to the list of paths to check for imported packages
	-N
		disable optimization
	-S
		write assembly language text to standard output
	-V
		print the compiler version

There are also a number of debugging flags; run the command with no arguments
to get a usage message.

*/
package documentation
<|endoftext|>


<commit_before>//    Gonews is a webapp that provides a forum where users can post and discuss links
//
//    Copyright (C) 2016  mparaiso <mparaiso@online.fr>
//
//    This program is free software: you can redistribute it and/or modify
//    it under the terms of the GNU Affero General Public License as published
//    by the Free Software Foundation, either version 3 of the License, or
//    (at your option) any later version.
//
//    This program is distributed in the hope that it will be useful,
//    but WITHOUT ANY WARRANTY; without even the implied warranty of
//    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
//    GNU Affero General Public License for more details.
//
//    You should have received a copy of the GNU Affero General Public License
//    along with this program.  If not, see <http://www.gnu.org/licenses/>.

package gonews

import (
	// "fmt"
	"github.com/gorilla/securecookie"
	"golang.org/x/net/xsrftoken"
)

const CsrfSessionKey = "csrf-unique-id"

// CSRFGenerator generates and validate csrf tokens
type CSRFGenerator interface {
	Generate(actionID string) string
	Valid(token, actionID string) bool
}

// DefaultCSRFProvider implements CSRFProvider
type DefaultCSRFGenerator struct {
	Session SessionWrapper
	Secret  string
}

// Generate generates a new token
func (d *DefaultCSRFGenerator) Generate(actionID string) string {
	if _, ok := d.Session.Get(CsrfSessionKey).(string); !d.Session.Has(CsrfSessionKey) || !ok {
		d.Session.Set(CsrfSessionKey, string(securecookie.GenerateRandomKey(16)))
	}
	t := xsrftoken.Generate(d.Secret, d.Session.Get(CsrfSessionKey).(string), actionID)
	return t
}

// Valid valides a token
func (d *DefaultCSRFGenerator) Valid(token, actionID string) bool {
	userUniqueId, ok := d.Session.Get(CsrfSessionKey).(string)
	if !ok {
		return false
	}
	return xsrftoken.Valid(token, d.Secret, userUniqueId, actionID)
}
<commit_msg>using base64 for unique key encoding<commit_after>//    Gonews is a webapp that provides a forum where users can post and discuss links
//
//    Copyright (C) 2016  mparaiso <mparaiso@online.fr>
//
//    This program is free software: you can redistribute it and/or modify
//    it under the terms of the GNU Affero General Public License as published
//    by the Free Software Foundation, either version 3 of the License, or
//    (at your option) any later version.
//
//    This program is distributed in the hope that it will be useful,
//    but WITHOUT ANY WARRANTY; without even the implied warranty of
//    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
//    GNU Affero General Public License for more details.
//
//    You should have received a copy of the GNU Affero General Public License
//    along with this program.  If not, see <http://www.gnu.org/licenses/>.

package gonews

import (
	// "fmt"
	"encoding/base64"

	"github.com/gorilla/securecookie"
	"golang.org/x/net/xsrftoken"
)

const CsrfSessionKey = "csrf-unique-id"

// CSRFGenerator generates and validate csrf tokens
type CSRFGenerator interface {
	Generate(actionID string) string
	Valid(token, actionID string) bool
}

// DefaultCSRFProvider implements CSRFProvider
type DefaultCSRFGenerator struct {
	Session SessionWrapper
	Secret  string
}

// Generate generates a new token
func (d *DefaultCSRFGenerator) Generate(actionID string) string {
	if _, ok := d.Session.Get(CsrfSessionKey).(string); !d.Session.Has(CsrfSessionKey) || !ok {
		d.Session.Set(CsrfSessionKey, base64.StdEncoding.EncodeToString(securecookie.GenerateRandomKey(16)))
	}
	t := xsrftoken.Generate(d.Secret, d.Session.Get(CsrfSessionKey).(string), actionID)
	return t
}

// Valid valides a token
func (d *DefaultCSRFGenerator) Valid(token, actionID string) bool {
	userUniqueId, ok := d.Session.Get(CsrfSessionKey).(string)
	if !ok {
		return false
	}
	return xsrftoken.Valid(token, d.Secret, userUniqueId, actionID)
}
<|endoftext|>


<commit_before>package core

import "encoding/binary"

const (
	SEQEND       = 0
	LITERAL_EXPR = 1
	VECTOR_EXPR  = 2
	MAP_EXPR     = 3
	SET_EXPR     = 4
	IF_EXPR      = 5
	DEF_EXPR     = 6
	CALL_EXPR    = 7
	INT          = 100
)

type (
	PackEnv struct {
		Strings         map[*string]uint16
		nextStringIndex uint16
	}

	PackHeader struct {
		Strings []*string
	}
)

func (env *PackEnv) stringIndex(s *string) uint16 {
	index, ok := env.Strings[s]
	if ok {
		return index
	}
	env.Strings[s] = env.nextStringIndex
	env.nextStringIndex++
	return env.nextStringIndex - 1
}

func appendBool(p []byte, b bool) []byte {
	var bb byte
	if b {
		bb = 1
	}
	return append(p, bb)
}

func extractBool(p []byte) (bool, []byte) {
	var b bool
	if p[0] == 1 {
		b = true
	}
	return b, p[1:]
}

func appendUint16(p []byte, i uint16) []byte {
	pp := make([]byte, 2)
	binary.BigEndian.PutUint16(pp, i)
	p = append(p, pp...)
	return p
}

func extractUInt16(p []byte) (uint16, []byte) {
	return binary.BigEndian.Uint16(p[0:2]), p[2:]
}

func appendUint32(p []byte, i uint32) []byte {
	pp := make([]byte, 4)
	binary.BigEndian.PutUint32(pp, i)
	p = append(p, pp...)
	return p
}

func extractUInt32(p []byte) (uint32, []byte) {
	return binary.BigEndian.Uint32(p[0:4]), p[4:]
}

func appendInt(p []byte, i int) []byte {
	pp := make([]byte, 8)
	binary.BigEndian.PutUint64(pp, uint64(i))
	p = append(p, pp...)
	return p
}

func extractInt(p []byte) (int, []byte) {
	return int(binary.BigEndian.Uint64(p[0:8])), p[8:]
}

func (pos Position) Pack(p []byte, env *PackEnv) []byte {
	p = appendInt(p, pos.startLine)
	p = appendInt(p, pos.endLine)
	p = appendInt(p, pos.startColumn)
	p = appendInt(p, pos.endColumn)
	p = appendUint16(p, env.stringIndex(pos.filename))
	return p
}

func unpackPosition(p []byte, header *PackHeader) (pos Position, pp []byte) {
	pos.startLine, p = extractInt(p)
	pos.endLine, p = extractInt(p)
	pos.startColumn, p = extractInt(p)
	pos.endColumn, p = extractInt(p)
	i, p := extractUInt16(p)
	pos.filename = header.Strings[i]
	return pos, p
}

func (i Int) Pack(p []byte, env *PackEnv) []byte {
	p = i.info.Pack(p, env)
	p = appendInt(p, i.I)
	return p
}

func unpackInt(p []byte) (Int, []byte) {
	i, p := extractInt(p)
	return MakeInt(i), p
}

func (info *ObjectInfo) Pack(p []byte, env *PackEnv) []byte {
	return info.Pos().Pack(p, env)
}

func unpackObjectInfo(p []byte, header *PackHeader) (*ObjectInfo, []byte) {
	pos, p := unpackPosition(p, header)
	return &ObjectInfo{Position: pos}, p
}

func (s Symbol) Pack(p []byte, env *PackEnv) []byte {
	p = s.info.Pack(p, env)
	p = appendUint16(p, env.stringIndex(s.name))
	p = appendUint16(p, env.stringIndex(s.ns))
	p = appendUint32(p, s.hash)
	return p
}

func unpackSymbol(p []byte, header *PackHeader) (Symbol, []byte) {
	info, p := unpackObjectInfo(p, header)
	iname, p := extractUInt16(p)
	ins, p := extractUInt16(p)
	hash, p := extractUInt32(p)
	res := Symbol{
		InfoHolder: InfoHolder{info: info},
		name:       header.Strings[iname],
		ns:         header.Strings[ins],
		hash:       hash,
	}
	return res, p
}

func unpackObject(p []byte, header *PackHeader) (Object, []byte) {
	switch p[0] {
	case INT:
		return unpackInt(p[1:])
	default:
		panic(RT.NewError("Unknown pack tag"))
	}
}

func (expr *LiteralExpr) Pack(p []byte, env *PackEnv) []byte {
	p = append(p, LITERAL_EXPR)
	p = expr.Pos().Pack(p, env)
	p = appendBool(p, expr.isSurrogate)
	p = expr.obj.Pack(p, env)
	return p
}

func unpackLiteralExpr(p []byte, header *PackHeader) (*LiteralExpr, []byte) {
	pos, p := unpackPosition(p, header)
	isSurrogate, p := extractBool(p)
	obj, p := unpackObject(p, header)
	res := &LiteralExpr{
		obj:         obj,
		Position:    pos,
		isSurrogate: isSurrogate,
	}
	return res, p
}

func packSeq(p []byte, s []Expr, env *PackEnv) []byte {
	for _, e := range s {
		p = e.Pack(p, env)
	}
	p = append(p, SEQEND)
	return p
}

func unpackSeq(p []byte, header *PackHeader) ([]Expr, []byte) {
	var res []Expr
	for p[0] != SEQEND {
		var e Expr
		e, p = UnpackExpr(p, header)
		res = append(res, e)
	}
	return res, p[1:]
}

func (expr *VectorExpr) Pack(p []byte, env *PackEnv) []byte {
	p = append(p, VECTOR_EXPR)
	p = expr.Pos().Pack(p, env)
	return packSeq(p, expr.v, env)
}

func unpackSetExpr(p []byte, header *PackHeader) (*SetExpr, []byte) {
	pos, p := unpackPosition(p, header)
	v, p := unpackSeq(p, header)
	res := &SetExpr{
		Position: pos,
		elements: v,
	}
	return res, p
}

func (expr *SetExpr) Pack(p []byte, env *PackEnv) []byte {
	p = append(p, SET_EXPR)
	p = expr.Pos().Pack(p, env)
	return packSeq(p, expr.elements, env)
}

func unpackVectorExpr(p []byte, header *PackHeader) (*VectorExpr, []byte) {
	pos, p := unpackPosition(p, header)
	v, p := unpackSeq(p, header)
	res := &VectorExpr{
		Position: pos,
		v:        v,
	}
	return res, p
}

func (expr *MapExpr) Pack(p []byte, env *PackEnv) []byte {
	p = append(p, MAP_EXPR)
	p = expr.Pos().Pack(p, env)
	p = packSeq(p, expr.keys, env)
	p = packSeq(p, expr.values, env)
	return p
}

func unpackMapExpr(p []byte, header *PackHeader) (*MapExpr, []byte) {
	pos, p := unpackPosition(p, header)
	ks, p := unpackSeq(p, header)
	vs, p := unpackSeq(p, header)
	res := &MapExpr{
		Position: pos,
		keys:     ks,
		values:   vs,
	}
	return res, p
}

func (expr *IfExpr) Pack(p []byte, env *PackEnv) []byte {
	p = append(p, IF_EXPR)
	p = expr.Pos().Pack(p, env)
	p = expr.cond.Pack(p, env)
	p = expr.positive.Pack(p, env)
	p = expr.negative.Pack(p, env)
	return p
}

func unpackIfExpr(p []byte, header *PackHeader) (*IfExpr, []byte) {
	pos, p := unpackPosition(p, header)
	cond, p := UnpackExpr(p, header)
	positive, p := UnpackExpr(p, header)
	negative, p := UnpackExpr(p, header)
	res := &IfExpr{
		Position: pos,
		positive: positive,
		negative: negative,
		cond:     cond,
	}
	return res, p
}

func (expr *DefExpr) Pack(p []byte, env *PackEnv) []byte {
	p = append(p, DEF_EXPR)
	p = expr.Pos().Pack(p, env)
	p = expr.name.Pack(p, env)
	p = expr.value.Pack(p, env)
	p = expr.meta.Pack(p, env)
	return p
}

func unpackDefExpr(p []byte, header *PackHeader) (*DefExpr, []byte) {
	pos, p := unpackPosition(p, header)
	name, p := unpackSymbol(p, header)
	value, p := UnpackExpr(p, header)
	meta, p := UnpackExpr(p, header)
	res := &DefExpr{
		Position: pos,
		name:     name,
		value:    value,
		meta:     meta,
	}
	return res, p
}

func (expr *CallExpr) Pack(p []byte, env *PackEnv) []byte {
	p = append(p, CALL_EXPR)
	p = expr.Pos().Pack(p, env)
	p = expr.callable.Pack(p, env)
	p = packSeq(p, expr.args, env)
	return p
}

func unpackCallExpr(p []byte, header *PackHeader) (*CallExpr, []byte) {
	pos, p := unpackPosition(p, header)
	callable, p := UnpackExpr(p, header)
	args, p := unpackSeq(p, header)
	res := &CallExpr{
		Position: pos,
		callable: callable,
		args:     args,
	}
	return res, p
}

func UnpackExpr(p []byte, header *PackHeader) (Expr, []byte) {
	switch p[0] {
	case LITERAL_EXPR:
		return unpackLiteralExpr(p[1:], header)
	case VECTOR_EXPR:
		return unpackVectorExpr(p[1:], header)
	case MAP_EXPR:
		return unpackMapExpr(p[1:], header)
	case SET_EXPR:
		return unpackSetExpr(p[1:], header)
	case IF_EXPR:
		return unpackIfExpr(p[1:], header)
	case DEF_EXPR:
		return unpackDefExpr(p[1:], header)
	case CALL_EXPR:
		return unpackCallExpr(p[1:], header)
	default:
		panic(RT.NewError("Unknown pack tag"))
	}
}
<commit_msg>Pack RecurExpr, MetaExpr<commit_after>package core

import "encoding/binary"

const (
	SEQEND       = 0
	LITERAL_EXPR = 1
	VECTOR_EXPR  = 2
	MAP_EXPR     = 3
	SET_EXPR     = 4
	IF_EXPR      = 5
	DEF_EXPR     = 6
	CALL_EXPR    = 7
	RECUR_EXPR   = 8
	META_EXPR    = 9
	INT          = 100
)

type (
	PackEnv struct {
		Strings         map[*string]uint16
		nextStringIndex uint16
	}

	PackHeader struct {
		Strings []*string
	}
)

func (env *PackEnv) stringIndex(s *string) uint16 {
	index, ok := env.Strings[s]
	if ok {
		return index
	}
	env.Strings[s] = env.nextStringIndex
	env.nextStringIndex++
	return env.nextStringIndex - 1
}

func appendBool(p []byte, b bool) []byte {
	var bb byte
	if b {
		bb = 1
	}
	return append(p, bb)
}

func extractBool(p []byte) (bool, []byte) {
	var b bool
	if p[0] == 1 {
		b = true
	}
	return b, p[1:]
}

func appendUint16(p []byte, i uint16) []byte {
	pp := make([]byte, 2)
	binary.BigEndian.PutUint16(pp, i)
	p = append(p, pp...)
	return p
}

func extractUInt16(p []byte) (uint16, []byte) {
	return binary.BigEndian.Uint16(p[0:2]), p[2:]
}

func appendUint32(p []byte, i uint32) []byte {
	pp := make([]byte, 4)
	binary.BigEndian.PutUint32(pp, i)
	p = append(p, pp...)
	return p
}

func extractUInt32(p []byte) (uint32, []byte) {
	return binary.BigEndian.Uint32(p[0:4]), p[4:]
}

func appendInt(p []byte, i int) []byte {
	pp := make([]byte, 8)
	binary.BigEndian.PutUint64(pp, uint64(i))
	p = append(p, pp...)
	return p
}

func extractInt(p []byte) (int, []byte) {
	return int(binary.BigEndian.Uint64(p[0:8])), p[8:]
}

func (pos Position) Pack(p []byte, env *PackEnv) []byte {
	p = appendInt(p, pos.startLine)
	p = appendInt(p, pos.endLine)
	p = appendInt(p, pos.startColumn)
	p = appendInt(p, pos.endColumn)
	p = appendUint16(p, env.stringIndex(pos.filename))
	return p
}

func unpackPosition(p []byte, header *PackHeader) (pos Position, pp []byte) {
	pos.startLine, p = extractInt(p)
	pos.endLine, p = extractInt(p)
	pos.startColumn, p = extractInt(p)
	pos.endColumn, p = extractInt(p)
	i, p := extractUInt16(p)
	pos.filename = header.Strings[i]
	return pos, p
}

func (i Int) Pack(p []byte, env *PackEnv) []byte {
	p = i.info.Pack(p, env)
	p = appendInt(p, i.I)
	return p
}

func unpackInt(p []byte) (Int, []byte) {
	i, p := extractInt(p)
	return MakeInt(i), p
}

func (info *ObjectInfo) Pack(p []byte, env *PackEnv) []byte {
	return info.Pos().Pack(p, env)
}

func unpackObjectInfo(p []byte, header *PackHeader) (*ObjectInfo, []byte) {
	pos, p := unpackPosition(p, header)
	return &ObjectInfo{Position: pos}, p
}

func (s Symbol) Pack(p []byte, env *PackEnv) []byte {
	p = s.info.Pack(p, env)
	p = appendUint16(p, env.stringIndex(s.name))
	p = appendUint16(p, env.stringIndex(s.ns))
	p = appendUint32(p, s.hash)
	return p
}

func unpackSymbol(p []byte, header *PackHeader) (Symbol, []byte) {
	info, p := unpackObjectInfo(p, header)
	iname, p := extractUInt16(p)
	ins, p := extractUInt16(p)
	hash, p := extractUInt32(p)
	res := Symbol{
		InfoHolder: InfoHolder{info: info},
		name:       header.Strings[iname],
		ns:         header.Strings[ins],
		hash:       hash,
	}
	return res, p
}

func unpackObject(p []byte, header *PackHeader) (Object, []byte) {
	switch p[0] {
	case INT:
		return unpackInt(p[1:])
	default:
		panic(RT.NewError("Unknown pack tag"))
	}
}

func (expr *LiteralExpr) Pack(p []byte, env *PackEnv) []byte {
	p = append(p, LITERAL_EXPR)
	p = expr.Pos().Pack(p, env)
	p = appendBool(p, expr.isSurrogate)
	p = expr.obj.Pack(p, env)
	return p
}

func unpackLiteralExpr(p []byte, header *PackHeader) (*LiteralExpr, []byte) {
	pos, p := unpackPosition(p, header)
	isSurrogate, p := extractBool(p)
	obj, p := unpackObject(p, header)
	res := &LiteralExpr{
		obj:         obj,
		Position:    pos,
		isSurrogate: isSurrogate,
	}
	return res, p
}

func packSeq(p []byte, s []Expr, env *PackEnv) []byte {
	for _, e := range s {
		p = e.Pack(p, env)
	}
	p = append(p, SEQEND)
	return p
}

func unpackSeq(p []byte, header *PackHeader) ([]Expr, []byte) {
	var res []Expr
	for p[0] != SEQEND {
		var e Expr
		e, p = UnpackExpr(p, header)
		res = append(res, e)
	}
	return res, p[1:]
}

func (expr *VectorExpr) Pack(p []byte, env *PackEnv) []byte {
	p = append(p, VECTOR_EXPR)
	p = expr.Pos().Pack(p, env)
	return packSeq(p, expr.v, env)
}

func unpackSetExpr(p []byte, header *PackHeader) (*SetExpr, []byte) {
	pos, p := unpackPosition(p, header)
	v, p := unpackSeq(p, header)
	res := &SetExpr{
		Position: pos,
		elements: v,
	}
	return res, p
}

func (expr *SetExpr) Pack(p []byte, env *PackEnv) []byte {
	p = append(p, SET_EXPR)
	p = expr.Pos().Pack(p, env)
	return packSeq(p, expr.elements, env)
}

func unpackVectorExpr(p []byte, header *PackHeader) (*VectorExpr, []byte) {
	pos, p := unpackPosition(p, header)
	v, p := unpackSeq(p, header)
	res := &VectorExpr{
		Position: pos,
		v:        v,
	}
	return res, p
}

func (expr *MapExpr) Pack(p []byte, env *PackEnv) []byte {
	p = append(p, MAP_EXPR)
	p = expr.Pos().Pack(p, env)
	p = packSeq(p, expr.keys, env)
	p = packSeq(p, expr.values, env)
	return p
}

func unpackMapExpr(p []byte, header *PackHeader) (*MapExpr, []byte) {
	pos, p := unpackPosition(p, header)
	ks, p := unpackSeq(p, header)
	vs, p := unpackSeq(p, header)
	res := &MapExpr{
		Position: pos,
		keys:     ks,
		values:   vs,
	}
	return res, p
}

func (expr *IfExpr) Pack(p []byte, env *PackEnv) []byte {
	p = append(p, IF_EXPR)
	p = expr.Pos().Pack(p, env)
	p = expr.cond.Pack(p, env)
	p = expr.positive.Pack(p, env)
	p = expr.negative.Pack(p, env)
	return p
}

func unpackIfExpr(p []byte, header *PackHeader) (*IfExpr, []byte) {
	pos, p := unpackPosition(p, header)
	cond, p := UnpackExpr(p, header)
	positive, p := UnpackExpr(p, header)
	negative, p := UnpackExpr(p, header)
	res := &IfExpr{
		Position: pos,
		positive: positive,
		negative: negative,
		cond:     cond,
	}
	return res, p
}

func (expr *DefExpr) Pack(p []byte, env *PackEnv) []byte {
	p = append(p, DEF_EXPR)
	p = expr.Pos().Pack(p, env)
	p = expr.name.Pack(p, env)
	p = expr.value.Pack(p, env)
	p = expr.meta.Pack(p, env)
	return p
}

func unpackDefExpr(p []byte, header *PackHeader) (*DefExpr, []byte) {
	pos, p := unpackPosition(p, header)
	name, p := unpackSymbol(p, header)
	value, p := UnpackExpr(p, header)
	meta, p := UnpackExpr(p, header)
	res := &DefExpr{
		Position: pos,
		name:     name,
		value:    value,
		meta:     meta,
	}
	return res, p
}

func (expr *CallExpr) Pack(p []byte, env *PackEnv) []byte {
	p = append(p, CALL_EXPR)
	p = expr.Pos().Pack(p, env)
	p = expr.callable.Pack(p, env)
	p = packSeq(p, expr.args, env)
	return p
}

func unpackCallExpr(p []byte, header *PackHeader) (*CallExpr, []byte) {
	pos, p := unpackPosition(p, header)
	callable, p := UnpackExpr(p, header)
	args, p := unpackSeq(p, header)
	res := &CallExpr{
		Position: pos,
		callable: callable,
		args:     args,
	}
	return res, p
}

func (expr *RecurExpr) Pack(p []byte, env *PackEnv) []byte {
	p = append(p, RECUR_EXPR)
	p = expr.Pos().Pack(p, env)
	p = packSeq(p, expr.args, env)
	return p
}

func unpackRecurExpr(p []byte, header *PackHeader) (*RecurExpr, []byte) {
	pos, p := unpackPosition(p, header)
	args, p := unpackSeq(p, header)
	res := &RecurExpr{
		Position: pos,
		args:     args,
	}
	return res, p
}

func (expr *MetaExpr) Pack(p []byte, env *PackEnv) []byte {
	p = append(p, META_EXPR)
	p = expr.Pos().Pack(p, env)
	p = expr.meta.Pack(p, env)
	p = expr.expr.Pack(p, env)
	return p
}

func unpackMetaExpr(p []byte, header *PackHeader) (*MetaExpr, []byte) {
	pos, p := unpackPosition(p, header)
	meta, p := unpackMapExpr(p, header)
	expr, p := UnpackExpr(p, header)
	res := &MetaExpr{
		Position: pos,
		meta:     meta,
		expr:     expr,
	}
	return res, p
}

func UnpackExpr(p []byte, header *PackHeader) (Expr, []byte) {
	switch p[0] {
	case LITERAL_EXPR:
		return unpackLiteralExpr(p[1:], header)
	case VECTOR_EXPR:
		return unpackVectorExpr(p[1:], header)
	case MAP_EXPR:
		return unpackMapExpr(p[1:], header)
	case SET_EXPR:
		return unpackSetExpr(p[1:], header)
	case IF_EXPR:
		return unpackIfExpr(p[1:], header)
	case DEF_EXPR:
		return unpackDefExpr(p[1:], header)
	case CALL_EXPR:
		return unpackCallExpr(p[1:], header)
	case RECUR_EXPR:
		return unpackRecurExpr(p[1:], header)
	case META_EXPR:
		return unpackMetaExpr(p[1:], header)
	default:
		panic(RT.NewError("Unknown pack tag"))
	}
}
<|endoftext|>