<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Build toolchain using Go 1.4.
//
// The general strategy is to copy the source files we need into
// a new GOPATH workspace, adjust import paths appropriately,
// invoke the Go 1.4 go command to build those sources,
// and then copy the binaries back.

package main

import (
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
)

// bootstrapDirs is a list of directories holding code that must be
// compiled with a Go 1.4 toolchain to produce the bootstrapTargets.
// All directories in this list are relative to and must be below $GOROOT/src.
//
// The list has two kinds of entries: names beginning with cmd/ with
// no other slashes, which are commands, and other paths, which are packages
// supporting the commands. Packages in the standard library can be listed
// if a newer copy needs to be substituted for the Go 1.4 copy when used
// by the command packages. Paths ending with /... automatically
// include all packages within subdirectories as well.
// These will be imported during bootstrap as bootstrap/name, like bootstrap/math/big.
var bootstrapDirs = []string{
	"cmd/asm",
	"cmd/asm/internal/...",
	"cmd/cgo",
	"cmd/compile",
	"cmd/compile/internal/...",
	"cmd/internal/archive",
	"cmd/internal/bio",
	"cmd/internal/codesign",
	"cmd/internal/dwarf",
	"cmd/internal/edit",
	"cmd/internal/gcprog",
	"cmd/internal/goobj",
	"cmd/internal/notsha256",
	"cmd/internal/obj/...",
	"cmd/internal/objabi",
	"cmd/internal/pkgpath",
	"cmd/internal/quoted",
	"cmd/internal/src",
	"cmd/internal/sys",
	"cmd/link",
	"cmd/link/internal/...",
	"compress/flate",
	"compress/zlib",
	"container/heap",
	"debug/dwarf",
	"debug/elf",
	"debug/macho",
	"debug/pe",
	"go/constant",
	"internal/buildcfg",
	"internal/goexperiment",
	"internal/goversion",
	"internal/pkgbits",
	"internal/race",
	"internal/unsafeheader",
	"internal/xcoff",
	"math/big",
	"math/bits",
	"sort",
	"strconv",
}

// File prefixes that are ignored by go/build anyway, and cause
// problems with editor generated temporary files (#18931).
var ignorePrefixes = []string{
	".",
	"_",
	"#",
}

// File suffixes that use build tags introduced since Go 1.4.
// These must not be copied into the bootstrap build directory.
// Also ignore test files.
var ignoreSuffixes = []string{
	"_arm64.s",
	"_arm64.go",
	"_loong64.s",
	"_loong64.go",
	"_riscv64.s",
	"_riscv64.go",
	"_wasm.s",
	"_wasm.go",
	"_test.s",
	"_test.go",
}

var tryDirs = []string{
	"sdk/go1.17",
	"go1.17",
}

func bootstrapBuildTools() {
	goroot_bootstrap := os.Getenv("GOROOT_BOOTSTRAP")
	if goroot_bootstrap == "" {
		home := os.Getenv("HOME")
		goroot_bootstrap = pathf("%s/go1.4", home)
		for _, d := range tryDirs {
			if p := pathf("%s/%s", home, d); isdir(p) {
				goroot_bootstrap = p
			}
		}
	}
	xprintf("Building Go toolchain1 using %s.\n", goroot_bootstrap)

	mkbuildcfg(pathf("%s/src/internal/buildcfg/zbootstrap.go", goroot))
	mkobjabi(pathf("%s/src/cmd/internal/objabi/zbootstrap.go", goroot))

	// Use $GOROOT/pkg/bootstrap as the bootstrap workspace root.
	// We use a subdirectory of $GOROOT/pkg because that's the
	// space within $GOROOT where we store all generated objects.
	// We could use a temporary directory outside $GOROOT instead,
	// but it is easier to debug on failure if the files are in a known location.
	workspace := pathf("%s/pkg/bootstrap", goroot)
	xremoveall(workspace)
	xatexit(func() { xremoveall(workspace) })
	base := pathf("%s/src/bootstrap", workspace)
	xmkdirall(base)

	// Copy source code into $GOROOT/pkg/bootstrap and rewrite import paths.
	writefile("module bootstrap\n", pathf("%s/%s", base, "go.mod"), 0)
	for _, dir := range bootstrapDirs {
		recurse := strings.HasSuffix(dir, "/...")
		dir = strings.TrimSuffix(dir, "/...")
		filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
			if err != nil {
				fatalf("walking bootstrap dirs failed: %v: %v", path, err)
			}

			name := filepath.Base(path)
			src := pathf("%s/src/%s", goroot, path)
			dst := pathf("%s/%s", base, path)

			if info.IsDir() {
				if !recurse && path != dir || name == "testdata" {
					return filepath.SkipDir
				}

				xmkdirall(dst)
				if path == "cmd/cgo" {
					// Write to src because we need the file both for bootstrap
					// and for later in the main build.
					mkzdefaultcc("", pathf("%s/zdefaultcc.go", src))
					mkzdefaultcc("", pathf("%s/zdefaultcc.go", dst))
				}
				return nil
			}

			for _, pre := range ignorePrefixes {
				if strings.HasPrefix(name, pre) {
					return nil
				}
			}
			for _, suf := range ignoreSuffixes {
				if strings.HasSuffix(name, suf) {
					return nil
				}
			}

			text := bootstrapRewriteFile(src)
			writefile(text, dst, 0)
			return nil
		})
	}

	// Set up environment for invoking Go 1.4 go command.
	// GOROOT points at Go 1.4 GOROOT,
	// GOPATH points at our bootstrap workspace,
	// GOBIN is empty, so that binaries are installed to GOPATH/bin,
	// and GOOS, GOHOSTOS, GOARCH, and GOHOSTOS are empty,
	// so that Go 1.4 builds whatever kind of binary it knows how to build.
	// Restore GOROOT, GOPATH, and GOBIN when done.
	// Don't bother with GOOS, GOHOSTOS, GOARCH, and GOHOSTARCH,
	// because setup will take care of those when bootstrapBuildTools returns.

	defer os.Setenv("GOROOT", os.Getenv("GOROOT"))
	os.Setenv("GOROOT", goroot_bootstrap)

	defer os.Setenv("GOPATH", os.Getenv("GOPATH"))
	os.Setenv("GOPATH", workspace)

	defer os.Setenv("GOBIN", os.Getenv("GOBIN"))
	os.Setenv("GOBIN", "")

	os.Setenv("GOOS", "")
	os.Setenv("GOHOSTOS", "")
	os.Setenv("GOARCH", "")
	os.Setenv("GOHOSTARCH", "")

	// Run Go 1.4 to build binaries. Use -gcflags=-l to disable inlining to
	// workaround bugs in Go 1.4's compiler. See discussion thread:
	// https://groups.google.com/d/msg/golang-dev/Ss7mCKsvk8w/Gsq7VYI0AwAJ
	// Use the math_big_pure_go build tag to disable the assembly in math/big
	// which may contain unsupported instructions.
	// Note that if we are using Go 1.10 or later as bootstrap, the -gcflags=-l
	// only applies to the final cmd/go binary, but that's OK: if this is Go 1.10
	// or later we don't need to disable inlining to work around bugs in the Go 1.4 compiler.
	cmd := []string{
		pathf("%s/bin/go", goroot_bootstrap),
		"install",
		"-gcflags=-l",
		"-tags=math_big_pure_go compiler_bootstrap",
	}
	if vflag > 0 {
		cmd = append(cmd, "-v")
	}
	if tool := os.Getenv("GOBOOTSTRAP_TOOLEXEC"); tool != "" {
		cmd = append(cmd, "-toolexec="+tool)
	}
	cmd = append(cmd, "bootstrap/cmd/...")
	run(base, ShowOutput|CheckExit, cmd...)

	// Copy binaries into tool binary directory.
	for _, name := range bootstrapDirs {
		if !strings.HasPrefix(name, "cmd/") {
			continue
		}
		name = name[len("cmd/"):]
		if !strings.Contains(name, "/") {
			copyfile(pathf("%s/%s%s", tooldir, name, exe), pathf("%s/bin/%s%s", workspace, name, exe), writeExec)
		}
	}

	if vflag > 0 {
		xprintf("\n")
	}
}

var ssaRewriteFileSubstring = filepath.FromSlash("src/cmd/compile/internal/ssa/rewrite")

// isUnneededSSARewriteFile reports whether srcFile is a
// src/cmd/compile/internal/ssa/rewriteARCHNAME.go file for an
// architecture that isn't for the current runtime.GOARCH.
//
// When unneeded is true archCaps is the rewrite base filename without
// the "rewrite" prefix or ".go" suffix: AMD64, 386, ARM, ARM64, etc.
func isUnneededSSARewriteFile(srcFile string) (archCaps string, unneeded bool) {
	if !strings.Contains(srcFile, ssaRewriteFileSubstring) {
		return "", false
	}
	fileArch := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(srcFile), "rewrite"), ".go")
	if fileArch == "" {
		return "", false
	}
	b := fileArch[0]
	if b == '_' || ('a' <= b && b <= 'z') {
		return "", false
	}
	archCaps = fileArch
	fileArch = strings.ToLower(fileArch)
	fileArch = strings.TrimSuffix(fileArch, "splitload")
	if fileArch == os.Getenv("GOHOSTARCH") {
		return "", false
	}
	if fileArch == strings.TrimSuffix(runtime.GOARCH, "le") {
		return "", false
	}
	if fileArch == strings.TrimSuffix(os.Getenv("GOARCH"), "le") {
		return "", false
	}
	return archCaps, true
}

func bootstrapRewriteFile(srcFile string) string {
	// During bootstrap, generate dummy rewrite files for
	// irrelevant architectures. We only need to build a bootstrap
	// binary that works for the current runtime.GOARCH.
	// This saves 6+ seconds of bootstrap.
	if archCaps, ok := isUnneededSSARewriteFile(srcFile); ok {
		return fmt.Sprintf(`// Code generated by go tool dist; DO NOT EDIT.

package ssa

func rewriteValue%s(v *Value) bool { panic("unused during bootstrap") }
func rewriteBlock%s(b *Block) bool { panic("unused during bootstrap") }
`, archCaps, archCaps)
	}

	return bootstrapFixImports(srcFile)
}

func bootstrapFixImports(srcFile string) string {
	text := readfile(srcFile)
	if !strings.Contains(srcFile, "/cmd/") && !strings.Contains(srcFile, `\cmd\`) {
		text = regexp.MustCompile(`\bany\b`).ReplaceAllString(text, "interface{}")
	}
	lines := strings.SplitAfter(text, "\n")
	inBlock := false
	for i, line := range lines {
		if strings.HasPrefix(line, "import (") {
			inBlock = true
			continue
		}
		if inBlock && strings.HasPrefix(line, ")") {
			inBlock = false
			continue
		}
		if strings.HasPrefix(line, `import "`) || strings.HasPrefix(line, `import . "`) ||
			inBlock && (strings.HasPrefix(line, "\t\"") || strings.HasPrefix(line, "\t. \"") || strings.HasPrefix(line, "\texec \"")) {
			line = strings.Replace(line, `"cmd/`, `"bootstrap/cmd/`, -1)
			for _, dir := range bootstrapDirs {
				if strings.HasPrefix(dir, "cmd/") {
					continue
				}
				line = strings.Replace(line, `"`+dir+`"`, `"bootstrap/`+dir+`"`, -1)
			}
			lines[i] = line
		}
	}

	lines[0] = "// Code generated by go tool dist; DO NOT EDIT.\n// This is a bootstrap copy of " + srcFile + "\n\n//line " + srcFile + ":1\n" + lines[0]

	return strings.Join(lines, "")
}
<commit_msg>cmd/dist: use gohostarch for ssa rewrite check<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Build toolchain using Go 1.4.
//
// The general strategy is to copy the source files we need into
// a new GOPATH workspace, adjust import paths appropriately,
// invoke the Go 1.4 go command to build those sources,
// and then copy the binaries back.

package main

import (
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

// bootstrapDirs is a list of directories holding code that must be
// compiled with a Go 1.4 toolchain to produce the bootstrapTargets.
// All directories in this list are relative to and must be below $GOROOT/src.
//
// The list has two kinds of entries: names beginning with cmd/ with
// no other slashes, which are commands, and other paths, which are packages
// supporting the commands. Packages in the standard library can be listed
// if a newer copy needs to be substituted for the Go 1.4 copy when used
// by the command packages. Paths ending with /... automatically
// include all packages within subdirectories as well.
// These will be imported during bootstrap as bootstrap/name, like bootstrap/math/big.
var bootstrapDirs = []string{
	"cmd/asm",
	"cmd/asm/internal/...",
	"cmd/cgo",
	"cmd/compile",
	"cmd/compile/internal/...",
	"cmd/internal/archive",
	"cmd/internal/bio",
	"cmd/internal/codesign",
	"cmd/internal/dwarf",
	"cmd/internal/edit",
	"cmd/internal/gcprog",
	"cmd/internal/goobj",
	"cmd/internal/notsha256",
	"cmd/internal/obj/...",
	"cmd/internal/objabi",
	"cmd/internal/pkgpath",
	"cmd/internal/quoted",
	"cmd/internal/src",
	"cmd/internal/sys",
	"cmd/link",
	"cmd/link/internal/...",
	"compress/flate",
	"compress/zlib",
	"container/heap",
	"debug/dwarf",
	"debug/elf",
	"debug/macho",
	"debug/pe",
	"go/constant",
	"internal/buildcfg",
	"internal/goexperiment",
	"internal/goversion",
	"internal/pkgbits",
	"internal/race",
	"internal/unsafeheader",
	"internal/xcoff",
	"math/big",
	"math/bits",
	"sort",
	"strconv",
}

// File prefixes that are ignored by go/build anyway, and cause
// problems with editor generated temporary files (#18931).
var ignorePrefixes = []string{
	".",
	"_",
	"#",
}

// File suffixes that use build tags introduced since Go 1.4.
// These must not be copied into the bootstrap build directory.
// Also ignore test files.
var ignoreSuffixes = []string{
	"_arm64.s",
	"_arm64.go",
	"_loong64.s",
	"_loong64.go",
	"_riscv64.s",
	"_riscv64.go",
	"_wasm.s",
	"_wasm.go",
	"_test.s",
	"_test.go",
}

var tryDirs = []string{
	"sdk/go1.17",
	"go1.17",
}

func bootstrapBuildTools() {
	goroot_bootstrap := os.Getenv("GOROOT_BOOTSTRAP")
	if goroot_bootstrap == "" {
		home := os.Getenv("HOME")
		goroot_bootstrap = pathf("%s/go1.4", home)
		for _, d := range tryDirs {
			if p := pathf("%s/%s", home, d); isdir(p) {
				goroot_bootstrap = p
			}
		}
	}
	xprintf("Building Go toolchain1 using %s.\n", goroot_bootstrap)

	mkbuildcfg(pathf("%s/src/internal/buildcfg/zbootstrap.go", goroot))
	mkobjabi(pathf("%s/src/cmd/internal/objabi/zbootstrap.go", goroot))

	// Use $GOROOT/pkg/bootstrap as the bootstrap workspace root.
	// We use a subdirectory of $GOROOT/pkg because that's the
	// space within $GOROOT where we store all generated objects.
	// We could use a temporary directory outside $GOROOT instead,
	// but it is easier to debug on failure if the files are in a known location.
	workspace := pathf("%s/pkg/bootstrap", goroot)
	xremoveall(workspace)
	xatexit(func() { xremoveall(workspace) })
	base := pathf("%s/src/bootstrap", workspace)
	xmkdirall(base)

	// Copy source code into $GOROOT/pkg/bootstrap and rewrite import paths.
	writefile("module bootstrap\n", pathf("%s/%s", base, "go.mod"), 0)
	for _, dir := range bootstrapDirs {
		recurse := strings.HasSuffix(dir, "/...")
		dir = strings.TrimSuffix(dir, "/...")
		filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
			if err != nil {
				fatalf("walking bootstrap dirs failed: %v: %v", path, err)
			}

			name := filepath.Base(path)
			src := pathf("%s/src/%s", goroot, path)
			dst := pathf("%s/%s", base, path)

			if info.IsDir() {
				if !recurse && path != dir || name == "testdata" {
					return filepath.SkipDir
				}

				xmkdirall(dst)
				if path == "cmd/cgo" {
					// Write to src because we need the file both for bootstrap
					// and for later in the main build.
					mkzdefaultcc("", pathf("%s/zdefaultcc.go", src))
					mkzdefaultcc("", pathf("%s/zdefaultcc.go", dst))
				}
				return nil
			}

			for _, pre := range ignorePrefixes {
				if strings.HasPrefix(name, pre) {
					return nil
				}
			}
			for _, suf := range ignoreSuffixes {
				if strings.HasSuffix(name, suf) {
					return nil
				}
			}

			text := bootstrapRewriteFile(src)
			writefile(text, dst, 0)
			return nil
		})
	}

	// Set up environment for invoking Go 1.4 go command.
	// GOROOT points at Go 1.4 GOROOT,
	// GOPATH points at our bootstrap workspace,
	// GOBIN is empty, so that binaries are installed to GOPATH/bin,
	// and GOOS, GOHOSTOS, GOARCH, and GOHOSTOS are empty,
	// so that Go 1.4 builds whatever kind of binary it knows how to build.
	// Restore GOROOT, GOPATH, and GOBIN when done.
	// Don't bother with GOOS, GOHOSTOS, GOARCH, and GOHOSTARCH,
	// because setup will take care of those when bootstrapBuildTools returns.

	defer os.Setenv("GOROOT", os.Getenv("GOROOT"))
	os.Setenv("GOROOT", goroot_bootstrap)

	defer os.Setenv("GOPATH", os.Getenv("GOPATH"))
	os.Setenv("GOPATH", workspace)

	defer os.Setenv("GOBIN", os.Getenv("GOBIN"))
	os.Setenv("GOBIN", "")

	os.Setenv("GOOS", "")
	os.Setenv("GOHOSTOS", "")
	os.Setenv("GOARCH", "")
	os.Setenv("GOHOSTARCH", "")

	// Run Go 1.4 to build binaries. Use -gcflags=-l to disable inlining to
	// workaround bugs in Go 1.4's compiler. See discussion thread:
	// https://groups.google.com/d/msg/golang-dev/Ss7mCKsvk8w/Gsq7VYI0AwAJ
	// Use the math_big_pure_go build tag to disable the assembly in math/big
	// which may contain unsupported instructions.
	// Note that if we are using Go 1.10 or later as bootstrap, the -gcflags=-l
	// only applies to the final cmd/go binary, but that's OK: if this is Go 1.10
	// or later we don't need to disable inlining to work around bugs in the Go 1.4 compiler.
	cmd := []string{
		pathf("%s/bin/go", goroot_bootstrap),
		"install",
		"-gcflags=-l",
		"-tags=math_big_pure_go compiler_bootstrap",
	}
	if vflag > 0 {
		cmd = append(cmd, "-v")
	}
	if tool := os.Getenv("GOBOOTSTRAP_TOOLEXEC"); tool != "" {
		cmd = append(cmd, "-toolexec="+tool)
	}
	cmd = append(cmd, "bootstrap/cmd/...")
	run(base, ShowOutput|CheckExit, cmd...)

	// Copy binaries into tool binary directory.
	for _, name := range bootstrapDirs {
		if !strings.HasPrefix(name, "cmd/") {
			continue
		}
		name = name[len("cmd/"):]
		if !strings.Contains(name, "/") {
			copyfile(pathf("%s/%s%s", tooldir, name, exe), pathf("%s/bin/%s%s", workspace, name, exe), writeExec)
		}
	}

	if vflag > 0 {
		xprintf("\n")
	}
}

var ssaRewriteFileSubstring = filepath.FromSlash("src/cmd/compile/internal/ssa/rewrite")

// isUnneededSSARewriteFile reports whether srcFile is a
// src/cmd/compile/internal/ssa/rewriteARCHNAME.go file for an
// architecture that isn't for the given GOARCH.
//
// When unneeded is true archCaps is the rewrite base filename without
// the "rewrite" prefix or ".go" suffix: AMD64, 386, ARM, ARM64, etc.
func isUnneededSSARewriteFile(srcFile, goArch string) (archCaps string, unneeded bool) {
	if !strings.Contains(srcFile, ssaRewriteFileSubstring) {
		return "", false
	}
	fileArch := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(srcFile), "rewrite"), ".go")
	if fileArch == "" {
		return "", false
	}
	b := fileArch[0]
	if b == '_' || ('a' <= b && b <= 'z') {
		return "", false
	}
	archCaps = fileArch
	fileArch = strings.ToLower(fileArch)
	fileArch = strings.TrimSuffix(fileArch, "splitload")
	if fileArch == goArch {
		return "", false
	}
	if fileArch == strings.TrimSuffix(goArch, "le") {
		return "", false
	}
	return archCaps, true
}

func bootstrapRewriteFile(srcFile string) string {
	// During bootstrap, generate dummy rewrite files for
	// irrelevant architectures. We only need to build a bootstrap
	// binary that works for the current gohostarch.
	// This saves 6+ seconds of bootstrap.
	if archCaps, ok := isUnneededSSARewriteFile(srcFile, gohostarch); ok {
		return fmt.Sprintf(`// Code generated by go tool dist; DO NOT EDIT.

package ssa

func rewriteValue%s(v *Value) bool { panic("unused during bootstrap") }
func rewriteBlock%s(b *Block) bool { panic("unused during bootstrap") }
`, archCaps, archCaps)
	}

	return bootstrapFixImports(srcFile)
}

func bootstrapFixImports(srcFile string) string {
	text := readfile(srcFile)
	if !strings.Contains(srcFile, "/cmd/") && !strings.Contains(srcFile, `\cmd\`) {
		text = regexp.MustCompile(`\bany\b`).ReplaceAllString(text, "interface{}")
	}
	lines := strings.SplitAfter(text, "\n")
	inBlock := false
	for i, line := range lines {
		if strings.HasPrefix(line, "import (") {
			inBlock = true
			continue
		}
		if inBlock && strings.HasPrefix(line, ")") {
			inBlock = false
			continue
		}
		if strings.HasPrefix(line, `import "`) || strings.HasPrefix(line, `import . "`) ||
			inBlock && (strings.HasPrefix(line, "\t\"") || strings.HasPrefix(line, "\t. \"") || strings.HasPrefix(line, "\texec \"")) {
			line = strings.Replace(line, `"cmd/`, `"bootstrap/cmd/`, -1)
			for _, dir := range bootstrapDirs {
				if strings.HasPrefix(dir, "cmd/") {
					continue
				}
				line = strings.Replace(line, `"`+dir+`"`, `"bootstrap/`+dir+`"`, -1)
			}
			lines[i] = line
		}
	}

	lines[0] = "// Code generated by go tool dist; DO NOT EDIT.\n// This is a bootstrap copy of " + srcFile + "\n\n//line " + srcFile + ":1\n" + lines[0]

	return strings.Join(lines, "")
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Run "make install" to build package.

package main

import (
	"bytes"
	"os"
	"template"
)

// domake builds the package in dir.
// If local is false, the package was copied from an external system.
// For non-local packages or packages without Makefiles,
// domake generates a standard Makefile and passes it
// to make on standard input.
func domake(dir, pkg string, local bool) (err os.Error) {
	needMakefile := true
	if local {
		_, err := os.Stat(dir + "/Makefile")
		if err == nil {
			needMakefile = false
		}
	}
	cmd := []string{"gomake"}
	var makefile []byte
	if needMakefile {
		if makefile, err = makeMakefile(dir, pkg); err != nil {
			return err
		}
		cmd = append(cmd, "-f-")
	}
	if *clean {
		cmd = append(cmd, "clean")
	}
	cmd = append(cmd, "install")
	return run(dir, makefile, cmd...)
}

// makeMakefile computes the standard Makefile for the directory dir
// installing as package pkg.  It includes all *.go files in the directory
// except those in package main and those ending in _test.go.
func makeMakefile(dir, pkg string) ([]byte, os.Error) {
	dirInfo, err := scanDir(dir, false)
	if err != nil {
		return nil, err
	}

	if len(dirInfo.cgoFiles) == 0 && len(dirInfo.cFiles) > 0 {
		// When using cgo, .c files are compiled with gcc.  Without cgo,
		// they may be intended for 6c.  Just error out for now.
		return nil, os.ErrorString("C files found in non-cgo package")
	}

	cgoFiles := dirInfo.cgoFiles
	isCgo := make(map[string]bool, len(cgoFiles))
	for _, file := range cgoFiles {
		isCgo[file] = true
	}

	oFiles := make([]string, 0, len(dirInfo.cFiles))
	for _, file := range dirInfo.cFiles {
		oFiles = append(oFiles, file[:len(file)-2]+".o")
	}

	goFiles := make([]string, 0, len(dirInfo.goFiles))
	for _, file := range dirInfo.goFiles {
		if !isCgo[file] {
			goFiles = append(goFiles, file)
		}
	}

	var buf bytes.Buffer
	md := makedata{pkg, goFiles, cgoFiles, oFiles}
	if err := makefileTemplate.Execute(&md, &buf); err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

// makedata is the data type for the makefileTemplate.
type makedata struct {
	pkg      string   // package import path
	goFiles  []string // list of non-cgo .go files
	cgoFiles []string // list of cgo .go files
	oFiles   []string // list of ofiles for cgo
}

var makefileTemplate = template.MustParse(`
include $(GOROOT)/src/Make.inc

TARG={pkg}

{.section goFiles}
GOFILES=\
{.repeated section goFiles}
	{@}\
{.end}

{.end}
{.section cgoFiles}
CGOFILES=\
{.repeated section cgoFiles}
	{@}\
{.end}

{.end}
{.section oFiles}
CGO_OFILES=\
{.repeated section oFiles}
	{@}\
{.end}

{.end}
include $(GOROOT)/src/Make.pkg
`,
	nil)
<commit_msg>goinstall: Fix template to use exported fields<commit_after>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Run "make install" to build package.

package main

import (
	"bytes"
	"os"
	"template"
)

// domake builds the package in dir.
// If local is false, the package was copied from an external system.
// For non-local packages or packages without Makefiles,
// domake generates a standard Makefile and passes it
// to make on standard input.
func domake(dir, pkg string, local bool) (err os.Error) {
	needMakefile := true
	if local {
		_, err := os.Stat(dir + "/Makefile")
		if err == nil {
			needMakefile = false
		}
	}
	cmd := []string{"gomake"}
	var makefile []byte
	if needMakefile {
		if makefile, err = makeMakefile(dir, pkg); err != nil {
			return err
		}
		cmd = append(cmd, "-f-")
	}
	if *clean {
		cmd = append(cmd, "clean")
	}
	cmd = append(cmd, "install")
	return run(dir, makefile, cmd...)
}

// makeMakefile computes the standard Makefile for the directory dir
// installing as package pkg.  It includes all *.go files in the directory
// except those in package main and those ending in _test.go.
func makeMakefile(dir, pkg string) ([]byte, os.Error) {
	dirInfo, err := scanDir(dir, false)
	if err != nil {
		return nil, err
	}

	if len(dirInfo.cgoFiles) == 0 && len(dirInfo.cFiles) > 0 {
		// When using cgo, .c files are compiled with gcc.  Without cgo,
		// they may be intended for 6c.  Just error out for now.
		return nil, os.ErrorString("C files found in non-cgo package")
	}

	cgoFiles := dirInfo.cgoFiles
	isCgo := make(map[string]bool, len(cgoFiles))
	for _, file := range cgoFiles {
		isCgo[file] = true
	}

	oFiles := make([]string, 0, len(dirInfo.cFiles))
	for _, file := range dirInfo.cFiles {
		oFiles = append(oFiles, file[:len(file)-2]+".o")
	}

	goFiles := make([]string, 0, len(dirInfo.goFiles))
	for _, file := range dirInfo.goFiles {
		if !isCgo[file] {
			goFiles = append(goFiles, file)
		}
	}

	var buf bytes.Buffer
	md := makedata{pkg, goFiles, cgoFiles, oFiles}
	if err := makefileTemplate.Execute(&md, &buf); err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

// makedata is the data type for the makefileTemplate.
type makedata struct {
	Pkg      string   // package import path
	GoFiles  []string // list of non-cgo .go files
	CgoFiles []string // list of cgo .go files
	OFiles   []string // list of ofiles for cgo
}

var makefileTemplate = template.MustParse(`
include $(GOROOT)/src/Make.inc

TARG={Pkg}

{.section GoFiles}
GOFILES=\
{.repeated section GoFiles}
	{@}\
{.end}

{.end}
{.section CgoFiles}
CGOFILES=\
{.repeated section CgoFiles}
	{@}\
{.end}

{.end}
{.section OFiles}
CGO_OFILES=\
{.repeated section OFiles}
	{@}\
{.end}

{.end}
include $(GOROOT)/src/Make.pkg
`,
	nil)
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// A Go scanner. Takes a []byte as source which can then be
// tokenized through repeated calls to the Scan() function.
//
// Sample use:
//
//	import "token"
//	import "scanner"
//
//	func tokenize(src []byte) {
//		var s scanner.Scanner;
//		s.Init(src, nil /* no error handler */, false /* ignore comments */);
//		for {
//			pos, tok, lit := s.Scan();
//			if tok == Scanner.EOF {
//				return;
//			}
//			println(pos, token.TokenString(tok), string(lit));
//		}
//	}
//
package scanner

import (
	"utf8";
	"unicode";
	"strconv";
	"token";
)

// An implementation of an ErrorHandler must be provided to the Scanner.
// If a syntax error is encountered, Error() is called with the exact
// token position (the byte position of the token in the source) and the
// error message.
//
type ErrorHandler interface {
	Error(pos int, msg string);
}

// A Scanner holds the scanner's internal state while processing
// a given text.  It can be allocated as part of another data
// structure but must be initialized via Init() before use.
// See also the package comment for a sample use.
//
type Scanner struct {
	// immutable state
	src []byte;  // source
	err ErrorHandler;  // error reporting
	scan_comments bool;  // if set, comments are reported as tokens

	// scanning state
	pos int;  // current reading position
	ch int;  // one char look-ahead
	chpos int;  // position of ch
}

func isLetter(ch int) bool {
	return
		'a' <= ch && ch <= 'z' ||
		'A' <= ch && ch <= 'Z' ||
		ch == '_' ||
		ch >= 0x80 && unicode.IsLetter(ch);
}

func digitVal(ch int) int {
	switch {
	case '0' <= ch && ch <= '9': return ch - '0';
	case 'a' <= ch && ch <= 'f': return ch - 'a' + 10;
	case 'A' <= ch && ch <= 'F': return ch - 'A' + 10;
	}
	return 16;  // larger than any legal digit val
}

// Read the next Unicode char into S.ch.
// S.ch < 0 means end-of-file.
func (S *Scanner) next() {
	if S.pos < len(S.src) {
		// assume ASCII
		r, w := int(S.src[S.pos]), 1;
		if r >= 0x80 {
			// not ASCII
			r, w = utf8.DecodeRune(S.src[S.pos : len(S.src)]);
		}
		S.ch = r;
		S.chpos = S.pos;
		S.pos += w;
	} else {
		S.ch = -1;  // eof
		S.chpos = len(S.src);
	}
}

// Init() prepares the scanner S to tokenize the text src. Calls to Scan()
// will use the error handler err if they encounter a syntax error. The boolean
// scan_comments specifies whether newline characters and comments should be
// recognized and returned by Scan as token.COMMENT. If scan_comments is false,
// they are treated as white space and ignored.
//
func (S *Scanner) Init(src []byte, err ErrorHandler, scan_comments bool) {
	S.src = src;
	S.err = err;
	S.scan_comments = scan_comments;
	S.next();
}

func charString(ch int) string {
	s := string(ch);
	switch ch {
	case '\a': s = `\a`;
	case '\b': s = `\b`;
	case '\f': s = `\f`;
	case '\n': s = `\n`;
	case '\r': s = `\r`;
	case '\t': s = `\t`;
	case '\v': s = `\v`;
	case '\\': s = `\\`;
	case '\'': s = `\'`;
	}
	return "'" + s + "' (U+" + strconv.Itob(ch, 16) + ")";
}

func (S *Scanner) error(pos int, msg string) {
	S.err.Error(pos, msg);
}

func (S *Scanner) expect(ch int) {
	if S.ch != ch {
		S.error(S.chpos, "expected " + charString(ch) + ", found " + charString(S.ch));
	}
	S.next();  // always make progress
}

func (S *Scanner) skipWhitespace() {
	for {
		switch S.ch {
		case '\t', '\r', ' ':
			// nothing to do
		case '\n':
			if S.scan_comments {
				return;
			}
		default:
			return;
		}
		S.next();
	}
	panic("UNREACHABLE");
}

func (S *Scanner) scanComment() []byte {
	// first '/' already consumed
	pos := S.chpos - 1;

	if S.ch == '/' {
		//-style comment
		for S.ch >= 0 {
			S.next();
			if S.ch == '\n' {
				// '\n' terminates comment but we do not include
				// it in the comment (otherwise we don't see the
				// start of a newline in skipWhitespace()).
				return S.src[pos : S.chpos];
			}
		}

	} else {
		/*-style comment */
		S.expect('*');
		for S.ch >= 0 {
			ch := S.ch;
			S.next();
			if ch == '*' && S.ch == '/' {
				S.next();
				return S.src[pos : S.chpos];
			}
		}
	}

	S.error(pos, "comment not terminated");
	return S.src[pos : S.chpos];
}

func (S *Scanner) scanIdentifier() (tok int, lit []byte) {
	pos := S.chpos;
	for isLetter(S.ch) || digitVal(S.ch) < 10 {
		S.next();
	}
	lit = S.src[pos : S.chpos];
	return token.Lookup(lit), lit;
}

func (S *Scanner) scanMantissa(base int) {
	for digitVal(S.ch) < base {
		S.next();
	}
}

func (S *Scanner) scanNumber(seen_decimal_point bool) (tok int, lit []byte) {
	pos := S.chpos;
	tok = token.INT;

	if seen_decimal_point {
		tok = token.FLOAT;
		pos--;  // '.' is one byte
		S.scanMantissa(10);
		goto exponent;
	}

	if S.ch == '0' {
		// int or float
		S.next();
		if S.ch == 'x' || S.ch == 'X' {
			// hexadecimal int
			S.next();
			S.scanMantissa(16);
		} else {
			// octal int or float
			S.scanMantissa(8);
			if digitVal(S.ch) < 10 || S.ch == '.' || S.ch == 'e' || S.ch == 'E' {
				// float
				tok = token.FLOAT;
				goto mantissa;
			}
			// octal int
		}
		goto exit;
	}

mantissa:
	// decimal int or float
	S.scanMantissa(10);

	if S.ch == '.' {
		// float
		tok = token.FLOAT;
		S.next();
		S.scanMantissa(10)
	}

exponent:
	if S.ch == 'e' || S.ch == 'E' {
		// float
		tok = token.FLOAT;
		S.next();
		if S.ch == '-' || S.ch == '+' {
			S.next();
		}
		S.scanMantissa(10);
	}

exit:
	return tok, S.src[pos : S.chpos];
}

func (S *Scanner) scanDigits(n int, base int) {
	for digitVal(S.ch) < base {
		S.next();
		n--;
	}
	if n > 0 {
		S.error(S.chpos, "illegal char escape");
	}
}

func (S *Scanner) scanEscape(quote int) {
	ch := S.ch;
	pos := S.chpos;
	S.next();
	switch ch {
	case 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\', quote:
		// nothing to do
	case '0', '1', '2', '3', '4', '5', '6', '7':
		S.scanDigits(3 - 1, 8);  // 1 char read already
	case 'x':
		S.scanDigits(2, 16);
	case 'u':
		S.scanDigits(4, 16);
	case 'U':
		S.scanDigits(8, 16);
	default:
		S.error(pos, "illegal char escape");
	}
}

func (S *Scanner) scanChar() []byte {
	// '\'' already consumed

	pos := S.chpos - 1;
	ch := S.ch;
	S.next();
	if ch == '\\' {
		S.scanEscape('\'');
	}

	S.expect('\'');
	return S.src[pos : S.chpos];
}

func (S *Scanner) scanString() []byte {
	// '"' already consumed

	pos := S.chpos - 1;
	for S.ch != '"' {
		ch := S.ch;
		S.next();
		if ch == '\n' || ch < 0 {
			S.error(pos, "string not terminated");
			break;
		}
		if ch == '\\' {
			S.scanEscape('"');
		}
	}

	S.next();
	return S.src[pos : S.chpos];
}

func (S *Scanner) scanRawString() []byte {
	// '`' already consumed

	pos := S.chpos - 1;
	for S.ch != '`' {
		ch := S.ch;
		S.next();
		if ch == '\n' || ch < 0 {
			S.error(pos, "string not terminated");
			break;
		}
	}

	S.next();
	return S.src[pos : S.chpos];
}

// Helper functions for scanning multi-byte tokens such as >> += >>= .
// Different routines recognize different length tok_i based on matches
// of ch_i. If a token ends in '=', the result is tok1 or tok3
// respectively. Otherwise, the result is tok0 if there was no other
// matching character, or tok2 if the matching character was ch2.

func (S *Scanner) switch2(tok0, tok1 int) int {
	if S.ch == '=' {
		S.next();
		return tok1;
	}
	return tok0;
}

func (S *Scanner) switch3(tok0, tok1, ch2, tok2 int) int {
	if S.ch == '=' {
		S.next();
		return tok1;
	}
	if S.ch == ch2 {
		S.next();
		return tok2;
	}
	return tok0;
}

func (S *Scanner) switch4(tok0, tok1, ch2, tok2, tok3 int) int {
	if S.ch == '=' {
		S.next();
		return tok1;
	}
	if S.ch == ch2 {
		S.next();
		if S.ch == '=' {
			S.next();
			return tok3;
		}
		return tok2;
	}
	return tok0;
}

// Scan() scans the next token and returns the token byte position in the
// source, its token value, and the corresponding literal text if the token
// is an identifier, basic type literal (token.IsLiteral(tok) == true), or
// comment.
//
func (S *Scanner) Scan() (pos, tok int, lit []byte) {
scan_again:
	S.skipWhitespace();

	pos, tok = S.chpos, token.ILLEGAL;

	switch ch := S.ch; {
	case isLetter(ch):
		tok, lit = S.scanIdentifier();
	case digitVal(ch) < 10:
		tok, lit = S.scanNumber(false);
	default:
		S.next();  // always make progress
		switch ch {
		case -1  : tok = token.EOF;
		case '\n': tok, lit = token.COMMENT, []byte{'\n'};
		case '"' : tok, lit = token.STRING, S.scanString();
		case '\'': tok, lit = token.CHAR, S.scanChar();
		case '`' : tok, lit = token.STRING, S.scanRawString();
		case ':' : tok = S.switch2(token.COLON, token.DEFINE);
		case '.' :
			if digitVal(S.ch) < 10 {
				tok, lit = S.scanNumber(true);
			} else if S.ch == '.' {
				S.next();
				if S.ch == '.' {
					S.next();
					tok = token.ELLIPSIS;
				}
			} else {
				tok = token.PERIOD;
			}
		case ',': tok = token.COMMA;
		case ';': tok = token.SEMICOLON;
		case '(': tok = token.LPAREN;
		case ')': tok = token.RPAREN;
		case '[': tok = token.LBRACK;
		case ']': tok = token.RBRACK;
		case '{': tok = token.LBRACE;
		case '}': tok = token.RBRACE;
		case '+': tok = S.switch3(token.ADD, token.ADD_ASSIGN, '+', token.INC);
		case '-': tok = S.switch3(token.SUB, token.SUB_ASSIGN, '-', token.DEC);
		case '*': tok = S.switch2(token.MUL, token.MUL_ASSIGN);
		case '/':
			if S.ch == '/' || S.ch == '*' {
				tok, lit = token.COMMENT, S.scanComment();
				if !S.scan_comments {
					goto scan_again;
				}
			} else {
				tok = S.switch2(token.QUO, token.QUO_ASSIGN);
			}
		case '%': tok = S.switch2(token.REM, token.REM_ASSIGN);
		case '^': tok = S.switch2(token.XOR, token.XOR_ASSIGN);
		case '<':
			if S.ch == '-' {
				S.next();
				tok = token.ARROW;
			} else {
				tok = S.switch4(token.LSS, token.LEQ, '<', token.SHL, token.SHL_ASSIGN);
			}
		case '>': tok = S.switch4(token.GTR, token.GEQ, '>', token.SHR, token.SHR_ASSIGN);
		case '=': tok = S.switch2(token.ASSIGN, token.EQL);
		case '!': tok = S.switch2(token.NOT, token.NEQ);
		case '&': tok = S.switch3(token.AND, token.AND_ASSIGN, '&', token.LAND);
		case '|': tok = S.switch3(token.OR, token.OR_ASSIGN, '|', token.LOR);
		default: S.error(pos, "illegal character " + charString(ch));
		}
	}

	return pos, tok, lit;
}
<commit_msg>Fixing comment.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// A scanner for Go source text. Takes a []byte as source which can
// then be tokenized through repeated calls to the Scan() function.
//
// Sample use:
//
//	import "token"
//	import "scanner"
//
//	func tokenize(src []byte) {
//		var s scanner.Scanner;
//		s.Init(src, nil /* no error handler */, false /* ignore comments */);
//		for {
//			pos, tok, lit := s.Scan();
//			if tok == Scanner.EOF {
//				return;
//			}
//			println(pos, token.TokenString(tok), string(lit));
//		}
//	}
//
package scanner

import (
	"utf8";
	"unicode";
	"strconv";
	"token";
)

// An implementation of an ErrorHandler must be provided to the Scanner.
// If a syntax error is encountered, Error() is called with the exact
// token position (the byte position of the token in the source) and the
// error message.
//
type ErrorHandler interface {
	Error(pos int, msg string);
}

// A Scanner holds the scanner's internal state while processing
// a given text.  It can be allocated as part of another data
// structure but must be initialized via Init() before use.
// See also the package comment for a sample use.
//
type Scanner struct {
	// immutable state
	src []byte;  // source
	err ErrorHandler;  // error reporting
	scan_comments bool;  // if set, comments are reported as tokens

	// scanning state
	pos int;  // current reading position
	ch int;  // one char look-ahead
	chpos int;  // position of ch
}

func isLetter(ch int) bool {
	return
		'a' <= ch && ch <= 'z' ||
		'A' <= ch && ch <= 'Z' ||
		ch == '_' ||
		ch >= 0x80 && unicode.IsLetter(ch);
}

func digitVal(ch int) int {
	switch {
	case '0' <= ch && ch <= '9': return ch - '0';
	case 'a' <= ch && ch <= 'f': return ch - 'a' + 10;
	case 'A' <= ch && ch <= 'F': return ch - 'A' + 10;
	}
	return 16;  // larger than any legal digit val
}

// Read the next Unicode char into S.ch.
// S.ch < 0 means end-of-file.
func (S *Scanner) next() {
	if S.pos < len(S.src) {
		// assume ASCII
		r, w := int(S.src[S.pos]), 1;
		if r >= 0x80 {
			// not ASCII
			r, w = utf8.DecodeRune(S.src[S.pos : len(S.src)]);
		}
		S.ch = r;
		S.chpos = S.pos;
		S.pos += w;
	} else {
		S.ch = -1;  // eof
		S.chpos = len(S.src);
	}
}

// Init() prepares the scanner S to tokenize the text src. Calls to Scan()
// will use the error handler err if they encounter a syntax error. The boolean
// scan_comments specifies whether newline characters and comments should be
// recognized and returned by Scan as token.COMMENT. If scan_comments is false,
// they are treated as white space and ignored.
//
func (S *Scanner) Init(src []byte, err ErrorHandler, scan_comments bool) {
	S.src = src;
	S.err = err;
	S.scan_comments = scan_comments;
	S.next();
}

func charString(ch int) string {
	s := string(ch);
	switch ch {
	case '\a': s = `\a`;
	case '\b': s = `\b`;
	case '\f': s = `\f`;
	case '\n': s = `\n`;
	case '\r': s = `\r`;
	case '\t': s = `\t`;
	case '\v': s = `\v`;
	case '\\': s = `\\`;
	case '\'': s = `\'`;
	}
	return "'" + s + "' (U+" + strconv.Itob(ch, 16) + ")";
}

func (S *Scanner) error(pos int, msg string) {
	S.err.Error(pos, msg);
}

func (S *Scanner) expect(ch int) {
	if S.ch != ch {
		S.error(S.chpos, "expected " + charString(ch) + ", found " + charString(S.ch));
	}
	S.next();  // always make progress
}

func (S *Scanner) skipWhitespace() {
	for {
		switch S.ch {
		case '\t', '\r', ' ':
			// nothing to do
		case '\n':
			if S.scan_comments {
				return;
			}
		default:
			return;
		}
		S.next();
	}
	panic("UNREACHABLE");
}

func (S *Scanner) scanComment() []byte {
	// first '/' already consumed
	pos := S.chpos - 1;

	if S.ch == '/' {
		//-style comment
		for S.ch >= 0 {
			S.next();
			if S.ch == '\n' {
				// '\n' terminates comment but we do not include
				// it in the comment (otherwise we don't see the
				// start of a newline in skipWhitespace()).
				return S.src[pos : S.chpos];
			}
		}

	} else {
		/*-style comment */
		S.expect('*');
		for S.ch >= 0 {
			ch := S.ch;
			S.next();
			if ch == '*' && S.ch == '/' {
				S.next();
				return S.src[pos : S.chpos];
			}
		}
	}

	S.error(pos, "comment not terminated");
	return S.src[pos : S.chpos];
}

func (S *Scanner) scanIdentifier() (tok int, lit []byte) {
	pos := S.chpos;
	for isLetter(S.ch) || digitVal(S.ch) < 10 {
		S.next();
	}
	lit = S.src[pos : S.chpos];
	return token.Lookup(lit), lit;
}

func (S *Scanner) scanMantissa(base int) {
	for digitVal(S.ch) < base {
		S.next();
	}
}

func (S *Scanner) scanNumber(seen_decimal_point bool) (tok int, lit []byte) {
	pos := S.chpos;
	tok = token.INT;

	if seen_decimal_point {
		tok = token.FLOAT;
		pos--;  // '.' is one byte
		S.scanMantissa(10);
		goto exponent;
	}

	if S.ch == '0' {
		// int or float
		S.next();
		if S.ch == 'x' || S.ch == 'X' {
			// hexadecimal int
			S.next();
			S.scanMantissa(16);
		} else {
			// octal int or float
			S.scanMantissa(8);
			if digitVal(S.ch) < 10 || S.ch == '.' || S.ch == 'e' || S.ch == 'E' {
				// float
				tok = token.FLOAT;
				goto mantissa;
			}
			// octal int
		}
		goto exit;
	}

mantissa:
	// decimal int or float
	S.scanMantissa(10);

	if S.ch == '.' {
		// float
		tok = token.FLOAT;
		S.next();
		S.scanMantissa(10)
	}

exponent:
	if S.ch == 'e' || S.ch == 'E' {
		// float
		tok = token.FLOAT;
		S.next();
		if S.ch == '-' || S.ch == '+' {
			S.next();
		}
		S.scanMantissa(10);
	}

exit:
	return tok, S.src[pos : S.chpos];
}

func (S *Scanner) scanDigits(n int, base int) {
	for digitVal(S.ch) < base {
		S.next();
		n--;
	}
	if n > 0 {
		S.error(S.chpos, "illegal char escape");
	}
}

func (S *Scanner) scanEscape(quote int) {
	ch := S.ch;
	pos := S.chpos;
	S.next();
	switch ch {
	case 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\', quote:
		// nothing to do
	case '0', '1', '2', '3', '4', '5', '6', '7':
		S.scanDigits(3 - 1, 8);  // 1 char read already
	case 'x':
		S.scanDigits(2, 16);
	case 'u':
		S.scanDigits(4, 16);
	case 'U':
		S.scanDigits(8, 16);
	default:
		S.error(pos, "illegal char escape");
	}
}

func (S *Scanner) scanChar() []byte {
	// '\'' already consumed

	pos := S.chpos - 1;
	ch := S.ch;
	S.next();
	if ch == '\\' {
		S.scanEscape('\'');
	}

	S.expect('\'');
	return S.src[pos : S.chpos];
}

func (S *Scanner) scanString() []byte {
	// '"' already consumed

	pos := S.chpos - 1;
	for S.ch != '"' {
		ch := S.ch;
		S.next();
		if ch == '\n' || ch < 0 {
			S.error(pos, "string not terminated");
			break;
		}
		if ch == '\\' {
			S.scanEscape('"');
		}
	}

	S.next();
	return S.src[pos : S.chpos];
}

func (S *Scanner) scanRawString() []byte {
	// '`' already consumed

	pos := S.chpos - 1;
	for S.ch != '`' {
		ch := S.ch;
		S.next();
		if ch == '\n' || ch < 0 {
			S.error(pos, "string not terminated");
			break;
		}
	}

	S.next();
	return S.src[pos : S.chpos];
}

// Helper functions for scanning multi-byte tokens such as >> += >>= .
// Different routines recognize different length tok_i based on matches
// of ch_i. If a token ends in '=', the result is tok1 or tok3
// respectively. Otherwise, the result is tok0 if there was no other
// matching character, or tok2 if the matching character was ch2.

func (S *Scanner) switch2(tok0, tok1 int) int {
	if S.ch == '=' {
		S.next();
		return tok1;
	}
	return tok0;
}

func (S *Scanner) switch3(tok0, tok1, ch2, tok2 int) int {
	if S.ch == '=' {
		S.next();
		return tok1;
	}
	if S.ch == ch2 {
		S.next();
		return tok2;
	}
	return tok0;
}

func (S *Scanner) switch4(tok0, tok1, ch2, tok2, tok3 int) int {
	if S.ch == '=' {
		S.next();
		return tok1;
	}
	if S.ch == ch2 {
		S.next();
		if S.ch == '=' {
			S.next();
			return tok3;
		}
		return tok2;
	}
	return tok0;
}

// Scan() scans the next token and returns the token byte position in the
// source, its token value, and the corresponding literal text if the token
// is an identifier, basic type literal (token.IsLiteral(tok) == true), or
// comment.
//
func (S *Scanner) Scan() (pos, tok int, lit []byte) {
scan_again:
	S.skipWhitespace();

	pos, tok = S.chpos, token.ILLEGAL;

	switch ch := S.ch; {
	case isLetter(ch):
		tok, lit = S.scanIdentifier();
	case digitVal(ch) < 10:
		tok, lit = S.scanNumber(false);
	default:
		S.next();  // always make progress
		switch ch {
		case -1  : tok = token.EOF;
		case '\n': tok, lit = token.COMMENT, []byte{'\n'};
		case '"' : tok, lit = token.STRING, S.scanString();
		case '\'': tok, lit = token.CHAR, S.scanChar();
		case '`' : tok, lit = token.STRING, S.scanRawString();
		case ':' : tok = S.switch2(token.COLON, token.DEFINE);
		case '.' :
			if digitVal(S.ch) < 10 {
				tok, lit = S.scanNumber(true);
			} else if S.ch == '.' {
				S.next();
				if S.ch == '.' {
					S.next();
					tok = token.ELLIPSIS;
				}
			} else {
				tok = token.PERIOD;
			}
		case ',': tok = token.COMMA;
		case ';': tok = token.SEMICOLON;
		case '(': tok = token.LPAREN;
		case ')': tok = token.RPAREN;
		case '[': tok = token.LBRACK;
		case ']': tok = token.RBRACK;
		case '{': tok = token.LBRACE;
		case '}': tok = token.RBRACE;
		case '+': tok = S.switch3(token.ADD, token.ADD_ASSIGN, '+', token.INC);
		case '-': tok = S.switch3(token.SUB, token.SUB_ASSIGN, '-', token.DEC);
		case '*': tok = S.switch2(token.MUL, token.MUL_ASSIGN);
		case '/':
			if S.ch == '/' || S.ch == '*' {
				tok, lit = token.COMMENT, S.scanComment();
				if !S.scan_comments {
					goto scan_again;
				}
			} else {
				tok = S.switch2(token.QUO, token.QUO_ASSIGN);
			}
		case '%': tok = S.switch2(token.REM, token.REM_ASSIGN);
		case '^': tok = S.switch2(token.XOR, token.XOR_ASSIGN);
		case '<':
			if S.ch == '-' {
				S.next();
				tok = token.ARROW;
			} else {
				tok = S.switch4(token.LSS, token.LEQ, '<', token.SHL, token.SHL_ASSIGN);
			}
		case '>': tok = S.switch4(token.GTR, token.GEQ, '>', token.SHR, token.SHR_ASSIGN);
		case '=': tok = S.switch2(token.ASSIGN, token.EQL);
		case '!': tok = S.switch2(token.NOT, token.NEQ);
		case '&': tok = S.switch3(token.AND, token.AND_ASSIGN, '&', token.LAND);
		case '|': tok = S.switch3(token.OR, token.OR_ASSIGN, '|', token.LOR);
		default: S.error(pos, "illegal character " + charString(ch));
		}
	}

	return pos, tok, lit;
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package scanner

// A Go scanner. Takes a []byte as source which can then be
// tokenized through repeated calls to the Scan() function.
//
// Sample use:
//
//	import "token"
//	import "scanner"
//
//	func tokenize(src []byte) {
//		var s scanner.Scanner;
//		s.Init(src, nil /* no error handler */, false /* ignore comments */);
//		for {
//			pos, tok, lit := s.Scan();
//			if tok == Scanner.EOF {
//				return;
//			}
//			println(pos, token.TokenString(tok), string(lit));
//		}
//	}

import (
	"utf8";
	"unicode";
	"strconv";
	"token";
)

// An implementation of an ErrorHandler must be provided to the Scanner.
// If a syntax error is encountered, Error() is called with the exact
// token position (the byte position of the token in the source) and the
// error message.

type ErrorHandler interface {
	Error(pos int, msg string);
}

type Scanner struct {
	// immutable state
	src []byte;  // source
	err ErrorHandler;  // error reporting
	scan_comments bool;  // if set, comments are reported as tokens

	// scanning state
	pos int;  // current reading position
	ch int;  // one char look-ahead
	chpos int;  // position of ch
}

func isLetter(ch int) bool {
	return
		'a' <= ch && ch <= 'z' ||
		'A' <= ch && ch <= 'Z' ||
		ch == '_' ||
		ch >= 0x80 && unicode.IsLetter(ch);
}

func digitVal(ch int) int {
	switch {
	case '0' <= ch && ch <= '9': return ch - '0';
	case 'a' <= ch && ch <= 'f': return ch - 'a' + 10;
	case 'A' <= ch && ch <= 'F': return ch - 'A' + 10;
	}
	return 16;  // larger than any legal digit val
}

// Read the next Unicode char into S.ch.
// S.ch < 0 means end-of-file.
func (S *Scanner) next() {
	if S.pos < len(S.src) {
		// assume ASCII
		r, w := int(S.src[S.pos]), 1;
		if r >= 0x80 {
			// not ASCII
			r, w = utf8.DecodeRune(S.src[S.pos : len(S.src)]);
		}
		S.ch = r;
		S.chpos = S.pos;
		S.pos += w;
	} else {
		S.ch = -1;  // eof
		S.chpos = len(S.src);
	}
}

// Initialize the scanner.
//
// The error handler (err) is called when an illegal token is encountered.
// If scan_comments is set to true, newline characters ('\n') and comments
// are recognized as token.COMMENT, otherwise they are treated as white
// space and ignored.

func (S *Scanner) Init(src []byte, err ErrorHandler, scan_comments bool) {
	S.src = src;
	S.err = err;
	S.scan_comments = scan_comments;
	S.next();
}

func charString(ch int) string {
	s := string(ch);
	switch ch {
	case '\a': s = `\a`;
	case '\b': s = `\b`;
	case '\f': s = `\f`;
	case '\n': s = `\n`;
	case '\r': s = `\r`;
	case '\t': s = `\t`;
	case '\v': s = `\v`;
	case '\\': s = `\\`;
	case '\'': s = `\'`;
	}
	return "'" + s + "' (U+" + strconv.Itob(ch, 16) + ")";
}

func (S *Scanner) error(pos int, msg string) {
	S.err.Error(pos, msg);
}

func (S *Scanner) expect(ch int) {
	if S.ch != ch {
		S.error(S.chpos, "expected " + charString(ch) + ", found " + charString(S.ch));
	}
	S.next();  // always make progress
}

func (S *Scanner) skipWhitespace() {
	for {
		switch S.ch {
		case '\t', '\r', ' ':
			// nothing to do
		case '\n':
			if S.scan_comments {
				return;
			}
		default:
			return;
		}
		S.next();
	}
	panic("UNREACHABLE");
}

func (S *Scanner) scanComment() []byte {
	// first '/' already consumed
	pos := S.chpos - 1;

	if S.ch == '/' {
		//-style comment
		for S.ch >= 0 {
			S.next();
			if S.ch == '\n' {
				// '\n' terminates comment but we do not include
				// it in the comment (otherwise we don't see the
				// start of a newline in skipWhitespace()).
				return S.src[pos : S.chpos];
			}
		}

	} else {
		/*-style comment */
		S.expect('*');
		for S.ch >= 0 {
			ch := S.ch;
			S.next();
			if ch == '*' && S.ch == '/' {
				S.next();
				return S.src[pos : S.chpos];
			}
		}
	}

	S.error(pos, "comment not terminated");
	return S.src[pos : S.chpos];
}

func (S *Scanner) scanIdentifier() (tok int, lit []byte) {
	pos := S.chpos;
	for isLetter(S.ch) || digitVal(S.ch) < 10 {
		S.next();
	}
	lit = S.src[pos : S.chpos];
	return token.Lookup(lit), lit;
}

func (S *Scanner) scanMantissa(base int) {
	for digitVal(S.ch) < base {
		S.next();
	}
}

func (S *Scanner) scanNumber(seen_decimal_point bool) (tok int, lit []byte) {
	pos := S.chpos;
	tok = token.INT;

	if seen_decimal_point {
		tok = token.FLOAT;
		pos--;  // '.' is one byte
		S.scanMantissa(10);
		goto exponent;
	}

	if S.ch == '0' {
		// int or float
		S.next();
		if S.ch == 'x' || S.ch == 'X' {
			// hexadecimal int
			S.next();
			S.scanMantissa(16);
		} else {
			// octal int or float
			S.scanMantissa(8);
			if digitVal(S.ch) < 10 || S.ch == '.' || S.ch == 'e' || S.ch == 'E' {
				// float
				tok = token.FLOAT;
				goto mantissa;
			}
			// octal int
		}
		goto exit;
	}

mantissa:
	// decimal int or float
	S.scanMantissa(10);

	if S.ch == '.' {
		// float
		tok = token.FLOAT;
		S.next();
		S.scanMantissa(10)
	}

exponent:
	if S.ch == 'e' || S.ch == 'E' {
		// float
		tok = token.FLOAT;
		S.next();
		if S.ch == '-' || S.ch == '+' {
			S.next();
		}
		S.scanMantissa(10);
	}

exit:
	return tok, S.src[pos : S.chpos];
}

func (S *Scanner) scanDigits(n int, base int) {
	for digitVal(S.ch) < base {
		S.next();
		n--;
	}
	if n > 0 {
		S.error(S.chpos, "illegal char escape");
	}
}

func (S *Scanner) scanEscape(quote int) {
	ch := S.ch;
	pos := S.chpos;
	S.next();
	switch ch {
	case 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\', quote:
		// nothing to do
	case '0', '1', '2', '3', '4', '5', '6', '7':
		S.scanDigits(3 - 1, 8);  // 1 char read already
	case 'x':
		S.scanDigits(2, 16);
	case 'u':
		S.scanDigits(4, 16);
	case 'U':
		S.scanDigits(8, 16);
	default:
		S.error(pos, "illegal char escape");
	}
}

func (S *Scanner) scanChar() []byte {
	// '\'' already consumed

	pos := S.chpos - 1;
	ch := S.ch;
	S.next();
	if ch == '\\' {
		S.scanEscape('\'');
	}

	S.expect('\'');
	return S.src[pos : S.chpos];
}

func (S *Scanner) scanString() []byte {
	// '"' already consumed

	pos := S.chpos - 1;
	for S.ch != '"' {
		ch := S.ch;
		S.next();
		if ch == '\n' || ch < 0 {
			S.error(pos, "string not terminated");
			break;
		}
		if ch == '\\' {
			S.scanEscape('"');
		}
	}

	S.next();
	return S.src[pos : S.chpos];
}

func (S *Scanner) scanRawString() []byte {
	// '`' already consumed

	pos := S.chpos - 1;
	for S.ch != '`' {
		ch := S.ch;
		S.next();
		if ch == '\n' || ch < 0 {
			S.error(pos, "string not terminated");
			break;
		}
	}

	S.next();
	return S.src[pos : S.chpos];
}

// Helper functions for scanning multi-byte tokens such as >> += >>= .
// Different routines recognize different length tok_i based on matches
// of ch_i. If a token ends in '=', the result is tok1 or tok3
// respectively. Otherwise, the result is tok0 if there was no other
// matching character, or tok2 if the matching character was ch2.

func (S *Scanner) switch2(tok0, tok1 int) int {
	if S.ch == '=' {
		S.next();
		return tok1;
	}
	return tok0;
}

func (S *Scanner) switch3(tok0, tok1, ch2, tok2 int) int {
	if S.ch == '=' {
		S.next();
		return tok1;
	}
	if S.ch == ch2 {
		S.next();
		return tok2;
	}
	return tok0;
}

func (S *Scanner) switch4(tok0, tok1, ch2, tok2, tok3 int) int {
	if S.ch == '=' {
		S.next();
		return tok1;
	}
	if S.ch == ch2 {
		S.next();
		if S.ch == '=' {
			S.next();
			return tok3;
		}
		return tok2;
	}
	return tok0;
}

// Scans the next token. Returns the token byte position in the source,
// its token value, and the corresponding literal text if the token is
// an identifier or basic type literal (token.IsLiteral(tok) == true).

func (S *Scanner) Scan() (pos, tok int, lit []byte) {
scan_again:
	S.skipWhitespace();

	pos, tok = S.chpos, token.ILLEGAL;

	switch ch := S.ch; {
	case isLetter(ch):
		tok, lit = S.scanIdentifier();
	case digitVal(ch) < 10:
		tok, lit = S.scanNumber(false);
	default:
		S.next();  // always make progress
		switch ch {
		case -1  : tok = token.EOF;
		case '\n': tok, lit = token.COMMENT, []byte{'\n'};
		case '"' : tok, lit = token.STRING, S.scanString();
		case '\'': tok, lit = token.CHAR, S.scanChar();
		case '`' : tok, lit = token.STRING, S.scanRawString();
		case ':' : tok = S.switch2(token.COLON, token.DEFINE);
		case '.' :
			if digitVal(S.ch) < 10 {
				tok, lit = S.scanNumber(true);
			} else if S.ch == '.' {
				S.next();
				if S.ch == '.' {
					S.next();
					tok = token.ELLIPSIS;
				}
			} else {
				tok = token.PERIOD;
			}
		case ',': tok = token.COMMA;
		case ';': tok = token.SEMICOLON;
		case '(': tok = token.LPAREN;
		case ')': tok = token.RPAREN;
		case '[': tok = token.LBRACK;
		case ']': tok = token.RBRACK;
		case '{': tok = token.LBRACE;
		case '}': tok = token.RBRACE;
		case '+': tok = S.switch3(token.ADD, token.ADD_ASSIGN, '+', token.INC);
		case '-': tok = S.switch3(token.SUB, token.SUB_ASSIGN, '-', token.DEC);
		case '*': tok = S.switch2(token.MUL, token.MUL_ASSIGN);
		case '/':
			if S.ch == '/' || S.ch == '*' {
				tok, lit = token.COMMENT, S.scanComment();
				if !S.scan_comments {
					goto scan_again;
				}
			} else {
				tok = S.switch2(token.QUO, token.QUO_ASSIGN);
			}
		case '%': tok = S.switch2(token.REM, token.REM_ASSIGN);
		case '^': tok = S.switch2(token.XOR, token.XOR_ASSIGN);
		case '<':
			if S.ch == '-' {
				S.next();
				tok = token.ARROW;
			} else {
				tok = S.switch4(token.LSS, token.LEQ, '<', token.SHL, token.SHL_ASSIGN);
			}
		case '>': tok = S.switch4(token.GTR, token.GEQ, '>', token.SHR, token.SHR_ASSIGN);
		case '=': tok = S.switch2(token.ASSIGN, token.EQL);
		case '!': tok = S.switch2(token.NOT, token.NEQ);
		case '&': tok = S.switch3(token.AND, token.AND_ASSIGN, '&', token.LAND);
		case '|': tok = S.switch3(token.OR, token.OR_ASSIGN, '|', token.LOR);
		default: S.error(pos, "illegal character " + charString(ch));
		}
	}

	return pos, tok, lit;
}
<commit_msg>scanner.go documentation<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// A Go scanner. Takes a []byte as source which can then be
// tokenized through repeated calls to the Scan() function.
//
// Sample use:
//
//	import "token"
//	import "scanner"
//
//	func tokenize(src []byte) {
//		var s scanner.Scanner;
//		s.Init(src, nil /* no error handler */, false /* ignore comments */);
//		for {
//			pos, tok, lit := s.Scan();
//			if tok == Scanner.EOF {
//				return;
//			}
//			println(pos, token.TokenString(tok), string(lit));
//		}
//	}
//
package scanner

import (
	"utf8";
	"unicode";
	"strconv";
	"token";
)

// An implementation of an ErrorHandler must be provided to the Scanner.
// If a syntax error is encountered, Error() is called with the exact
// token position (the byte position of the token in the source) and the
// error message.
//
type ErrorHandler interface {
	Error(pos int, msg string);
}

// A Scanner holds the scanner's internal state while processing
// a given text.  It can be allocated as part of another data
// structure but must be initialized via Init() before use.
// See also the package comment for a sample use.
//
type Scanner struct {
	// immutable state
	src []byte;  // source
	err ErrorHandler;  // error reporting
	scan_comments bool;  // if set, comments are reported as tokens

	// scanning state
	pos int;  // current reading position
	ch int;  // one char look-ahead
	chpos int;  // position of ch
}

func isLetter(ch int) bool {
	return
		'a' <= ch && ch <= 'z' ||
		'A' <= ch && ch <= 'Z' ||
		ch == '_' ||
		ch >= 0x80 && unicode.IsLetter(ch);
}

func digitVal(ch int) int {
	switch {
	case '0' <= ch && ch <= '9': return ch - '0';
	case 'a' <= ch && ch <= 'f': return ch - 'a' + 10;
	case 'A' <= ch && ch <= 'F': return ch - 'A' + 10;
	}
	return 16;  // larger than any legal digit val
}

// Read the next Unicode char into S.ch.
// S.ch < 0 means end-of-file.
func (S *Scanner) next() {
	if S.pos < len(S.src) {
		// assume ASCII
		r, w := int(S.src[S.pos]), 1;
		if r >= 0x80 {
			// not ASCII
			r, w = utf8.DecodeRune(S.src[S.pos : len(S.src)]);
		}
		S.ch = r;
		S.chpos = S.pos;
		S.pos += w;
	} else {
		S.ch = -1;  // eof
		S.chpos = len(S.src);
	}
}

// Init() prepares the scanner S to tokenize the text src. Calls to Scan()
// will use the error handler err if they encounter a syntax error. The boolean
// scan_comments specifies whether newline characters and comments should be
// recognized and returned by Scan as token.COMMENT. If scan_comments is false,
// they are treated as white space and ignored.
//
func (S *Scanner) Init(src []byte, err ErrorHandler, scan_comments bool) {
	S.src = src;
	S.err = err;
	S.scan_comments = scan_comments;
	S.next();
}

func charString(ch int) string {
	s := string(ch);
	switch ch {
	case '\a': s = `\a`;
	case '\b': s = `\b`;
	case '\f': s = `\f`;
	case '\n': s = `\n`;
	case '\r': s = `\r`;
	case '\t': s = `\t`;
	case '\v': s = `\v`;
	case '\\': s = `\\`;
	case '\'': s = `\'`;
	}
	return "'" + s + "' (U+" + strconv.Itob(ch, 16) + ")";
}

func (S *Scanner) error(pos int, msg string) {
	S.err.Error(pos, msg);
}

func (S *Scanner) expect(ch int) {
	if S.ch != ch {
		S.error(S.chpos, "expected " + charString(ch) + ", found " + charString(S.ch));
	}
	S.next();  // always make progress
}

func (S *Scanner) skipWhitespace() {
	for {
		switch S.ch {
		case '\t', '\r', ' ':
			// nothing to do
		case '\n':
			if S.scan_comments {
				return;
			}
		default:
			return;
		}
		S.next();
	}
	panic("UNREACHABLE");
}

func (S *Scanner) scanComment() []byte {
	// first '/' already consumed
	pos := S.chpos - 1;

	if S.ch == '/' {
		//-style comment
		for S.ch >= 0 {
			S.next();
			if S.ch == '\n' {
				// '\n' terminates comment but we do not include
				// it in the comment (otherwise we don't see the
				// start of a newline in skipWhitespace()).
				return S.src[pos : S.chpos];
			}
		}

	} else {
		/*-style comment */
		S.expect('*');
		for S.ch >= 0 {
			ch := S.ch;
			S.next();
			if ch == '*' && S.ch == '/' {
				S.next();
				return S.src[pos : S.chpos];
			}
		}
	}

	S.error(pos, "comment not terminated");
	return S.src[pos : S.chpos];
}

func (S *Scanner) scanIdentifier() (tok int, lit []byte) {
	pos := S.chpos;
	for isLetter(S.ch) || digitVal(S.ch) < 10 {
		S.next();
	}
	lit = S.src[pos : S.chpos];
	return token.Lookup(lit), lit;
}

func (S *Scanner) scanMantissa(base int) {
	for digitVal(S.ch) < base {
		S.next();
	}
}

func (S *Scanner) scanNumber(seen_decimal_point bool) (tok int, lit []byte) {
	pos := S.chpos;
	tok = token.INT;

	if seen_decimal_point {
		tok = token.FLOAT;
		pos--;  // '.' is one byte
		S.scanMantissa(10);
		goto exponent;
	}

	if S.ch == '0' {
		// int or float
		S.next();
		if S.ch == 'x' || S.ch == 'X' {
			// hexadecimal int
			S.next();
			S.scanMantissa(16);
		} else {
			// octal int or float
			S.scanMantissa(8);
			if digitVal(S.ch) < 10 || S.ch == '.' || S.ch == 'e' || S.ch == 'E' {
				// float
				tok = token.FLOAT;
				goto mantissa;
			}
			// octal int
		}
		goto exit;
	}

mantissa:
	// decimal int or float
	S.scanMantissa(10);

	if S.ch == '.' {
		// float
		tok = token.FLOAT;
		S.next();
		S.scanMantissa(10)
	}

exponent:
	if S.ch == 'e' || S.ch == 'E' {
		// float
		tok = token.FLOAT;
		S.next();
		if S.ch == '-' || S.ch == '+' {
			S.next();
		}
		S.scanMantissa(10);
	}

exit:
	return tok, S.src[pos : S.chpos];
}

func (S *Scanner) scanDigits(n int, base int) {
	for digitVal(S.ch) < base {
		S.next();
		n--;
	}
	if n > 0 {
		S.error(S.chpos, "illegal char escape");
	}
}

func (S *Scanner) scanEscape(quote int) {
	ch := S.ch;
	pos := S.chpos;
	S.next();
	switch ch {
	case 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\', quote:
		// nothing to do
	case '0', '1', '2', '3', '4', '5', '6', '7':
		S.scanDigits(3 - 1, 8);  // 1 char read already
	case 'x':
		S.scanDigits(2, 16);
	case 'u':
		S.scanDigits(4, 16);
	case 'U':
		S.scanDigits(8, 16);
	default:
		S.error(pos, "illegal char escape");
	}
}

func (S *Scanner) scanChar() []byte {
	// '\'' already consumed

	pos := S.chpos - 1;
	ch := S.ch;
	S.next();
	if ch == '\\' {
		S.scanEscape('\'');
	}

	S.expect('\'');
	return S.src[pos : S.chpos];
}

func (S *Scanner) scanString() []byte {
	// '"' already consumed

	pos := S.chpos - 1;
	for S.ch != '"' {
		ch := S.ch;
		S.next();
		if ch == '\n' || ch < 0 {
			S.error(pos, "string not terminated");
			break;
		}
		if ch == '\\' {
			S.scanEscape('"');
		}
	}

	S.next();
	return S.src[pos : S.chpos];
}

func (S *Scanner) scanRawString() []byte {
	// '`' already consumed

	pos := S.chpos - 1;
	for S.ch != '`' {
		ch := S.ch;
		S.next();
		if ch == '\n' || ch < 0 {
			S.error(pos, "string not terminated");
			break;
		}
	}

	S.next();
	return S.src[pos : S.chpos];
}

// Helper functions for scanning multi-byte tokens such as >> += >>= .
// Different routines recognize different length tok_i based on matches
// of ch_i. If a token ends in '=', the result is tok1 or tok3
// respectively. Otherwise, the result is tok0 if there was no other
// matching character, or tok2 if the matching character was ch2.

func (S *Scanner) switch2(tok0, tok1 int) int {
	if S.ch == '=' {
		S.next();
		return tok1;
	}
	return tok0;
}

func (S *Scanner) switch3(tok0, tok1, ch2, tok2 int) int {
	if S.ch == '=' {
		S.next();
		return tok1;
	}
	if S.ch == ch2 {
		S.next();
		return tok2;
	}
	return tok0;
}

func (S *Scanner) switch4(tok0, tok1, ch2, tok2, tok3 int) int {
	if S.ch == '=' {
		S.next();
		return tok1;
	}
	if S.ch == ch2 {
		S.next();
		if S.ch == '=' {
			S.next();
			return tok3;
		}
		return tok2;
	}
	return tok0;
}

// Scan() scans the next token and returns the token byte position in the
// source, its token value, and the corresponding literal text if the token
// is an identifier, basic type literal (token.IsLiteral(tok) == true), or
// comment.
//
func (S *Scanner) Scan() (pos, tok int, lit []byte) {
scan_again:
	S.skipWhitespace();

	pos, tok = S.chpos, token.ILLEGAL;

	switch ch := S.ch; {
	case isLetter(ch):
		tok, lit = S.scanIdentifier();
	case digitVal(ch) < 10:
		tok, lit = S.scanNumber(false);
	default:
		S.next();  // always make progress
		switch ch {
		case -1  : tok = token.EOF;
		case '\n': tok, lit = token.COMMENT, []byte{'\n'};
		case '"' : tok, lit = token.STRING, S.scanString();
		case '\'': tok, lit = token.CHAR, S.scanChar();
		case '`' : tok, lit = token.STRING, S.scanRawString();
		case ':' : tok = S.switch2(token.COLON, token.DEFINE);
		case '.' :
			if digitVal(S.ch) < 10 {
				tok, lit = S.scanNumber(true);
			} else if S.ch == '.' {
				S.next();
				if S.ch == '.' {
					S.next();
					tok = token.ELLIPSIS;
				}
			} else {
				tok = token.PERIOD;
			}
		case ',': tok = token.COMMA;
		case ';': tok = token.SEMICOLON;
		case '(': tok = token.LPAREN;
		case ')': tok = token.RPAREN;
		case '[': tok = token.LBRACK;
		case ']': tok = token.RBRACK;
		case '{': tok = token.LBRACE;
		case '}': tok = token.RBRACE;
		case '+': tok = S.switch3(token.ADD, token.ADD_ASSIGN, '+', token.INC);
		case '-': tok = S.switch3(token.SUB, token.SUB_ASSIGN, '-', token.DEC);
		case '*': tok = S.switch2(token.MUL, token.MUL_ASSIGN);
		case '/':
			if S.ch == '/' || S.ch == '*' {
				tok, lit = token.COMMENT, S.scanComment();
				if !S.scan_comments {
					goto scan_again;
				}
			} else {
				tok = S.switch2(token.QUO, token.QUO_ASSIGN);
			}
		case '%': tok = S.switch2(token.REM, token.REM_ASSIGN);
		case '^': tok = S.switch2(token.XOR, token.XOR_ASSIGN);
		case '<':
			if S.ch == '-' {
				S.next();
				tok = token.ARROW;
			} else {
				tok = S.switch4(token.LSS, token.LEQ, '<', token.SHL, token.SHL_ASSIGN);
			}
		case '>': tok = S.switch4(token.GTR, token.GEQ, '>', token.SHR, token.SHR_ASSIGN);
		case '=': tok = S.switch2(token.ASSIGN, token.EQL);
		case '!': tok = S.switch2(token.NOT, token.NEQ);
		case '&': tok = S.switch3(token.AND, token.AND_ASSIGN, '&', token.LAND);
		case '|': tok = S.switch3(token.OR, token.OR_ASSIGN, '|', token.LOR);
		default: S.error(pos, "illegal character " + charString(ch));
		}
	}

	return pos, tok, lit;
}
<|endoftext|>


<commit_before>// Copyright 2011, 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package cloudinit_test

import (
	"fmt"
	"testing"

	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/cloudinit"
	"launchpad.net/juju-core/testing/testbase"
	sshtesting "launchpad.net/juju-core/utils/ssh/testing"
)

// TODO integration tests, but how?

type S struct {
	testbase.LoggingSuite
}

var _ = gc.Suite(S{})

func Test1(t *testing.T) {
	gc.TestingT(t)
}

var ctests = []struct {
	name      string
	expect    string
	setOption func(cfg *cloudinit.Config)
}{
	{
		"User",
		"user: me\n",
		func(cfg *cloudinit.Config) {
			cfg.SetUser("me")
		},
	},
	{
		"AptUpgrade",
		"apt_upgrade: true\n",
		func(cfg *cloudinit.Config) {
			cfg.SetAptUpgrade(true)
		},
	},
	{
		"AptUpdate",
		"apt_update: true\n",
		func(cfg *cloudinit.Config) {
			cfg.SetAptUpdate(true)
		},
	},
	{
		"AptProxy",
		"apt_proxy: http://foo.com\n",
		func(cfg *cloudinit.Config) {
			cfg.SetAptProxy("http://foo.com")
		},
	},
	{
		"AptMirror",
		"apt_mirror: http://foo.com\n",
		func(cfg *cloudinit.Config) {
			cfg.SetAptMirror("http://foo.com")
		},
	},
	{
		"AptPreserveSourcesList",
		"apt_mirror: true\n",
		func(cfg *cloudinit.Config) {
			cfg.SetAptPreserveSourcesList(true)
		},
	},
	{
		"DebconfSelections",
		"debconf_selections: '# Force debconf priority to critical.\n\n  debconf debconf/priority select critical\n\n'\n",
		func(cfg *cloudinit.Config) {
			cfg.SetDebconfSelections("# Force debconf priority to critical.\ndebconf debconf/priority select critical\n")
		},
	},
	{
		"DisableEC2Metadata",
		"disable_ec2_metadata: true\n",
		func(cfg *cloudinit.Config) {
			cfg.SetDisableEC2Metadata(true)
		},
	},
	{
		"FinalMessage",
		"final_message: goodbye\n",
		func(cfg *cloudinit.Config) {
			cfg.SetFinalMessage("goodbye")
		},
	},
	{
		"Locale",
		"locale: en_us\n",
		func(cfg *cloudinit.Config) {
			cfg.SetLocale("en_us")
		},
	},
	{
		"DisableRoot",
		"disable_root: false\n",
		func(cfg *cloudinit.Config) {
			cfg.SetDisableRoot(false)
		},
	},
	{
		"SSHAuthorizedKeys",
		fmt.Sprintf(
			"ssh_authorized_keys:\n- %s Juju:user@host\n- %s Juju:another@host\n",
			sshtesting.ValidKeyOne.Key, sshtesting.ValidKeyTwo.Key),
		func(cfg *cloudinit.Config) {
			cfg.AddSSHAuthorizedKeys(sshtesting.ValidKeyOne.Key + " Juju:user@host")
			cfg.AddSSHAuthorizedKeys(sshtesting.ValidKeyTwo.Key + " another@host")
		},
	},
	{
		"SSHAuthorizedKeys",
		fmt.Sprintf(
			"ssh_authorized_keys:\n- %s Juju:sshkey\n- %s Juju:user@host\n- %s Juju:another@host\n",
			sshtesting.ValidKeyOne.Key, sshtesting.ValidKeyTwo.Key, sshtesting.ValidKeyThree.Key),
		func(cfg *cloudinit.Config) {
			cfg.AddSSHAuthorizedKeys("#command\n" + sshtesting.ValidKeyOne.Key)
			cfg.AddSSHAuthorizedKeys(
				sshtesting.ValidKeyTwo.Key + " user@host\n# comment\n\n" +
					sshtesting.ValidKeyThree.Key + " another@host")
			cfg.AddSSHAuthorizedKeys("")
		},
	},
	{
		"SSHKeys RSAPrivate",
		"ssh_keys:\n  rsa_private: key1data\n",
		func(cfg *cloudinit.Config) {
			cfg.AddSSHKey(cloudinit.RSAPrivate, "key1data")
		},
	},
	{
		"SSHKeys RSAPublic",
		"ssh_keys:\n  rsa_public: key2data\n",
		func(cfg *cloudinit.Config) {
			cfg.AddSSHKey(cloudinit.RSAPublic, "key2data")
		},
	},
	{
		"SSHKeys DSAPublic",
		"ssh_keys:\n  dsa_public: key1data\n",
		func(cfg *cloudinit.Config) {
			cfg.AddSSHKey(cloudinit.DSAPublic, "key1data")
		},
	},
	{
		"SSHKeys DSAPrivate",
		"ssh_keys:\n  dsa_private: key2data\n",
		func(cfg *cloudinit.Config) {
			cfg.AddSSHKey(cloudinit.DSAPrivate, "key2data")
		},
	},
	{
		"Output",
		"output:\n  all:\n  - '>foo'\n  - '|bar'\n",
		func(cfg *cloudinit.Config) {
			cfg.SetOutput("all", ">foo", "|bar")
		},
	},
	{
		"Output",
		"output:\n  all: '>foo'\n",
		func(cfg *cloudinit.Config) {
			cfg.SetOutput(cloudinit.OutAll, ">foo", "")
		},
	},
	{
		"AptSources",
		"apt_sources:\n- source: keyName\n  key: someKey\n",
		func(cfg *cloudinit.Config) {
			cfg.AddAptSource("keyName", "someKey")
		},
	},
	{
		"Packages",
		"packages:\n- juju\n- ubuntu\n",
		func(cfg *cloudinit.Config) {
			cfg.AddPackage("juju")
			cfg.AddPackage("ubuntu")
		},
	},
	{
		"BootCmd",
		"bootcmd:\n- ls > /dev\n- - ls\n  - '>with space'\n",
		func(cfg *cloudinit.Config) {
			cfg.AddBootCmd("ls > /dev")
			cfg.AddBootCmdArgs("ls", ">with space")
		},
	},
	{
		"Mounts",
		"mounts:\n- - x\n  - \"y\"\n- - z\n  - w\n",
		func(cfg *cloudinit.Config) {
			cfg.AddMount("x", "y")
			cfg.AddMount("z", "w")
		},
	},
	{
		"Attr",
		"arbitraryAttr: someValue\n",
		func(cfg *cloudinit.Config) {
			cfg.SetAttr("arbitraryAttr", "someValue")
		},
	},
	{
		"RunCmd",
		"runcmd:\n- ifconfig\n",
		func(cfg *cloudinit.Config) {
			cfg.AddRunCmd("ifconfig")
		},
	},
	{
		"AddScripts",
		"runcmd:\n- echo 'Hello World'\n- ifconfig\n",
		func(cfg *cloudinit.Config) {
			cfg.AddScripts(
				"echo 'Hello World'",
				"ifconfig",
			)
		},
	},
	{
		"AddFile",
		addFileExpected,
		func(cfg *cloudinit.Config) {
			cfg.AddFile(
				"/etc/apt/apt.conf.d/99proxy",
				`"Acquire::http::Proxy "http://10.0.3.1:3142";`,
				0644,
			)
		},
	},
}

const (
	header          = "#cloud-config\n"
	addFileExpected = `runcmd:
- install -m 644 /dev/null '/etc/apt/apt.conf.d/99proxy'
- printf '%s\n' '"Acquire::http::Proxy "http://10.0.3.1:3142";' > '/etc/apt/apt.conf.d/99proxy'
`
)

func (S) TestOutput(c *gc.C) {
	for _, t := range ctests {
		cfg := cloudinit.New()
		t.setOption(cfg)
		data, err := cfg.Render()
		c.Assert(err, gc.IsNil)
		c.Assert(data, gc.NotNil)
		c.Assert(string(data), gc.Equals, header+t.expect, gc.Commentf("test %q output differs", t.name))
	}
}

func (S) TestRunCmds(c *gc.C) {
	cfg := cloudinit.New()
	c.Assert(cfg.RunCmds(), gc.HasLen, 0)
	cfg.AddScripts("a", "b")
	cfg.AddRunCmdArgs("c", "d")
	cfg.AddRunCmd("e")
	c.Assert(cfg.RunCmds(), gc.DeepEquals, []interface{}{
			"a", "b", []string{"c", "d"}, "e",
		})
}

func (S) TestPackages(c *gc.C) {
	cfg := cloudinit.New()
	c.Assert(cfg.Packages(), gc.HasLen, 0)
	cfg.AddPackage("a b c")
	cfg.AddPackage("d!")
	c.Assert(cfg.Packages(), gc.DeepEquals, []string{"a b c", "d!"})
}

func (S) TestSetOutput(c *gc.C) {
	type test struct {
		kind   cloudinit.OutputKind
		stdout string
		stderr string
	}
	tests := []test{{
		cloudinit.OutAll, "a", "",
	}, {
		cloudinit.OutAll, "", "b",
	}, {
		cloudinit.OutInit, "a", "b",
	}, {
		cloudinit.OutAll, "a", "b",
	}, {
		cloudinit.OutAll, "", "",
	}}

	cfg := cloudinit.New()
	stdout, stderr := cfg.Output(cloudinit.OutAll)
	c.Assert(stdout, gc.Equals, "")
	c.Assert(stderr, gc.Equals, "")
	for i, t := range tests {
		c.Logf("test %d: %+v", i, t)
		cfg.SetOutput(t.kind, t.stdout, t.stderr)
		stdout, stderr = cfg.Output(t.kind)
		c.Assert(stdout, gc.Equals, t.stdout)
		c.Assert(stderr, gc.Equals, t.stderr)
	}
}

//#cloud-config
//packages:
//- juju
//- ubuntu
func ExampleConfig() {
	cfg := cloudinit.New()
	cfg.AddPackage("juju")
	cfg.AddPackage("ubuntu")
	data, err := cfg.Render()
	if err != nil {
		fmt.Printf("render error: %v", err)
		return
	}
	fmt.Printf("%s", data)
}
<commit_msg>go fmt<commit_after>// Copyright 2011, 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package cloudinit_test

import (
	"fmt"
	"testing"

	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/cloudinit"
	"launchpad.net/juju-core/testing/testbase"
	sshtesting "launchpad.net/juju-core/utils/ssh/testing"
)

// TODO integration tests, but how?

type S struct {
	testbase.LoggingSuite
}

var _ = gc.Suite(S{})

func Test1(t *testing.T) {
	gc.TestingT(t)
}

var ctests = []struct {
	name      string
	expect    string
	setOption func(cfg *cloudinit.Config)
}{
	{
		"User",
		"user: me\n",
		func(cfg *cloudinit.Config) {
			cfg.SetUser("me")
		},
	},
	{
		"AptUpgrade",
		"apt_upgrade: true\n",
		func(cfg *cloudinit.Config) {
			cfg.SetAptUpgrade(true)
		},
	},
	{
		"AptUpdate",
		"apt_update: true\n",
		func(cfg *cloudinit.Config) {
			cfg.SetAptUpdate(true)
		},
	},
	{
		"AptProxy",
		"apt_proxy: http://foo.com\n",
		func(cfg *cloudinit.Config) {
			cfg.SetAptProxy("http://foo.com")
		},
	},
	{
		"AptMirror",
		"apt_mirror: http://foo.com\n",
		func(cfg *cloudinit.Config) {
			cfg.SetAptMirror("http://foo.com")
		},
	},
	{
		"AptPreserveSourcesList",
		"apt_mirror: true\n",
		func(cfg *cloudinit.Config) {
			cfg.SetAptPreserveSourcesList(true)
		},
	},
	{
		"DebconfSelections",
		"debconf_selections: '# Force debconf priority to critical.\n\n  debconf debconf/priority select critical\n\n'\n",
		func(cfg *cloudinit.Config) {
			cfg.SetDebconfSelections("# Force debconf priority to critical.\ndebconf debconf/priority select critical\n")
		},
	},
	{
		"DisableEC2Metadata",
		"disable_ec2_metadata: true\n",
		func(cfg *cloudinit.Config) {
			cfg.SetDisableEC2Metadata(true)
		},
	},
	{
		"FinalMessage",
		"final_message: goodbye\n",
		func(cfg *cloudinit.Config) {
			cfg.SetFinalMessage("goodbye")
		},
	},
	{
		"Locale",
		"locale: en_us\n",
		func(cfg *cloudinit.Config) {
			cfg.SetLocale("en_us")
		},
	},
	{
		"DisableRoot",
		"disable_root: false\n",
		func(cfg *cloudinit.Config) {
			cfg.SetDisableRoot(false)
		},
	},
	{
		"SSHAuthorizedKeys",
		fmt.Sprintf(
			"ssh_authorized_keys:\n- %s Juju:user@host\n- %s Juju:another@host\n",
			sshtesting.ValidKeyOne.Key, sshtesting.ValidKeyTwo.Key),
		func(cfg *cloudinit.Config) {
			cfg.AddSSHAuthorizedKeys(sshtesting.ValidKeyOne.Key + " Juju:user@host")
			cfg.AddSSHAuthorizedKeys(sshtesting.ValidKeyTwo.Key + " another@host")
		},
	},
	{
		"SSHAuthorizedKeys",
		fmt.Sprintf(
			"ssh_authorized_keys:\n- %s Juju:sshkey\n- %s Juju:user@host\n- %s Juju:another@host\n",
			sshtesting.ValidKeyOne.Key, sshtesting.ValidKeyTwo.Key, sshtesting.ValidKeyThree.Key),
		func(cfg *cloudinit.Config) {
			cfg.AddSSHAuthorizedKeys("#command\n" + sshtesting.ValidKeyOne.Key)
			cfg.AddSSHAuthorizedKeys(
				sshtesting.ValidKeyTwo.Key + " user@host\n# comment\n\n" +
					sshtesting.ValidKeyThree.Key + " another@host")
			cfg.AddSSHAuthorizedKeys("")
		},
	},
	{
		"SSHKeys RSAPrivate",
		"ssh_keys:\n  rsa_private: key1data\n",
		func(cfg *cloudinit.Config) {
			cfg.AddSSHKey(cloudinit.RSAPrivate, "key1data")
		},
	},
	{
		"SSHKeys RSAPublic",
		"ssh_keys:\n  rsa_public: key2data\n",
		func(cfg *cloudinit.Config) {
			cfg.AddSSHKey(cloudinit.RSAPublic, "key2data")
		},
	},
	{
		"SSHKeys DSAPublic",
		"ssh_keys:\n  dsa_public: key1data\n",
		func(cfg *cloudinit.Config) {
			cfg.AddSSHKey(cloudinit.DSAPublic, "key1data")
		},
	},
	{
		"SSHKeys DSAPrivate",
		"ssh_keys:\n  dsa_private: key2data\n",
		func(cfg *cloudinit.Config) {
			cfg.AddSSHKey(cloudinit.DSAPrivate, "key2data")
		},
	},
	{
		"Output",
		"output:\n  all:\n  - '>foo'\n  - '|bar'\n",
		func(cfg *cloudinit.Config) {
			cfg.SetOutput("all", ">foo", "|bar")
		},
	},
	{
		"Output",
		"output:\n  all: '>foo'\n",
		func(cfg *cloudinit.Config) {
			cfg.SetOutput(cloudinit.OutAll, ">foo", "")
		},
	},
	{
		"AptSources",
		"apt_sources:\n- source: keyName\n  key: someKey\n",
		func(cfg *cloudinit.Config) {
			cfg.AddAptSource("keyName", "someKey")
		},
	},
	{
		"Packages",
		"packages:\n- juju\n- ubuntu\n",
		func(cfg *cloudinit.Config) {
			cfg.AddPackage("juju")
			cfg.AddPackage("ubuntu")
		},
	},
	{
		"BootCmd",
		"bootcmd:\n- ls > /dev\n- - ls\n  - '>with space'\n",
		func(cfg *cloudinit.Config) {
			cfg.AddBootCmd("ls > /dev")
			cfg.AddBootCmdArgs("ls", ">with space")
		},
	},
	{
		"Mounts",
		"mounts:\n- - x\n  - \"y\"\n- - z\n  - w\n",
		func(cfg *cloudinit.Config) {
			cfg.AddMount("x", "y")
			cfg.AddMount("z", "w")
		},
	},
	{
		"Attr",
		"arbitraryAttr: someValue\n",
		func(cfg *cloudinit.Config) {
			cfg.SetAttr("arbitraryAttr", "someValue")
		},
	},
	{
		"RunCmd",
		"runcmd:\n- ifconfig\n",
		func(cfg *cloudinit.Config) {
			cfg.AddRunCmd("ifconfig")
		},
	},
	{
		"AddScripts",
		"runcmd:\n- echo 'Hello World'\n- ifconfig\n",
		func(cfg *cloudinit.Config) {
			cfg.AddScripts(
				"echo 'Hello World'",
				"ifconfig",
			)
		},
	},
	{
		"AddFile",
		addFileExpected,
		func(cfg *cloudinit.Config) {
			cfg.AddFile(
				"/etc/apt/apt.conf.d/99proxy",
				`"Acquire::http::Proxy "http://10.0.3.1:3142";`,
				0644,
			)
		},
	},
}

const (
	header          = "#cloud-config\n"
	addFileExpected = `runcmd:
- install -m 644 /dev/null '/etc/apt/apt.conf.d/99proxy'
- printf '%s\n' '"Acquire::http::Proxy "http://10.0.3.1:3142";' > '/etc/apt/apt.conf.d/99proxy'
`
)

func (S) TestOutput(c *gc.C) {
	for _, t := range ctests {
		cfg := cloudinit.New()
		t.setOption(cfg)
		data, err := cfg.Render()
		c.Assert(err, gc.IsNil)
		c.Assert(data, gc.NotNil)
		c.Assert(string(data), gc.Equals, header+t.expect, gc.Commentf("test %q output differs", t.name))
	}
}

func (S) TestRunCmds(c *gc.C) {
	cfg := cloudinit.New()
	c.Assert(cfg.RunCmds(), gc.HasLen, 0)
	cfg.AddScripts("a", "b")
	cfg.AddRunCmdArgs("c", "d")
	cfg.AddRunCmd("e")
	c.Assert(cfg.RunCmds(), gc.DeepEquals, []interface{}{
		"a", "b", []string{"c", "d"}, "e",
	})
}

func (S) TestPackages(c *gc.C) {
	cfg := cloudinit.New()
	c.Assert(cfg.Packages(), gc.HasLen, 0)
	cfg.AddPackage("a b c")
	cfg.AddPackage("d!")
	c.Assert(cfg.Packages(), gc.DeepEquals, []string{"a b c", "d!"})
}

func (S) TestSetOutput(c *gc.C) {
	type test struct {
		kind   cloudinit.OutputKind
		stdout string
		stderr string
	}
	tests := []test{{
		cloudinit.OutAll, "a", "",
	}, {
		cloudinit.OutAll, "", "b",
	}, {
		cloudinit.OutInit, "a", "b",
	}, {
		cloudinit.OutAll, "a", "b",
	}, {
		cloudinit.OutAll, "", "",
	}}

	cfg := cloudinit.New()
	stdout, stderr := cfg.Output(cloudinit.OutAll)
	c.Assert(stdout, gc.Equals, "")
	c.Assert(stderr, gc.Equals, "")
	for i, t := range tests {
		c.Logf("test %d: %+v", i, t)
		cfg.SetOutput(t.kind, t.stdout, t.stderr)
		stdout, stderr = cfg.Output(t.kind)
		c.Assert(stdout, gc.Equals, t.stdout)
		c.Assert(stderr, gc.Equals, t.stderr)
	}
}

//#cloud-config
//packages:
//- juju
//- ubuntu
func ExampleConfig() {
	cfg := cloudinit.New()
	cfg.AddPackage("juju")
	cfg.AddPackage("ubuntu")
	data, err := cfg.Render()
	if err != nil {
		fmt.Printf("render error: %v", err)
		return
	}
	fmt.Printf("%s", data)
}
<|endoftext|>


<commit_before>package cluster

import (
	"fmt"

	"github.com/AsynkronIT/protoactor-go/actor"
)

var (
	membershipPID *actor.PID
)

func spawnMembershipActor() {
	membershipPID = actor.SpawnNamed(actor.FromProducer(NewMembershipActor()), "#membership")
}

func NewMembershipActor() actor.Producer {
	return func() actor.Actor {
		return &membershipActor{}
	}
}

func init() {
	spawnMembershipActor()

	//subscribe the membership actor to the MemberStatusBatch event
	actor.EventStream.SubscribePID(func(m interface{}) bool {
		_, ok := m.(MemberStatusBatch)
		return ok
	}, membershipPID)
}

//membershipActor is responsible to keep track of the current cluster topology
//it does so by listening to changes from the ClusterProvider.
//the default ClusterProvider is consul_cluster.ConsulProvider which uses the Consul HTTP API to scan for changes
//TODO: we need some way of creating a hashring per "kind", maybe we should have a child actor to the membership actor that handles nodes
//per kind.
type membershipActor struct {
	members map[string]*MemberStatus
}

type MemberStatusEvent interface {
	MemberStatusEvent()
}

type MemberEvent struct {
	Address string
	Port    int
}

func (*MemberEvent) MemberStatusEvent() {}

type MemberJoinedEvent struct {
	MemberEvent
}

type MemberLeftEvent struct {
	MemberEvent
}

type MemberUnavailableEvent struct {
	MemberEvent
}

type MemberAvailableEvent struct {
	MemberEvent
}

func (a *membershipActor) Receive(ctx actor.Context) {
	switch msg := ctx.Message().(type) {
	case *actor.Started:
		a.members = make(map[string]*MemberStatus)
	case MemberStatusBatch:
		//TODO: keys that are present in the map but not in the message, are nodes that have left/been deregistered
		//we need to handle this too..
		for _, new := range msg {

			//key is address:port
			key := fmt.Sprintf("%v:%v", new.Address, new.Port)
			old := a.members[key]
			a.members[key] = new
			address := MemberEvent{
				Address: new.Address,
				Port:    new.Port,
			}
			if old == nil {
				//notify joined
				joined := &MemberJoinedEvent{MemberEvent: address}
				actor.EventStream.Publish(joined)
			} else {
				if old.Alive && !new.Alive {
					//notify node unavailable
					unavailable := &MemberUnavailableEvent{MemberEvent: address}
					actor.EventStream.Publish(unavailable)
				} else if !old.Alive && new.Alive {
					//notify node reachable
					available := &MemberAvailableEvent{MemberEvent: address}
					actor.EventStream.Publish(available)
				} else {
					//Ignore, no change...
				}
			}
		}
	}
}
<commit_msg>cluster status notifications<commit_after>package cluster

import (
	"fmt"

	"github.com/AsynkronIT/protoactor-go/actor"
)

var (
	membershipPID *actor.PID
)

func spawnMembershipActor() {
	membershipPID = actor.SpawnNamed(actor.FromProducer(newMembershipActor()), "#membership")
}

func newMembershipActor() actor.Producer {
	return func() actor.Actor {
		return &membershipActor{}
	}
}

func init() {
	spawnMembershipActor()

	//subscribe the membership actor to the MemberStatusBatch event
	actor.EventStream.SubscribePID(func(m interface{}) bool {
		_, ok := m.(MemberStatusBatch)
		return ok
	}, membershipPID)
}

//membershipActor is responsible to keep track of the current cluster topology
//it does so by listening to changes from the ClusterProvider.
//the default ClusterProvider is consul_cluster.ConsulProvider which uses the Consul HTTP API to scan for changes
//TODO: we need some way of creating a hashring per "kind", maybe we should have a child actor to the membership actor that handles nodes
//per kind.
type membershipActor struct {
	members map[string]*MemberStatus
}

type MemberStatusEvent interface {
	MemberStatusEvent()
}

type MemberEvent struct {
	Address string
	Port    int
}

func (*MemberEvent) MemberStatusEvent() {}

type MemberJoinedEvent struct {
	MemberEvent
}

type MemberLeftEvent struct {
	MemberEvent
}

type MemberUnavailableEvent struct {
	MemberEvent
}

type MemberAvailableEvent struct {
	MemberEvent
}

func (a *membershipActor) Receive(ctx actor.Context) {
	switch msg := ctx.Message().(type) {
	case *actor.Started:
		a.members = make(map[string]*MemberStatus)
	case MemberStatusBatch:

		//build a lookup for the new statuses
		tmp := make(map[string]*MemberStatus)
		for _, new := range msg {
			//key is address:port
			key := fmt.Sprintf("%v:%v", new.Address, new.Port)
			tmp[key] = new
		}

		//find the entires that only exist in the old set but not in the new
		for key, old := range a.members {
			new := tmp[key]
			if new == nil {
				a.notify(new, old)
			}
		}

		//find all the entries that exist in the new set
		for key, new := range tmp {
			old := a.members[key]
			a.members[key] = new
			a.notify(new, old)
		}
	}
}

func (a *membershipActor) notify(new *MemberStatus, old *MemberStatus) {
	address := MemberEvent{
		Address: new.Address,
		Port:    new.Port,
	}
	if new == nil && old == nil {
		//ignore, not possible
		return
	}
	if new == nil {
		//notify left
		left := &MemberLeftEvent{MemberEvent: address}
		actor.EventStream.Publish(left)
		return
	}
	if old == nil {
		//notify joined
		joined := &MemberJoinedEvent{MemberEvent: address}
		actor.EventStream.Publish(joined)
		return
	}
	if old.Alive && !new.Alive {
		//notify node unavailable
		unavailable := &MemberUnavailableEvent{MemberEvent: address}
		actor.EventStream.Publish(unavailable)
		return
	}
	if !old.Alive && new.Alive {
		//notify node reachable
		available := &MemberAvailableEvent{MemberEvent: address}
		actor.EventStream.Publish(available)
	}
}
<|endoftext|>


<commit_before>package store

import (
  "encoding/json"
  "fmt"
  "github.com/vole/gouuid"
  "path"
  "time"
)

/**
 * Post.
 */
type Post struct {
  // Properties that should be saved to disk.
  Id      string `json:"id"`
  Title   string `json:"title"`
  Created int64  `json:"created"`

  // Properties that are used by Vole backend and frontend, but not saved to disk
  // when the post is marshaled.
  UserId     string `json:"user_id,omitempty"`
  UserName   string `json:"user_name,omitempty"`
  UserAvatar string `json:"user_avatar,omitempty"`
  IsMyPost   bool   `json:"is_my_post,omitempty"`

  // Properties that are only used by the backend and thus don't have
  // to be marshaled to JSON for either the frontend or disk.
  FullPath string `json:"-"`
}

/**
 * InitNew()
 *
 * Initialize a new post creating the id and other fields.
 */
func (post *Post) InitNew(title, userPath, userId, userName, userAvatar string, isMyUser bool) {
  // Create a new UUID
  uuidBytes, _ := uuid.NewV4()
  uuid := fmt.Sprintf("%s", uuidBytes)

  // Get the timestamp.
  created := time.Now().UnixNano()

  // The full path to the post.
  fullPath := path.Join(userPath, "posts", fmt.Sprintf("%d-post-%s.json", created, uuid))

  post.Id = uuid
  post.Title = title
  post.Created = created
  post.UserId = userId
  post.UserName = userName
  post.UserAvatar = userAvatar
  post.IsMyPost = isMyUser
  post.FullPath = fullPath
}

/**
 * InitFromJson()
 *
 * Initialize a new post from json data from disk.
 */
func (post *Post) InitFromJson(rawJson []byte, fullPath string, userId string, userName string, userAvatar string, isMyUser bool) error {
  if err := json.Unmarshal(rawJson, post); err != nil {
    return err
  }
  post.UserId = userId
  post.UserName = userName
  post.UserAvatar = userAvatar
  post.IsMyPost = isMyUser
  post.FullPath = fullPath
  return nil
}

/**
 * Save()
 *
 * Save post to disk.
 */
func (post *Post) Save() error {
  // Before marshaling JSON for saving to disk, we set all properties
  // that should not be saved to empty, so they are ignored by marshaller.
  postClone := *post
  postClone.UserId = ""
  postClone.UserName = ""
  postClone.UserAvatar = ""

  rawJson, err := json.Marshal(postClone)
  if err != nil {
    return err
  }

  return Write(postClone.FullPath, rawJson)
}

func (post *Post) Delete() error {
  return Delete(post.FullPath)
}

/**
 * Collection()
 *
 * Return a post collection wrapping this user.
 */
func (post *Post) Collection() *PostCollection {
  return &PostCollection{[]Post{*post}}
}

/**
 * Container()
 *
 * Return a post container wrapping this post.
 */
func (post *Post) Container() *PostContainer {
  return &PostContainer{*post}
}
<commit_msg>Dont save ismypost<commit_after>package store

import (
  "encoding/json"
  "fmt"
  "github.com/vole/gouuid"
  "path"
  "time"
)

/**
 * Post.
 */
type Post struct {
  // Properties that should be saved to disk.
  Id      string `json:"id"`
  Title   string `json:"title"`
  Created int64  `json:"created"`

  // Properties that are used by Vole backend and frontend, but not saved to disk
  // when the post is marshaled.
  UserId     string `json:"user_id,omitempty"`
  UserName   string `json:"user_name,omitempty"`
  UserAvatar string `json:"user_avatar,omitempty"`
  IsMyPost   bool   `json:"is_my_post,omitempty"`

  // Properties that are only used by the backend and thus don't have
  // to be marshaled to JSON for either the frontend or disk.
  FullPath string `json:"-"`
}

/**
 * InitNew()
 *
 * Initialize a new post creating the id and other fields.
 */
func (post *Post) InitNew(title, userPath, userId, userName, userAvatar string, isMyUser bool) {
  // Create a new UUID
  uuidBytes, _ := uuid.NewV4()
  uuid := fmt.Sprintf("%s", uuidBytes)

  // Get the timestamp.
  created := time.Now().UnixNano()

  // The full path to the post.
  fullPath := path.Join(userPath, "posts", fmt.Sprintf("%d-post-%s.json", created, uuid))

  post.Id = uuid
  post.Title = title
  post.Created = created
  post.UserId = userId
  post.UserName = userName
  post.UserAvatar = userAvatar
  post.IsMyPost = isMyUser
  post.FullPath = fullPath
}

/**
 * InitFromJson()
 *
 * Initialize a new post from json data from disk.
 */
func (post *Post) InitFromJson(rawJson []byte, fullPath string, userId string, userName string, userAvatar string, isMyUser bool) error {
  if err := json.Unmarshal(rawJson, post); err != nil {
    return err
  }
  post.UserId = userId
  post.UserName = userName
  post.UserAvatar = userAvatar
  post.IsMyPost = isMyUser
  post.FullPath = fullPath
  return nil
}

/**
 * Save()
 *
 * Save post to disk.
 */
func (post *Post) Save() error {
  // Before marshaling JSON for saving to disk, we set all properties
  // that should not be saved to empty, so they are ignored by marshaller.
  postClone := *post
  postClone.UserId = ""
  postClone.UserName = ""
  postClone.UserAvatar = ""
  postClone.IsMyPost = false

  rawJson, err := json.Marshal(postClone)
  if err != nil {
    return err
  }

  return Write(postClone.FullPath, rawJson)
}

func (post *Post) Delete() error {
  return Delete(post.FullPath)
}

/**
 * Collection()
 *
 * Return a post collection wrapping this user.
 */
func (post *Post) Collection() *PostCollection {
  return &PostCollection{[]Post{*post}}
}

/**
 * Container()
 *
 * Return a post container wrapping this post.
 */
func (post *Post) Container() *PostContainer {
  return &PostContainer{*post}
}
<|endoftext|>


<commit_before>package main

import (
    "github.com/qmsk/clusterf/config"
    "github.com/qmsk/clusterf/docker"
	"github.com/jessevdk/go-flags"
    "log"
	"os"
	"os/signal"
	"syscall"
)

var Options struct {
	ConfigWriter	config.WriterOptions	`group:"Config Writer"`
	Docker			docker.Options

	ExitFlush		bool	`long:"exit-flush" description:"Flush backends on exit signal"`

	RouteNetwork	string  `long:"route-network" value-name:"NETWORK-NAME" description:"Advertise docker network by name"`
	RouteGateway4	string	`long:"route-gateway4" value-name:"IPV4-ADDRESS" description:"Advertise docker network routes with IPv4 gateway"`
	RouteGateway6	string	`long:"route-gateway6" value-name:"IPV6-ADDRESS" description:"Advertise docker network routes with IPv6 gateway"`
	RouteIPVSMethod string	`long:"route-ipvs-method" value-name:"masq|tunnel|droute" description:"Advertise docker network routes with ipvs-method"`
}

var flagsParser = flags.NewParser(&Options,  flags.Default)

// Flush service backends when stopping
func stop(configWriter *config.Writer) {
	log.Printf("Flush.....")

	if err := configWriter.Flush(); err != nil {
		log.Fatalf("config:Writer.Flush: %v", err)
	} else {
		log.Printf("Flushed")
	}
}

// Listen for updated docker.State, compile to config.Config and update config.Writer.
//
// Stops on os.Signal
func run(configWriter *config.Writer, dockerListen chan docker.State, stopChan chan os.Signal) {
	defer stop(configWriter)

	for {
		select {
		case dockerState, ok := <-dockerListen:
			if !ok {
				// docker quit? exit and restart
				return
			}

			if config, err := makeConfig(dockerState); err != nil {
				log.Fatalf("configContainers: %v", err)
			} else if err := configWriter.Write(config); err != nil {
				log.Fatalf("config:Writer.Write: %v", err)
			} else {
				log.Printf("Update config...")
			}

		case s := <-stopChan:
			log.Printf("Stopping on %v...", s)

			// reset signal in case stopping gets stuck
			signal.Stop(stopChan)

			return
		}
	}

	log.Printf("Stop...")
}

func main() {
	if _, err := flagsParser.Parse(); err != nil {
		log.Fatalf("flags.Parser.Parse: %v", err)
	}

	configWriter, err := Options.ConfigWriter.Writer()
	if err != nil {
		log.Fatalf("config.Writer: %v", err)
	}

    docker, err := Options.Docker.Open()
	if err != nil {
        log.Fatalf("docker:Docker.Open: %v", err)
    } else {
        log.Printf("docker:Docker.Open: %v", docker)
    }

    dockerChan, err := docker.Listen()
	if err != nil {
        log.Fatalf("docker:Docker.Listen: %v", err)
    } else {
        log.Printf("docker:Docker.Listen...")
	}

	// optionally arrange to stop on signal
	var stopChan chan os.Signal

	if Options.ExitFlush {
		stopChan = make(chan os.Signal)

		signal.Notify(stopChan, syscall.SIGINT, syscall.SIGTERM)
	}

	// mainloop
	run(configWriter, dockerChan, stopChan)
}
<commit_msg>cmd/clusterf-docker: fix stop logging<commit_after>package main

import (
    "github.com/qmsk/clusterf/config"
    "github.com/qmsk/clusterf/docker"
	"github.com/jessevdk/go-flags"
    "log"
	"os"
	"os/signal"
	"syscall"
)

var Options struct {
	ConfigWriter	config.WriterOptions	`group:"Config Writer"`
	Docker			docker.Options

	ExitFlush		bool	`long:"exit-flush" description:"Flush backends on exit signal"`

	RouteNetwork	string  `long:"route-network" value-name:"NETWORK-NAME" description:"Advertise docker network by name"`
	RouteGateway4	string	`long:"route-gateway4" value-name:"IPV4-ADDRESS" description:"Advertise docker network routes with IPv4 gateway"`
	RouteGateway6	string	`long:"route-gateway6" value-name:"IPV6-ADDRESS" description:"Advertise docker network routes with IPv6 gateway"`
	RouteIPVSMethod string	`long:"route-ipvs-method" value-name:"masq|tunnel|droute" description:"Advertise docker network routes with ipvs-method"`
}

var flagsParser = flags.NewParser(&Options,  flags.Default)

// Flush service backends when stopping
func stop(configWriter *config.Writer) {
	log.Printf("Flush.....")

	if err := configWriter.Flush(); err != nil {
		log.Fatalf("config:Writer.Flush: %v", err)
	} else {
		log.Printf("Flushed")
	}
}

// Listen for updated docker.State, compile to config.Config and update config.Writer.
//
// Stops on os.Signal
func run(configWriter *config.Writer, dockerListen chan docker.State, stopChan chan os.Signal) {
	defer stop(configWriter)

	for {
		select {
		case dockerState, ok := <-dockerListen:
			if !ok {
				// docker quit? exit and restart
				log.Printf("Stopping on Docker close...")
				return
			}

			if config, err := makeConfig(dockerState); err != nil {
				log.Fatalf("configContainers: %v", err)
			} else if err := configWriter.Write(config); err != nil {
				log.Fatalf("config:Writer.Write: %v", err)
			} else {
				log.Printf("Update config...")
			}

		case s := <-stopChan:
			log.Printf("Stopping on %v...", s)

			// reset signal in case stopping gets stuck
			signal.Stop(stopChan)

			return
		}
	}
}

func main() {
	if _, err := flagsParser.Parse(); err != nil {
		log.Fatalf("flags.Parser.Parse: %v", err)
	}

	configWriter, err := Options.ConfigWriter.Writer()
	if err != nil {
		log.Fatalf("config.Writer: %v", err)
	}

    docker, err := Options.Docker.Open()
	if err != nil {
        log.Fatalf("docker:Docker.Open: %v", err)
    } else {
        log.Printf("docker:Docker.Open: %v", docker)
    }

    dockerChan, err := docker.Listen()
	if err != nil {
        log.Fatalf("docker:Docker.Listen: %v", err)
    } else {
        log.Printf("docker:Docker.Listen...")
	}

	// optionally arrange to stop on signal
	var stopChan chan os.Signal

	if Options.ExitFlush {
		stopChan = make(chan os.Signal)

		signal.Notify(stopChan, syscall.SIGINT, syscall.SIGTERM)
	}

	// mainloop
	run(configWriter, dockerChan, stopChan)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"testing"
)

func BenchmarkRun(b *testing.B) {
	opt := &option{}
	for i := 0; i < b.N; i++ {
		buf := new(bytes.Buffer)
		if err := run(buf, []string{"github.com/haya14busa/go-typeconv"}, opt); err != nil {
			b.Fatal(err)
		}
	}
}
<commit_msg>cmd: add test for run()<commit_after>package main

import (
	"bytes"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"testing"

	ddiff "github.com/kylelemons/godebug/diff"
)

func TestRun_package(t *testing.T) {
	opt := &option{}
	buf := new(bytes.Buffer)
	if err := run(buf, []string{"github.com/haya14busa/go-typeconv"}, opt); err != nil {
		t.Fatal(err)
	}
	if buf.Len() == 0 {
		t.Error("run: output is empty")
	}
}

func TestRun_testdata(t *testing.T) {
	opt := &option{}
	files, err := filepath.Glob("../../testdata/*.input.go")
	if err != nil {
		t.Fatal(err)
	}
	for _, fname := range files {
		input := fname
		golden := strings.Replace(input, "input.go", "golden.go", 1)
		buf := new(bytes.Buffer)
		if err := run(buf, []string{input}, opt); err != nil {
			t.Fatal(err)
		}
		gf, err := os.Open(golden)
		if err != nil {
			t.Fatalf("%s: %v", fname, err)
		}
		defer gf.Close()
		b, err := ioutil.ReadAll(gf)
		if err != nil {
			t.Fatalf("%s: %v", fname, err)
		}

		if d := ddiff.Diff(buf.String(), string(b)); d != "" {
			t.Errorf("%s: diff: (-got +want):\n%s", fname, d)
		}
	}
}

func BenchmarkRun(b *testing.B) {
	opt := &option{}
	for i := 0; i < b.N; i++ {
		buf := new(bytes.Buffer)
		if err := run(buf, []string{"github.com/haya14busa/go-typeconv"}, opt); err != nil {
			b.Fatal(err)
		}
	}
}
<|endoftext|>