<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
)

// pathf is fmt.Sprintf for generating paths
// (on windows it turns / into \ after the printf).
func pathf(format string, args ...interface{}) string {
	return filepath.Clean(fmt.Sprintf(format, args...))
}

// filter returns a slice containing the elements x from list for which f(x) == true.
func filter(list []string, f func(string) bool) []string {
	var out []string
	for _, x := range list {
		if f(x) {
			out = append(out, x)
		}
	}
	return out
}

// uniq returns a sorted slice containing the unique elements of list.
func uniq(list []string) []string {
	out := make([]string, len(list))
	copy(out, list)
	sort.Strings(out)
	keep := out[:0]
	for _, x := range out {
		if len(keep) == 0 || keep[len(keep)-1] != x {
			keep = append(keep, x)
		}
	}
	return keep
}

const (
	CheckExit = 1 << iota
	ShowOutput
	Background
)

var outputLock sync.Mutex

// run runs the command line cmd in dir.
// If mode has ShowOutput set and Background unset, run passes cmd's output to
// stdout/stderr directly. Otherwise, run returns cmd's output as a string.
// If mode has CheckExit set and the command fails, run calls fatalf.
// If mode has Background set, this command is being run as a
// Background job. Only bgrun should use the Background mode,
// not other callers.
func run(dir string, mode int, cmd ...string) string {
	if vflag > 1 {
		errprintf("run: %s\n", strings.Join(cmd, " "))
	}

	xcmd := exec.Command(cmd[0], cmd[1:]...)
	xcmd.Dir = dir
	var data []byte
	var err error

	// If we want to show command output and this is not
	// a background command, assume it's the only thing
	// running, so we can just let it write directly stdout/stderr
	// as it runs without fear of mixing the output with some
	// other command's output. Not buffering lets the output
	// appear as it is printed instead of once the command exits.
	// This is most important for the invocation of 'go1.4 build -v bootstrap/...'.
	if mode&(Background|ShowOutput) == ShowOutput {
		xcmd.Stdout = os.Stdout
		xcmd.Stderr = os.Stderr
		err = xcmd.Run()
	} else {
		data, err = xcmd.CombinedOutput()
	}
	if err != nil && mode&CheckExit != 0 {
		outputLock.Lock()
		if len(data) > 0 {
			xprintf("%s\n", data)
		}
		outputLock.Unlock()
		if mode&Background != 0 {
			// Prevent fatalf from waiting on our own goroutine's
			// bghelper to exit:
			bghelpers.Done()
		}
		fatalf("FAILED: %v: %v", strings.Join(cmd, " "), err)
	}
	if mode&ShowOutput != 0 {
		outputLock.Lock()
		os.Stdout.Write(data)
		outputLock.Unlock()
	}
	if vflag > 2 {
		errprintf("run: %s DONE\n", strings.Join(cmd, " "))
	}
	return string(data)
}

var maxbg = 4 /* maximum number of jobs to run at once */

var (
	bgwork = make(chan func(), 1e5)

	bghelpers sync.WaitGroup

	dieOnce sync.Once // guards close of dying
	dying   = make(chan struct{})
)

func bginit() {
	bghelpers.Add(maxbg)
	for i := 0; i < maxbg; i++ {
		go bghelper()
	}
}

func bghelper() {
	defer bghelpers.Done()
	for {
		select {
		case <-dying:
			return
		case w := <-bgwork:
			// Dying takes precedence over doing more work.
			select {
			case <-dying:
				return
			default:
				w()
			}
		}
	}
}

// bgrun is like run but runs the command in the background.
// CheckExit|ShowOutput mode is implied (since output cannot be returned).
// bgrun adds 1 to wg immediately, and calls Done when the work completes.
func bgrun(wg *sync.WaitGroup, dir string, cmd ...string) {
	wg.Add(1)
	bgwork <- func() {
		defer wg.Done()
		run(dir, CheckExit|ShowOutput|Background, cmd...)
	}
}

// bgwait waits for pending bgruns to finish.
// bgwait must be called from only a single goroutine at a time.
func bgwait(wg *sync.WaitGroup) {
	done := make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()
	select {
	case <-done:
	case <-dying:
	}
}

// xgetwd returns the current directory.
func xgetwd() string {
	wd, err := os.Getwd()
	if err != nil {
		fatalf("%s", err)
	}
	return wd
}

// xrealwd returns the 'real' name for the given path.
// real is defined as what xgetwd returns in that directory.
func xrealwd(path string) string {
	old := xgetwd()
	if err := os.Chdir(path); err != nil {
		fatalf("chdir %s: %v", path, err)
	}
	real := xgetwd()
	if err := os.Chdir(old); err != nil {
		fatalf("chdir %s: %v", old, err)
	}
	return real
}

// isdir reports whether p names an existing directory.
func isdir(p string) bool {
	fi, err := os.Stat(p)
	return err == nil && fi.IsDir()
}

// isfile reports whether p names an existing file.
func isfile(p string) bool {
	fi, err := os.Stat(p)
	return err == nil && fi.Mode().IsRegular()
}

// mtime returns the modification time of the file p.
func mtime(p string) time.Time {
	fi, err := os.Stat(p)
	if err != nil {
		return time.Time{}
	}
	return fi.ModTime()
}

// readfile returns the content of the named file.
func readfile(file string) string {
	data, err := ioutil.ReadFile(file)
	if err != nil {
		fatalf("%v", err)
	}
	return string(data)
}

const (
	writeExec = 1 << iota
	writeSkipSame
)

// writefile writes text to the named file, creating it if needed.
// if exec is non-zero, marks the file as executable.
// If the file already exists and has the expected content,
// it is not rewritten, to avoid changing the time stamp.
func writefile(text, file string, flag int) {
	new := []byte(text)
	if flag&writeSkipSame != 0 {
		old, err := ioutil.ReadFile(file)
		if err == nil && bytes.Equal(old, new) {
			return
		}
	}
	mode := os.FileMode(0666)
	if flag&writeExec != 0 {
		mode = 0777
	}
	err := ioutil.WriteFile(file, new, mode)
	if err != nil {
		fatalf("%v", err)
	}
}

// xmkdir creates the directory p.
func xmkdir(p string) {
	err := os.Mkdir(p, 0777)
	if err != nil {
		fatalf("%v", err)
	}
}

// xmkdirall creates the directory p and its parents, as needed.
func xmkdirall(p string) {
	err := os.MkdirAll(p, 0777)
	if err != nil {
		fatalf("%v", err)
	}
}

// xremove removes the file p.
func xremove(p string) {
	if vflag > 2 {
		errprintf("rm %s\n", p)
	}
	os.Remove(p)
}

// xremoveall removes the file or directory tree rooted at p.
func xremoveall(p string) {
	if vflag > 2 {
		errprintf("rm -r %s\n", p)
	}
	os.RemoveAll(p)
}

// xreaddir replaces dst with a list of the names of the files and subdirectories in dir.
// The names are relative to dir; they are not full paths.
func xreaddir(dir string) []string {
	f, err := os.Open(dir)
	if err != nil {
		fatalf("%v", err)
	}
	defer f.Close()
	names, err := f.Readdirnames(-1)
	if err != nil {
		fatalf("reading %s: %v", dir, err)
	}
	return names
}

// xreaddir replaces dst with a list of the names of the files in dir.
// The names are relative to dir; they are not full paths.
func xreaddirfiles(dir string) []string {
	f, err := os.Open(dir)
	if err != nil {
		fatalf("%v", err)
	}
	defer f.Close()
	infos, err := f.Readdir(-1)
	if err != nil {
		fatalf("reading %s: %v", dir, err)
	}
	var names []string
	for _, fi := range infos {
		if !fi.IsDir() {
			names = append(names, fi.Name())
		}
	}
	return names
}

// xworkdir creates a new temporary directory to hold object files
// and returns the name of that directory.
func xworkdir() string {
	name, err := ioutil.TempDir(os.Getenv("GOTMPDIR"), "go-tool-dist-")
	if err != nil {
		fatalf("%v", err)
	}
	return name
}

// fatalf prints an error message to standard error and exits.
func fatalf(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "go tool dist: %s\n", fmt.Sprintf(format, args...))

	dieOnce.Do(func() { close(dying) })

	// Wait for background goroutines to finish,
	// so that exit handler that removes the work directory
	// is not fighting with active writes or open files.
	bghelpers.Wait()

	xexit(2)
}

var atexits []func()

// xexit exits the process with return code n.
func xexit(n int) {
	for i := len(atexits) - 1; i >= 0; i-- {
		atexits[i]()
	}
	os.Exit(n)
}

// xatexit schedules the exit-handler f to be run when the program exits.
func xatexit(f func()) {
	atexits = append(atexits, f)
}

// xprintf prints a message to standard output.
func xprintf(format string, args ...interface{}) {
	fmt.Printf(format, args...)
}

// errprintf prints a message to standard output.
func errprintf(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, format, args...)
}

// xsamefile reports whether f1 and f2 are the same file (or dir)
func xsamefile(f1, f2 string) bool {
	fi1, err1 := os.Stat(f1)
	fi2, err2 := os.Stat(f2)
	if err1 != nil || err2 != nil {
		return f1 == f2
	}
	return os.SameFile(fi1, fi2)
}

func xgetgoarm() string {
	if goos == "android" {
		// Assume all android devices have VFPv3.
		// These ports are also mostly cross-compiled, so it makes little
		// sense to auto-detect the setting.
		return "7"
	}
	if goos == "windows" {
		// windows/arm only works with ARMv7 executables.
		return "7"
	}
	if gohostarch != "arm" || goos != gohostos {
		// Conservative default for cross-compilation.
		return "5"
	}

	// Try to exec ourselves in a mode to detect VFP support.
	// Seeing how far it gets determines which instructions failed.
	// The test is OS-agnostic.
	out := run("", 0, os.Args[0], "-check-goarm")
	v1ok := strings.Contains(out, "VFPv1 OK.")
	v3ok := strings.Contains(out, "VFPv3 OK.")

	if v1ok && v3ok {
		return "7"
	}
	if v1ok {
		return "6"
	}
	return "5"
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

// elfIsLittleEndian detects if the ELF file is little endian.
func elfIsLittleEndian(fn string) bool {
	// read the ELF file header to determine the endianness without using the
	// debug/elf package.
	file, err := os.Open(fn)
	if err != nil {
		fatalf("failed to open file to determine endianness: %v", err)
	}
	defer file.Close()
	var hdr [16]byte
	if _, err := io.ReadFull(file, hdr[:]); err != nil {
		fatalf("failed to read ELF header to determine endianness: %v", err)
	}
	// hdr[5] is EI_DATA byte, 1 is ELFDATA2LSB and 2 is ELFDATA2MSB
	switch hdr[5] {
	default:
		fatalf("unknown ELF endianness of %s: EI_DATA = %d", fn, hdr[5])
	case 1:
		return true
	case 2:
		return false
	}
	panic("unreachable")
}

// count is a flag.Value that is like a flag.Bool and a flag.Int.
// If used as -name, it increments the count, but -name=x sets the count.
// Used for verbose flag -v.
type count int

func (c *count) String() string {
	return fmt.Sprint(int(*c))
}

func (c *count) Set(s string) error {
	switch s {
	case "true":
		*c++
	case "false":
		*c = 0
	default:
		n, err := strconv.Atoi(s)
		if err != nil {
			return fmt.Errorf("invalid count %q", s)
		}
		*c = count(n)
	}
	return nil
}

func (c *count) IsBoolFlag() bool {
	return true
}

func xflagparse(maxargs int) {
	flag.Var((*count)(&vflag), "v", "verbosity")
	flag.Parse()
	if maxargs >= 0 && flag.NArg() > maxargs {
		flag.Usage()
	}
}
<commit_msg>cmd/dist: defend self against misc/reboot test<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
)

// pathf is fmt.Sprintf for generating paths
// (on windows it turns / into \ after the printf).
func pathf(format string, args ...interface{}) string {
	return filepath.Clean(fmt.Sprintf(format, args...))
}

// filter returns a slice containing the elements x from list for which f(x) == true.
func filter(list []string, f func(string) bool) []string {
	var out []string
	for _, x := range list {
		if f(x) {
			out = append(out, x)
		}
	}
	return out
}

// uniq returns a sorted slice containing the unique elements of list.
func uniq(list []string) []string {
	out := make([]string, len(list))
	copy(out, list)
	sort.Strings(out)
	keep := out[:0]
	for _, x := range out {
		if len(keep) == 0 || keep[len(keep)-1] != x {
			keep = append(keep, x)
		}
	}
	return keep
}

const (
	CheckExit = 1 << iota
	ShowOutput
	Background
)

var outputLock sync.Mutex

// run runs the command line cmd in dir.
// If mode has ShowOutput set and Background unset, run passes cmd's output to
// stdout/stderr directly. Otherwise, run returns cmd's output as a string.
// If mode has CheckExit set and the command fails, run calls fatalf.
// If mode has Background set, this command is being run as a
// Background job. Only bgrun should use the Background mode,
// not other callers.
func run(dir string, mode int, cmd ...string) string {
	if vflag > 1 {
		errprintf("run: %s\n", strings.Join(cmd, " "))
	}

	xcmd := exec.Command(cmd[0], cmd[1:]...)
	xcmd.Dir = dir
	var data []byte
	var err error

	// If we want to show command output and this is not
	// a background command, assume it's the only thing
	// running, so we can just let it write directly stdout/stderr
	// as it runs without fear of mixing the output with some
	// other command's output. Not buffering lets the output
	// appear as it is printed instead of once the command exits.
	// This is most important for the invocation of 'go1.4 build -v bootstrap/...'.
	if mode&(Background|ShowOutput) == ShowOutput {
		xcmd.Stdout = os.Stdout
		xcmd.Stderr = os.Stderr
		err = xcmd.Run()
	} else {
		data, err = xcmd.CombinedOutput()
	}
	if err != nil && mode&CheckExit != 0 {
		outputLock.Lock()
		if len(data) > 0 {
			xprintf("%s\n", data)
		}
		outputLock.Unlock()
		if mode&Background != 0 {
			// Prevent fatalf from waiting on our own goroutine's
			// bghelper to exit:
			bghelpers.Done()
		}
		fatalf("FAILED: %v: %v", strings.Join(cmd, " "), err)
	}
	if mode&ShowOutput != 0 {
		outputLock.Lock()
		os.Stdout.Write(data)
		outputLock.Unlock()
	}
	if vflag > 2 {
		errprintf("run: %s DONE\n", strings.Join(cmd, " "))
	}
	return string(data)
}

var maxbg = 4 /* maximum number of jobs to run at once */

var (
	bgwork = make(chan func(), 1e5)

	bghelpers sync.WaitGroup

	dieOnce sync.Once // guards close of dying
	dying   = make(chan struct{})
)

func bginit() {
	bghelpers.Add(maxbg)
	for i := 0; i < maxbg; i++ {
		go bghelper()
	}
}

func bghelper() {
	defer bghelpers.Done()
	for {
		select {
		case <-dying:
			return
		case w := <-bgwork:
			// Dying takes precedence over doing more work.
			select {
			case <-dying:
				return
			default:
				w()
			}
		}
	}
}

// bgrun is like run but runs the command in the background.
// CheckExit|ShowOutput mode is implied (since output cannot be returned).
// bgrun adds 1 to wg immediately, and calls Done when the work completes.
func bgrun(wg *sync.WaitGroup, dir string, cmd ...string) {
	wg.Add(1)
	bgwork <- func() {
		defer wg.Done()
		run(dir, CheckExit|ShowOutput|Background, cmd...)
	}
}

// bgwait waits for pending bgruns to finish.
// bgwait must be called from only a single goroutine at a time.
func bgwait(wg *sync.WaitGroup) {
	done := make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()
	select {
	case <-done:
	case <-dying:
	}
}

// xgetwd returns the current directory.
func xgetwd() string {
	wd, err := os.Getwd()
	if err != nil {
		fatalf("%s", err)
	}
	return wd
}

// xrealwd returns the 'real' name for the given path.
// real is defined as what xgetwd returns in that directory.
func xrealwd(path string) string {
	old := xgetwd()
	if err := os.Chdir(path); err != nil {
		fatalf("chdir %s: %v", path, err)
	}
	real := xgetwd()
	if err := os.Chdir(old); err != nil {
		fatalf("chdir %s: %v", old, err)
	}
	return real
}

// isdir reports whether p names an existing directory.
func isdir(p string) bool {
	fi, err := os.Stat(p)
	return err == nil && fi.IsDir()
}

// isfile reports whether p names an existing file.
func isfile(p string) bool {
	fi, err := os.Stat(p)
	return err == nil && fi.Mode().IsRegular()
}

// mtime returns the modification time of the file p.
func mtime(p string) time.Time {
	fi, err := os.Stat(p)
	if err != nil {
		return time.Time{}
	}
	return fi.ModTime()
}

// readfile returns the content of the named file.
func readfile(file string) string {
	data, err := ioutil.ReadFile(file)
	if err != nil {
		fatalf("%v", err)
	}
	return string(data)
}

const (
	writeExec = 1 << iota
	writeSkipSame
)

// writefile writes text to the named file, creating it if needed.
// if exec is non-zero, marks the file as executable.
// If the file already exists and has the expected content,
// it is not rewritten, to avoid changing the time stamp.
func writefile(text, file string, flag int) {
	new := []byte(text)
	if flag&writeSkipSame != 0 {
		old, err := ioutil.ReadFile(file)
		if err == nil && bytes.Equal(old, new) {
			return
		}
	}
	mode := os.FileMode(0666)
	if flag&writeExec != 0 {
		mode = 0777
	}
	xremove(file) // in case of symlink tricks by misc/reboot test
	err := ioutil.WriteFile(file, new, mode)
	if err != nil {
		fatalf("%v", err)
	}
}

// xmkdir creates the directory p.
func xmkdir(p string) {
	err := os.Mkdir(p, 0777)
	if err != nil {
		fatalf("%v", err)
	}
}

// xmkdirall creates the directory p and its parents, as needed.
func xmkdirall(p string) {
	err := os.MkdirAll(p, 0777)
	if err != nil {
		fatalf("%v", err)
	}
}

// xremove removes the file p.
func xremove(p string) {
	if vflag > 2 {
		errprintf("rm %s\n", p)
	}
	os.Remove(p)
}

// xremoveall removes the file or directory tree rooted at p.
func xremoveall(p string) {
	if vflag > 2 {
		errprintf("rm -r %s\n", p)
	}
	os.RemoveAll(p)
}

// xreaddir replaces dst with a list of the names of the files and subdirectories in dir.
// The names are relative to dir; they are not full paths.
func xreaddir(dir string) []string {
	f, err := os.Open(dir)
	if err != nil {
		fatalf("%v", err)
	}
	defer f.Close()
	names, err := f.Readdirnames(-1)
	if err != nil {
		fatalf("reading %s: %v", dir, err)
	}
	return names
}

// xreaddir replaces dst with a list of the names of the files in dir.
// The names are relative to dir; they are not full paths.
func xreaddirfiles(dir string) []string {
	f, err := os.Open(dir)
	if err != nil {
		fatalf("%v", err)
	}
	defer f.Close()
	infos, err := f.Readdir(-1)
	if err != nil {
		fatalf("reading %s: %v", dir, err)
	}
	var names []string
	for _, fi := range infos {
		if !fi.IsDir() {
			names = append(names, fi.Name())
		}
	}
	return names
}

// xworkdir creates a new temporary directory to hold object files
// and returns the name of that directory.
func xworkdir() string {
	name, err := ioutil.TempDir(os.Getenv("GOTMPDIR"), "go-tool-dist-")
	if err != nil {
		fatalf("%v", err)
	}
	return name
}

// fatalf prints an error message to standard error and exits.
func fatalf(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "go tool dist: %s\n", fmt.Sprintf(format, args...))

	dieOnce.Do(func() { close(dying) })

	// Wait for background goroutines to finish,
	// so that exit handler that removes the work directory
	// is not fighting with active writes or open files.
	bghelpers.Wait()

	xexit(2)
}

var atexits []func()

// xexit exits the process with return code n.
func xexit(n int) {
	for i := len(atexits) - 1; i >= 0; i-- {
		atexits[i]()
	}
	os.Exit(n)
}

// xatexit schedules the exit-handler f to be run when the program exits.
func xatexit(f func()) {
	atexits = append(atexits, f)
}

// xprintf prints a message to standard output.
func xprintf(format string, args ...interface{}) {
	fmt.Printf(format, args...)
}

// errprintf prints a message to standard output.
func errprintf(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, format, args...)
}

// xsamefile reports whether f1 and f2 are the same file (or dir)
func xsamefile(f1, f2 string) bool {
	fi1, err1 := os.Stat(f1)
	fi2, err2 := os.Stat(f2)
	if err1 != nil || err2 != nil {
		return f1 == f2
	}
	return os.SameFile(fi1, fi2)
}

func xgetgoarm() string {
	if goos == "android" {
		// Assume all android devices have VFPv3.
		// These ports are also mostly cross-compiled, so it makes little
		// sense to auto-detect the setting.
		return "7"
	}
	if goos == "windows" {
		// windows/arm only works with ARMv7 executables.
		return "7"
	}
	if gohostarch != "arm" || goos != gohostos {
		// Conservative default for cross-compilation.
		return "5"
	}

	// Try to exec ourselves in a mode to detect VFP support.
	// Seeing how far it gets determines which instructions failed.
	// The test is OS-agnostic.
	out := run("", 0, os.Args[0], "-check-goarm")
	v1ok := strings.Contains(out, "VFPv1 OK.")
	v3ok := strings.Contains(out, "VFPv3 OK.")

	if v1ok && v3ok {
		return "7"
	}
	if v1ok {
		return "6"
	}
	return "5"
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

// elfIsLittleEndian detects if the ELF file is little endian.
func elfIsLittleEndian(fn string) bool {
	// read the ELF file header to determine the endianness without using the
	// debug/elf package.
	file, err := os.Open(fn)
	if err != nil {
		fatalf("failed to open file to determine endianness: %v", err)
	}
	defer file.Close()
	var hdr [16]byte
	if _, err := io.ReadFull(file, hdr[:]); err != nil {
		fatalf("failed to read ELF header to determine endianness: %v", err)
	}
	// hdr[5] is EI_DATA byte, 1 is ELFDATA2LSB and 2 is ELFDATA2MSB
	switch hdr[5] {
	default:
		fatalf("unknown ELF endianness of %s: EI_DATA = %d", fn, hdr[5])
	case 1:
		return true
	case 2:
		return false
	}
	panic("unreachable")
}

// count is a flag.Value that is like a flag.Bool and a flag.Int.
// If used as -name, it increments the count, but -name=x sets the count.
// Used for verbose flag -v.
type count int

func (c *count) String() string {
	return fmt.Sprint(int(*c))
}

func (c *count) Set(s string) error {
	switch s {
	case "true":
		*c++
	case "false":
		*c = 0
	default:
		n, err := strconv.Atoi(s)
		if err != nil {
			return fmt.Errorf("invalid count %q", s)
		}
		*c = count(n)
	}
	return nil
}

func (c *count) IsBoolFlag() bool {
	return true
}

func xflagparse(maxargs int) {
	flag.Var((*count)(&vflag), "v", "verbosity")
	flag.Parse()
	if maxargs >= 0 && flag.NArg() > maxargs {
		flag.Usage()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"strings"

	"go.pedge.io/env"

	pfscmds "github.com/pachyderm/pachyderm/src/cmd/pfs/cmds"
	ppscmds "github.com/pachyderm/pachyderm/src/cmd/pps/cmds"
	"github.com/spf13/cobra"
)

type appEnv struct {
	PachydermPfsd1Port string `env:"PACHYDERM_PFSD_1_PORT"`
	Address            string `env:"PFS_ADDRESS,default=0.0.0.0:650"`
}

func main() {
	env.Main(do, &appEnv{})
}

func do(appEnvObj interface{}) error {
	appEnv := appEnvObj.(*appEnv)
	address := appEnv.PachydermPfsd1Port
	if address == "" {
		address = appEnv.Address
	} else {
		address = strings.Replace(address, "tcp://", "", -1)
	}
	rootCmd := &cobra.Command{
		Use: "pach",
		Long: `Access the Pachyderm API.

The environment variable PFS_ADDRESS controls which PFS server the CLI connects to, the default is 0.0.0.0:650.
The environment variable PPS_ADDRESS controls what server the CLI connects to, the default is 0.0.0.0:651.`,
	}
	pfscmds, err := pfscmds.Cmds(address)
	if err != nil {
		return err
	}
	for _, cmd := range pfscmds {
		rootCmd.AddCommand(cmd)
	}
	ppscmds, err := ppscmds.Cmds(address)
	if err != nil {
		return err
	}
	for _, cmd := range ppscmds {
		rootCmd.AddCommand(cmd)
	}

	return rootCmd.Execute()
}

func getPfsdAddress() string {
	pfsdAddr := os.Getenv("PFSD_PORT_650_TCP_ADDR")
	if pfsdAddr == "" {
		return "0.0.0.0:650"
	}
	return fmt.Sprintf("%s:650", pfsdAddr)
}
<commit_msg>Add deploy cmds to pach.<commit_after>package main

import (
	"strings"

	"go.pedge.io/env"

	deploycmds "github.com/pachyderm/pachyderm/src/cmd/deploy/cmds"
	pfscmds "github.com/pachyderm/pachyderm/src/cmd/pfs/cmds"
	ppscmds "github.com/pachyderm/pachyderm/src/cmd/pps/cmds"
	"github.com/spf13/cobra"
)

type appEnv struct {
	PachydermPfsd1Port string `env:"PACHYDERM_PFSD_1_PORT"`
	PfsAddress         string `env:"PFS_ADDRESS,default=0.0.0.0:650"`
	PachydermPpsd1Port string `env:"PACHYDERM_PPSD_1_PORT"`
	PpsAddress         string `env:"PPS_ADDRESS,default=0.0.0.0:651"`
	KubernetesAddress  string `env:"KUBERNETES_ADDRESS,default=http://localhost:8080"`
	KubernetesUsername string `env:"KUBERNETES_USERNAME,default=admin"`
	KubernetesPassword string `env:"KUBERNETES_PASSWORD"`
	GCEProject         string `env:"GCE_PROJECT,required"`
	GCEZone            string `env:"GCE_ZONE,required"`
}

func main() {
	env.Main(do, &appEnv{})
}

func do(appEnvObj interface{}) error {
	appEnv := appEnvObj.(*appEnv)
	rootCmd := &cobra.Command{
		Use: "pach",
		Long: `Access the Pachyderm API.

The environment variable PFS_ADDRESS controls which PFS server the CLI connects to, the default is 0.0.0.0:650.
The environment variable PPS_ADDRESS controls what server the CLI connects to, the default is 0.0.0.0:651.`,
	}
	pfsAddress := appEnv.PachydermPfsd1Port
	if pfsAddress == "" {
		pfsAddress = appEnv.PfsAddress
	} else {
		pfsAddress = strings.Replace(pfsAddress, "tcp://", "", -1)
	}
	pfsCmds, err := pfscmds.Cmds(pfsAddress)
	if err != nil {
		return err
	}
	for _, cmd := range pfsCmds {
		rootCmd.AddCommand(cmd)
	}
	ppsAddress := appEnv.PachydermPpsd1Port
	if ppsAddress == "" {
		ppsAddress = appEnv.PpsAddress
	} else {
		ppsAddress = strings.Replace(ppsAddress, "tcp://", "", -1)
	}
	ppsCmds, err := ppscmds.Cmds(ppsAddress)
	if err != nil {
		return err
	}
	for _, cmd := range ppsCmds {
		rootCmd.AddCommand(cmd)
	}

	deployCmds, err := deploycmds.Cmds(appEnv.KubernetesAddress, appEnv.KubernetesUsername, appEnv.KubernetesAddress, appEnv.GCEProject, appEnv.GCEZone)
	if err != nil {
		return err
	}
	for _, cmd := range deployCmds {
		rootCmd.AddCommand(cmd)
	}

	return rootCmd.Execute()
}
<|endoftext|>


<commit_before>package main

import (
	tushttp "github.com/tus/tusd/src/http"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
)

const basePath = "/files/"

func main() {
	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)
	log.Printf("tusd started")

	addr := ":1080"
	if envPort := os.Getenv("TUSD_PORT"); envPort != "" {
		addr = ":" + envPort
	}

	maxSize := int64(1024 * 1024 * 1024)
	if envMaxSize := os.Getenv("TUSD_DATA_STORE_MAXSIZE"); envMaxSize != "" {
		parsed, err := strconv.ParseInt(envMaxSize, 10, 64)
		if err != nil {
			panic("bad TUSD_DATA_STORE_MAXSIZE: " + err.Error())
		}
		maxSize = parsed
	}

	dir := os.Getenv("TUSD_DATA_DIR")
	if dir == "" {
		if workingDir, err := os.Getwd(); err != nil {
			panic(err)
		} else {
			dir = filepath.Join(workingDir, "tus_data")
		}
	}

	tusConfig := tushttp.HandlerConfig{
		Dir:      dir,
		MaxSize:  maxSize,
		BasePath: basePath,
	}

	log.Printf("handler config: %+v", tusConfig)

	tusHandler, err := tushttp.NewHandler(tusConfig)
	if err != nil {
		panic(err)
	}

	http.HandleFunc(basePath, func(w http.ResponseWriter, r *http.Request) {
		// Allow CORS for almost everything. This needs to be revisted / limited to
		// routes and methods that need it.

		// Domains allowed to make requests
		w.Header().Add("Access-Control-Allow-Origin", "*")
		// Methods clients are allowed to use
		w.Header().Add("Access-Control-Allow-Methods", "HEAD,GET,PUT,POST,PATCH,DELETE")
		// Headers clients are allowed to send
		w.Header().Add("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept, Content-Disposition, Final-Length, Offset")
		// Headers clients are allowed to receive
		w.Header().Add("Access-Control-Expose-Headers", "Location, Range, Content-Disposition, Offset")

		if r.Method == "OPTIONS" {
			return
		}

		tusHandler.ServeHTTP(w, r)
	})

	go handleUploads(tusHandler)

	log.Printf("servering clients at http://localhost%s", addr)
	if err := http.ListenAndServe(addr, nil); err != nil {
		panic(err)
	}
}

func handleUploads(tus *tushttp.Handler) {
	for {
		select {
		case err := <-tus.Error:
			log.Printf("error: %s", err)
		}
	}
}
<commit_msg>fix bug: a broken http connection will cause io.Copy() stuck and file’s offset will not be updated<commit_after>package main

import (
	tushttp "github.com/tus/tusd/src/http"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"time"
)

const basePath = "/files/"

func main() {
	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)
	log.Printf("tusd started")

	addr := ":1080"
	if envPort := os.Getenv("TUSD_PORT"); envPort != "" {
		addr = ":" + envPort
	}

	maxSize := int64(1024 * 1024 * 1024)
	if envMaxSize := os.Getenv("TUSD_DATA_STORE_MAXSIZE"); envMaxSize != "" {
		parsed, err := strconv.ParseInt(envMaxSize, 10, 64)
		if err != nil {
			panic("bad TUSD_DATA_STORE_MAXSIZE: " + err.Error())
		}
		maxSize = parsed
	}

	dir := os.Getenv("TUSD_DATA_DIR")
	if dir == "" {
		if workingDir, err := os.Getwd(); err != nil {
			panic(err)
		} else {
			dir = filepath.Join(workingDir, "tus_data")
		}
	}

	tusConfig := tushttp.HandlerConfig{
		Dir:      dir,
		MaxSize:  maxSize,
		BasePath: basePath,
	}

	log.Printf("handler config: %+v", tusConfig)

	tusHandler, err := tushttp.NewHandler(tusConfig)
	if err != nil {
		panic(err)
	}

	http.HandleFunc(basePath, func(w http.ResponseWriter, r *http.Request) {
		// Allow CORS for almost everything. This needs to be revisted / limited to
		// routes and methods that need it.

		// Domains allowed to make requests
		w.Header().Add("Access-Control-Allow-Origin", "*")
		// Methods clients are allowed to use
		w.Header().Add("Access-Control-Allow-Methods", "HEAD,GET,PUT,POST,PATCH,DELETE")
		// Headers clients are allowed to send
		w.Header().Add("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept, Content-Disposition, Final-Length, Offset")
		// Headers clients are allowed to receive
		w.Header().Add("Access-Control-Expose-Headers", "Location, Range, Content-Disposition, Offset")

		if r.Method == "OPTIONS" {
			return
		}

		tusHandler.ServeHTTP(w, r)
	})

	go handleUploads(tusHandler)

	// On http package's default action, a broken http connection will cause io.Copy() stuck because it always suppose more data will coming and wait for them infinitely
	// To prevent it happen, we should set a specific timeout value on http server
	s := &http.Server{
		Addr:           addr,
		Handler:        nil,
		ReadTimeout:    8 * time.Second,
		WriteTimeout:   8 * time.Second,
		MaxHeaderBytes: 0,
	}

	log.Printf("servering clients at http://localhost%s", addr)
	if err := s.ListenAndServe(); err != nil {
		panic(err)
	}
}

func handleUploads(tus *tushttp.Handler) {
	for {
		select {
		case err := <-tus.Error:
			log.Printf("error: %s", err)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsproxy_test

import (
	"errors"
	"fmt"
	"testing"

	"github.com/googlecloudplatform/gcsfuse/gcsproxy"
	"github.com/googlecloudplatform/gcsfuse/gcsproxy/mock"
	"github.com/googlecloudplatform/gcsfuse/lease"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/gcs/mock_gcs"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/oglemock"
	. "github.com/jacobsa/ogletest"
	"golang.org/x/net/context"
)

func TestSync(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

type SyncTest struct {
	ctx context.Context

	srcObject gcs.Object
	content   mock_gcsproxy.MockMutableContent
	bucket    mock_gcs.MockBucket
}

var _ SetUpInterface = &SyncTest{}

func init() { RegisterTestSuite(&SyncTest{}) }

func (t *SyncTest) SetUp(ti *TestInfo) {
	t.ctx = ti.Ctx

	// Set up the source object.
	t.srcObject.Generation = 1234
	t.srcObject.Name = "foo"
	t.srcObject.Size = 17

	// Set up dependencies.
	t.content = mock_gcsproxy.NewMockMutableContent(
		ti.MockController,
		"content")

	t.bucket = mock_gcs.NewMockBucket(
		ti.MockController,
		"bucket")
}

func (t *SyncTest) call() (rp lease.ReadProxy, o *gcs.Object, err error) {
	rp, o, err = gcsproxy.Sync(
		t.ctx,
		&t.srcObject,
		t.content,
		t.bucket)

	return
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *SyncTest) StatFails() {
	// Stat
	ExpectCall(t.content, "Stat")(Any()).
		WillOnce(Return(gcsproxy.StatResult{}, errors.New("taco")))

	// Call
	_, _, err := t.call()

	ExpectThat(err, Error(HasSubstr("Stat")))
	ExpectThat(err, Error(HasSubstr("taco")))
}

func (t *SyncTest) StatReturnsWackyDirtyThreshold() {
	// Stat
	sr := gcsproxy.StatResult{
		DirtyThreshold: int64(t.srcObject.Size + 1),
	}

	ExpectCall(t.content, "Stat")(Any()).
		WillOnce(Return(sr, nil))

	// Call
	_, _, err := t.call()

	ExpectThat(err, Error(HasSubstr("Stat")))
	ExpectThat(err, Error(HasSubstr("DirtyThreshold")))
	ExpectThat(err, Error(HasSubstr(fmt.Sprint(t.srcObject.Size))))
	ExpectThat(err, Error(HasSubstr(fmt.Sprint(t.srcObject.Size+1))))
}

func (t *SyncTest) StatSaysNotDirty() {
	AssertTrue(false, "TODO")
}

func (t *SyncTest) CallsUpgrade() {
	AssertTrue(false, "TODO")
}

func (t *SyncTest) UpgradeFails() {
	AssertTrue(false, "TODO")
}

func (t *SyncTest) CallsBucket() {
	AssertTrue(false, "TODO")
}

func (t *SyncTest) BucketFails() {
	AssertTrue(false, "TODO")
}

func (t *SyncTest) BucketSucceeds() {
	AssertTrue(false, "TODO")
}
<commit_msg>SyncTest.StatSaysNotDirty<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsproxy_test

import (
	"errors"
	"fmt"
	"testing"

	"github.com/googlecloudplatform/gcsfuse/gcsproxy"
	"github.com/googlecloudplatform/gcsfuse/gcsproxy/mock"
	"github.com/googlecloudplatform/gcsfuse/lease"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/gcs/mock_gcs"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/oglemock"
	. "github.com/jacobsa/ogletest"
	"golang.org/x/net/context"
)

func TestSync(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

type SyncTest struct {
	ctx context.Context

	srcObject gcs.Object
	content   mock_gcsproxy.MockMutableContent
	bucket    mock_gcs.MockBucket
}

var _ SetUpInterface = &SyncTest{}

func init() { RegisterTestSuite(&SyncTest{}) }

func (t *SyncTest) SetUp(ti *TestInfo) {
	t.ctx = ti.Ctx

	// Set up the source object.
	t.srcObject.Generation = 1234
	t.srcObject.Name = "foo"
	t.srcObject.Size = 17

	// Set up dependencies.
	t.content = mock_gcsproxy.NewMockMutableContent(
		ti.MockController,
		"content")

	t.bucket = mock_gcs.NewMockBucket(
		ti.MockController,
		"bucket")
}

func (t *SyncTest) call() (rp lease.ReadProxy, o *gcs.Object, err error) {
	rp, o, err = gcsproxy.Sync(
		t.ctx,
		&t.srcObject,
		t.content,
		t.bucket)

	return
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *SyncTest) StatFails() {
	// Stat
	ExpectCall(t.content, "Stat")(Any()).
		WillOnce(Return(gcsproxy.StatResult{}, errors.New("taco")))

	// Call
	_, _, err := t.call()

	ExpectThat(err, Error(HasSubstr("Stat")))
	ExpectThat(err, Error(HasSubstr("taco")))
}

func (t *SyncTest) StatReturnsWackyDirtyThreshold() {
	// Stat
	sr := gcsproxy.StatResult{
		DirtyThreshold: int64(t.srcObject.Size + 1),
	}

	ExpectCall(t.content, "Stat")(Any()).
		WillOnce(Return(sr, nil))

	// Call
	_, _, err := t.call()

	ExpectThat(err, Error(HasSubstr("Stat")))
	ExpectThat(err, Error(HasSubstr("DirtyThreshold")))
	ExpectThat(err, Error(HasSubstr(fmt.Sprint(t.srcObject.Size))))
	ExpectThat(err, Error(HasSubstr(fmt.Sprint(t.srcObject.Size+1))))
}

func (t *SyncTest) StatSaysNotDirty() {
	// Stat
	sr := gcsproxy.StatResult{
		DirtyThreshold: int64(t.srcObject.Size),
	}

	ExpectCall(t.content, "Stat")(Any()).
		WillOnce(Return(sr, nil))

	// Call
	rp, o, err := t.call()

	AssertEq(nil, err)
	ExpectEq(nil, rp)
	ExpectEq(nil, o)
}

func (t *SyncTest) CallsUpgrade() {
	AssertTrue(false, "TODO")
}

func (t *SyncTest) UpgradeFails() {
	AssertTrue(false, "TODO")
}

func (t *SyncTest) CallsBucket() {
	AssertTrue(false, "TODO")
}

func (t *SyncTest) BucketFails() {
	AssertTrue(false, "TODO")
}

func (t *SyncTest) BucketSucceeds() {
	AssertTrue(false, "TODO")
}
<|endoftext|>


<commit_before>package Conll

// Package Conll reads ConLL format files
// For a description see http://ilk.uvt.nl/conll/#dataformat

import (
	"chukuparser/NLP/Parser/Dependency/Transition"
	NLP "chukuparser/NLP/Types"
	"chukuparser/Util"
	"encoding/csv"
	"errors"
	"fmt"
	"io"
	// "log"
	"os"
	"sort"
	"strconv"
	"strings"
)

const (
	FIELD_SEPARATOR      = '\t'
	NUM_FIELDS           = 10
	FEATURES_SEPARATOR   = "|"
	FEATURE_SEPARATOR    = "="
	FEATURE_CONCAT_DELIM = ","
)

type Features map[string]string

func (f Features) String() string {
	if f != nil || len(f) == 0 {
		return "_"
	}
	return fmt.Sprintf("%v", map[string]string(f))
}

func FormatFeatures(feat map[string]string) string {
	if feat == nil || len(feat) == 0 {
		return "_"
	}
	strs := make([]string, 0, len(feat))
	for k, v := range feat {
		strs = append(strs, fmt.Sprintf("%v%v%v", k, FEATURE_SEPARATOR, v))
	}
	sort.Strings(strs)
	return strings.Join(strs, FEATURES_SEPARATOR)
}

// A Row is a single parsed row of a conll data set
// *Commented fields are not in use
type Row struct {
	ID      int
	Form    string
	CPosTag string
	PosTag  string
	Feats   Features
	Head    int
	DepRel  string
	// Lemma string
	// PHead int
	// PDepRel string
}

func (r Row) String() string {
	fields := []string{
		fmt.Sprintf("%d", r.ID),
		r.Form,
		"_",
		r.CPosTag,
		r.PosTag,
		FormatFeatures(r.Feats),
		fmt.Sprintf("%d", r.Head),
		r.DepRel,
		"_",
		"_"}
	return strings.Join(fields, "\t")
}

// A Sentence is a map of Rows using their ids
type Sentence map[int]Row

type Sentences []Sentence

func ParseInt(value string) (int, error) {
	if value == "_" {
		return 0, nil
	}
	i, err := strconv.ParseInt(value, 10, 0)
	return int(i), err
}

func ParseString(value string) string {
	if value == "_" {
		return ""
	} else {
		return value
	}
}

func ParseFeatures(featuresStr string) (Features, error) {
	var featureMap Features
	if featuresStr == "_" {
		return featureMap, nil
	}

	featureList := strings.Split(featuresStr, FEATURES_SEPARATOR)
	if len(featureList) == 0 {
		return nil, errors.New("No features found, field should be '_'")
	}
	featureMap = make(Features, len(featureList))
	for _, featureStr := range featureList {
		featureKV := strings.Split(featureStr, FEATURE_SEPARATOR)
		if len(featureKV) != 2 {
			return nil, errors.New("Wrong number of fields for split of feature" + featureStr)
		}
		featName := featureKV[0]
		featValue := featureKV[1]
		existingFeatValue, featExist := featureMap[featName]
		if featExist {
			featureMap[featName] = existingFeatValue + FEATURE_CONCAT_DELIM + featValue
		} else {
			featureMap[featName] = featValue
		}
	}
	return featureMap, nil
}

func ParseRow(record []string) (Row, error) {
	var row Row
	id, err := ParseInt(record[0])
	if err != nil {
		return row, errors.New(fmt.Sprintf("Error parsing ID field (%s): %s", record[0], err.Error()))
	}
	row.ID = id

	form := ParseString(record[1])
	if form == "" {
		return row, errors.New("Empty FORM field")
	}
	row.Form = form

	// lemma := ParseString(record[2])
	// if lemma == "" {
	// 	return row, errors.New("Empty LEMMA field")
	// }
	// row.Lemma = lemma

	cpostag := ParseString(record[3])
	if cpostag == "" {
		return row, errors.New("Empty CPOSTAG field")
	}
	row.CPosTag = cpostag

	postag := ParseString(record[4])
	if postag == "" {
		return row, errors.New("Empty POSTAG field")
	}
	row.PosTag = postag

	head, err := ParseInt(record[6])
	if err != nil {
		return row, errors.New(fmt.Sprintf("Error parsing HEAD field (%s): %s", record[6], err.Error()))
	}
	row.Head = head

	deprel := ParseString(record[7])
	if deprel == "" {
		return row, errors.New("Empty DEPREL field")
	}
	row.DepRel = deprel

	// phead, err := ParseInt(record[8])
	// if err != nil {
	// 	return row, errors.New(fmt.Sprintf("Error parsing PHEAD field (%s): %s", record[8], err.Error()))
	// }
	// row.PHead = phead

	// pdeprel := ParseString(record[9])
	// if pdeprel == "" {
	// 	return row, errors.New("Empty PDEPREL field")
	// }
	// row.PDepRel = pdeprel

	features, err := ParseFeatures(record[5])
	if err != nil {
		return row, errors.New(fmt.Sprintf("Error parsing FEATS field (%s): %s", record[5], err.Error()))
	}
	row.Feats = features
	return row, nil
}

func Read(reader io.Reader) (Sentences, error) {
	var sentences []Sentence
	csvReader := csv.NewReader(reader)
	csvReader.Comma = FIELD_SEPARATOR
	csvReader.FieldsPerRecord = NUM_FIELDS

	records, err := csvReader.ReadAll()
	if err != nil {
		return nil, errors.New(fmt.Sprintf("Failure reading delimited file: %s", err.Error()))
	}

	var currentSent Sentence = nil
	for i, record := range records {
		// log.Println("At record", i)
		// a record with id '1' indicates a new sentence
		// since csv csvReader ignores empty lines
		if record[0] == "1" {
			// log.Println("At sentence", len(sentences))
			// store current sentence
			if currentSent != nil {
				sentences = append(sentences, currentSent)
			}
			currentSent = make(Sentence)
		}

		row, err := ParseRow(record)
		if err != nil {
			return nil, errors.New(fmt.Sprintf("Error processing record %d at statement %d: %s", i, len(sentences), err.Error()))
		}
		currentSent[row.ID] = row
	}
	sentences = append(sentences, currentSent)
	return sentences, nil
}

func ReadFile(filename string) ([]Sentence, error) {
	file, err := os.Open(filename)
	defer file.Close()
	if err != nil {
		return nil, err
	}

	return Read(file)
}

func Write(writer io.Writer, sents []Sentence) {
	for _, sent := range sents {
		for i := 1; i <= len(sent); i++ {
			row := sent[i]
			writer.Write(append([]byte(row.String()), '\n'))
		}
		writer.Write([]byte{'\n'})
	}
}

func WriteFile(filename string, sents []Sentence) error {
	file, err := os.Create(filename)
	defer file.Close()
	if err != nil {
		return err
	}
	Write(file, sents)
	return nil
}

func Graph2Conll(graph NLP.LabeledDependencyGraph) Sentence {
	sent := make(Sentence, graph.NumberOfNodes()-1)
	arcIndex := make(map[int]NLP.LabeledDepArc, graph.NumberOfNodes())
	var (
		posTag string
		node   NLP.DepNode
		arc    NLP.LabeledDepArc
		headID int
		depRel string
	)
	for _, arcID := range graph.GetEdges() {
		arc = graph.GetLabeledArc(arcID)
		if arc == nil {
			panic("Can't find arc")
		}
		arcIndex[arc.GetModifier()] = arc
	}
	for _, nodeID := range graph.GetVertices() {
		node = graph.GetNode(nodeID)
		posTag = ""

		taggedToken, ok := node.(*Transition.TaggedDepNode)
		if ok {
			posTag = taggedToken.RawPOS
		}

		if node == nil {
			panic("Can't find node")
		}
		arc, exists := arcIndex[node.ID()]
		if exists {
			headID = arc.GetHead()
			if headID == -1 {
				headID = 0
			}
			depRel = string(arc.GetRelation())
		} else {
			headID = 0
			depRel = "None"
		}
		row := Row{
			ID:      node.ID(),
			Form:    node.String(),
			CPosTag: posTag,
			PosTag:  posTag,
			Feats:   nil,
			Head:    headID,
			DepRel:  depRel,
		}
		sent[row.ID] = row
	}
	return sent
}

func Graph2ConllCorpus(corpus []NLP.LabeledDependencyGraph) []Sentence {
	sentCorpus := make([]Sentence, len(corpus))
	for i, graph := range corpus {
		sentCorpus[i] = Graph2Conll(graph)
	}
	return sentCorpus
}

func Conll2Graph(sent Sentence, eWord, ePOS, eWPOS, eRel *Util.EnumSet) NLP.LabeledDependencyGraph {
	var (
		arc   *Transition.BasicDepArc
		node  *Transition.TaggedDepNode
		index int
	)
	nodes := make([]NLP.DepNode, 0, len(sent)+2)
	// log.Println("\tNum Nodes:", len(nodes))
	arcs := make([]*Transition.BasicDepArc, 0, len(sent))
	// node.Token, _ = eWord.Add(NLP.ROOT_TOKEN)
	// node.POS, _ = ePOS.Add(NLP.ROOT_TOKEN)
	// node.TokenPOS, _ = eWPOS.Add([2]string{NLP.ROOT_TOKEN, NLP.ROOT_TOKEN})
	// nodes = append(nodes, NLP.DepNode(node)) // add root node

	for i, row := range sent {
		node = &Transition.TaggedDepNode{
			Id:       i - 1,
			RawToken: row.Form,
			RawPOS:   row.CPosTag,
		}
		node.Token, _ = eWord.Add(row.Form)
		node.POS, _ = ePOS.Add(row.CPosTag)
		node.TokenPOS, _ = eWPOS.Add([2]string{row.Form, row.CPosTag})
		index, _ = eRel.IndexOf(NLP.DepRel(row.DepRel))
		arc = &Transition.BasicDepArc{row.Head - 1, index, i - 1, NLP.DepRel(row.DepRel)}
		nodes = append(nodes, NLP.DepNode(node))
		arcs = append(arcs, arc)
	}
	return NLP.LabeledDependencyGraph(&Transition.BasicDepGraph{nodes, arcs})
}

func Conll2GraphCorpus(corpus []Sentence, eWord, ePOS, eWPOS, eRel *Util.EnumSet) []NLP.LabeledDependencyGraph {
	graphCorpus := make([]NLP.LabeledDependencyGraph, len(corpus))
	for i, sent := range corpus {
		// log.Println("Converting sentence", i)
		graphCorpus[i] = Conll2Graph(sent, eWord, ePOS, eWPOS, eRel)
	}
	return graphCorpus
}

func MorphGraph2Conll(graph NLP.MorphDependencyGraph) Sentence {
	sent := make(Sentence, graph.NumberOfNodes())
	arcIndex := make(map[int]NLP.LabeledDepArc, graph.NumberOfNodes())
	var (
		node   *NLP.EMorpheme
		arc    NLP.LabeledDepArc
		headID int
		depRel string
	)
	for _, arcID := range graph.GetEdges() {
		arc = graph.GetLabeledArc(arcID)
		if arc == nil {
			panic("Can't find arc")
		}
		arcIndex[arc.GetModifier()] = arc
	}
	for i, nodeID := range graph.GetVertices() {
		if nodeID == 0 {
			continue
		}
		node = graph.GetMorpheme(nodeID)

		if node == nil {
			panic("Can't find node")
		}

		arc, exists := arcIndex[i]
		if exists {
			headID = arc.GetHead()
			depRel = string(arc.GetRelation())
		} else {
			headID = 0
			depRel = "None"
		}
		row := Row{
			ID:      i,
			Form:    node.Form,
			CPosTag: node.CPOS,
			PosTag:  node.POS,
			Feats:   node.Features,
			Head:    headID,
			DepRel:  depRel,
		}
		sent[row.ID] = row
	}
	return sent
}

func MorphGraph2ConllCorpus(corpus []NLP.MorphDependencyGraph) []Sentence {
	sentCorpus := make([]Sentence, len(corpus))
	for i, graph := range corpus {
		sentCorpus[i] = MorphGraph2Conll(graph)
	}
	return sentCorpus
}
<commit_msg>Yet another fix for conll output<commit_after>package Conll

// Package Conll reads ConLL format files
// For a description see http://ilk.uvt.nl/conll/#dataformat

import (
	"chukuparser/NLP/Parser/Dependency/Transition"
	NLP "chukuparser/NLP/Types"
	"chukuparser/Util"
	"encoding/csv"
	"errors"
	"fmt"
	"io"
	// "log"
	"os"
	"sort"
	"strconv"
	"strings"
)

const (
	FIELD_SEPARATOR      = '\t'
	NUM_FIELDS           = 10
	FEATURES_SEPARATOR   = "|"
	FEATURE_SEPARATOR    = "="
	FEATURE_CONCAT_DELIM = ","
)

type Features map[string]string

func (f Features) String() string {
	if f != nil || len(f) == 0 {
		return "_"
	}
	return fmt.Sprintf("%v", map[string]string(f))
}

func FormatFeatures(feat map[string]string) string {
	if feat == nil || len(feat) == 0 {
		return "_"
	}
	strs := make([]string, 0, len(feat))
	for k, v := range feat {
		strs = append(strs, fmt.Sprintf("%v%v%v", k, FEATURE_SEPARATOR, v))
	}
	sort.Strings(strs)
	return strings.Join(strs, FEATURES_SEPARATOR)
}

// A Row is a single parsed row of a conll data set
// *Commented fields are not in use
type Row struct {
	ID      int
	Form    string
	CPosTag string
	PosTag  string
	Feats   Features
	Head    int
	DepRel  string
	// Lemma string
	// PHead int
	// PDepRel string
}

func (r Row) String() string {
	fields := []string{
		fmt.Sprintf("%d", r.ID),
		r.Form,
		"_",
		r.CPosTag,
		r.PosTag,
		FormatFeatures(r.Feats),
		fmt.Sprintf("%d", r.Head),
		r.DepRel,
		"_",
		"_"}
	return strings.Join(fields, "\t")
}

// A Sentence is a map of Rows using their ids
type Sentence map[int]Row

type Sentences []Sentence

func ParseInt(value string) (int, error) {
	if value == "_" {
		return 0, nil
	}
	i, err := strconv.ParseInt(value, 10, 0)
	return int(i), err
}

func ParseString(value string) string {
	if value == "_" {
		return ""
	} else {
		return value
	}
}

func ParseFeatures(featuresStr string) (Features, error) {
	var featureMap Features
	if featuresStr == "_" {
		return featureMap, nil
	}

	featureList := strings.Split(featuresStr, FEATURES_SEPARATOR)
	if len(featureList) == 0 {
		return nil, errors.New("No features found, field should be '_'")
	}
	featureMap = make(Features, len(featureList))
	for _, featureStr := range featureList {
		featureKV := strings.Split(featureStr, FEATURE_SEPARATOR)
		if len(featureKV) != 2 {
			return nil, errors.New("Wrong number of fields for split of feature" + featureStr)
		}
		featName := featureKV[0]
		featValue := featureKV[1]
		existingFeatValue, featExist := featureMap[featName]
		if featExist {
			featureMap[featName] = existingFeatValue + FEATURE_CONCAT_DELIM + featValue
		} else {
			featureMap[featName] = featValue
		}
	}
	return featureMap, nil
}

func ParseRow(record []string) (Row, error) {
	var row Row
	id, err := ParseInt(record[0])
	if err != nil {
		return row, errors.New(fmt.Sprintf("Error parsing ID field (%s): %s", record[0], err.Error()))
	}
	row.ID = id

	form := ParseString(record[1])
	if form == "" {
		return row, errors.New("Empty FORM field")
	}
	row.Form = form

	// lemma := ParseString(record[2])
	// if lemma == "" {
	// 	return row, errors.New("Empty LEMMA field")
	// }
	// row.Lemma = lemma

	cpostag := ParseString(record[3])
	if cpostag == "" {
		return row, errors.New("Empty CPOSTAG field")
	}
	row.CPosTag = cpostag

	postag := ParseString(record[4])
	if postag == "" {
		return row, errors.New("Empty POSTAG field")
	}
	row.PosTag = postag

	head, err := ParseInt(record[6])
	if err != nil {
		return row, errors.New(fmt.Sprintf("Error parsing HEAD field (%s): %s", record[6], err.Error()))
	}
	row.Head = head

	deprel := ParseString(record[7])
	if deprel == "" {
		return row, errors.New("Empty DEPREL field")
	}
	row.DepRel = deprel

	// phead, err := ParseInt(record[8])
	// if err != nil {
	// 	return row, errors.New(fmt.Sprintf("Error parsing PHEAD field (%s): %s", record[8], err.Error()))
	// }
	// row.PHead = phead

	// pdeprel := ParseString(record[9])
	// if pdeprel == "" {
	// 	return row, errors.New("Empty PDEPREL field")
	// }
	// row.PDepRel = pdeprel

	features, err := ParseFeatures(record[5])
	if err != nil {
		return row, errors.New(fmt.Sprintf("Error parsing FEATS field (%s): %s", record[5], err.Error()))
	}
	row.Feats = features
	return row, nil
}

func Read(reader io.Reader) (Sentences, error) {
	var sentences []Sentence
	csvReader := csv.NewReader(reader)
	csvReader.Comma = FIELD_SEPARATOR
	csvReader.FieldsPerRecord = NUM_FIELDS

	records, err := csvReader.ReadAll()
	if err != nil {
		return nil, errors.New(fmt.Sprintf("Failure reading delimited file: %s", err.Error()))
	}

	var currentSent Sentence = nil
	for i, record := range records {
		// log.Println("At record", i)
		// a record with id '1' indicates a new sentence
		// since csv csvReader ignores empty lines
		if record[0] == "1" {
			// log.Println("At sentence", len(sentences))
			// store current sentence
			if currentSent != nil {
				sentences = append(sentences, currentSent)
			}
			currentSent = make(Sentence)
		}

		row, err := ParseRow(record)
		if err != nil {
			return nil, errors.New(fmt.Sprintf("Error processing record %d at statement %d: %s", i, len(sentences), err.Error()))
		}
		currentSent[row.ID] = row
	}
	sentences = append(sentences, currentSent)
	return sentences, nil
}

func ReadFile(filename string) ([]Sentence, error) {
	file, err := os.Open(filename)
	defer file.Close()
	if err != nil {
		return nil, err
	}

	return Read(file)
}

func Write(writer io.Writer, sents []Sentence) {
	for _, sent := range sents {
		for i := 0; i <= len(sent)-1; i++ {
			row := sent[i]
			writer.Write(append([]byte(row.String()), '\n'))
		}
		writer.Write([]byte{'\n'})
	}
}

func WriteFile(filename string, sents []Sentence) error {
	file, err := os.Create(filename)
	defer file.Close()
	if err != nil {
		return err
	}
	Write(file, sents)
	return nil
}

func Graph2Conll(graph NLP.LabeledDependencyGraph) Sentence {
	sent := make(Sentence, graph.NumberOfNodes()-1)
	arcIndex := make(map[int]NLP.LabeledDepArc, graph.NumberOfNodes())
	var (
		posTag string
		node   NLP.DepNode
		arc    NLP.LabeledDepArc
		headID int
		depRel string
	)
	for _, arcID := range graph.GetEdges() {
		arc = graph.GetLabeledArc(arcID)
		if arc == nil {
			panic("Can't find arc")
		}
		arcIndex[arc.GetModifier()] = arc
	}
	for _, nodeID := range graph.GetVertices() {
		node = graph.GetNode(nodeID)
		posTag = ""

		taggedToken, ok := node.(*Transition.TaggedDepNode)
		if ok {
			posTag = taggedToken.RawPOS
		}

		if node == nil {
			panic("Can't find node")
		}
		arc, exists := arcIndex[node.ID()]
		if exists {
			headID = arc.GetHead()
			if headID == -1 {
				headID = 0
			}
			depRel = string(arc.GetRelation())
		} else {
			headID = 0
			depRel = "None"
		}
		row := Row{
			ID:      node.ID(),
			Form:    node.String(),
			CPosTag: posTag,
			PosTag:  posTag,
			Feats:   nil,
			Head:    headID,
			DepRel:  depRel,
		}
		sent[row.ID] = row
	}
	return sent
}

func Graph2ConllCorpus(corpus []NLP.LabeledDependencyGraph) []Sentence {
	sentCorpus := make([]Sentence, len(corpus))
	for i, graph := range corpus {
		sentCorpus[i] = Graph2Conll(graph)
	}
	return sentCorpus
}

func Conll2Graph(sent Sentence, eWord, ePOS, eWPOS, eRel *Util.EnumSet) NLP.LabeledDependencyGraph {
	var (
		arc   *Transition.BasicDepArc
		node  *Transition.TaggedDepNode
		index int
	)
	nodes := make([]NLP.DepNode, 0, len(sent)+2)
	// log.Println("\tNum Nodes:", len(nodes))
	arcs := make([]*Transition.BasicDepArc, 0, len(sent))
	// node.Token, _ = eWord.Add(NLP.ROOT_TOKEN)
	// node.POS, _ = ePOS.Add(NLP.ROOT_TOKEN)
	// node.TokenPOS, _ = eWPOS.Add([2]string{NLP.ROOT_TOKEN, NLP.ROOT_TOKEN})
	// nodes = append(nodes, NLP.DepNode(node)) // add root node

	for i, row := range sent {
		node = &Transition.TaggedDepNode{
			Id:       i - 1,
			RawToken: row.Form,
			RawPOS:   row.CPosTag,
		}
		node.Token, _ = eWord.Add(row.Form)
		node.POS, _ = ePOS.Add(row.CPosTag)
		node.TokenPOS, _ = eWPOS.Add([2]string{row.Form, row.CPosTag})
		index, _ = eRel.IndexOf(NLP.DepRel(row.DepRel))
		arc = &Transition.BasicDepArc{row.Head - 1, index, i - 1, NLP.DepRel(row.DepRel)}
		nodes = append(nodes, NLP.DepNode(node))
		arcs = append(arcs, arc)
	}
	return NLP.LabeledDependencyGraph(&Transition.BasicDepGraph{nodes, arcs})
}

func Conll2GraphCorpus(corpus []Sentence, eWord, ePOS, eWPOS, eRel *Util.EnumSet) []NLP.LabeledDependencyGraph {
	graphCorpus := make([]NLP.LabeledDependencyGraph, len(corpus))
	for i, sent := range corpus {
		// log.Println("Converting sentence", i)
		graphCorpus[i] = Conll2Graph(sent, eWord, ePOS, eWPOS, eRel)
	}
	return graphCorpus
}

func MorphGraph2Conll(graph NLP.MorphDependencyGraph) Sentence {
	sent := make(Sentence, graph.NumberOfNodes())
	arcIndex := make(map[int]NLP.LabeledDepArc, graph.NumberOfNodes())
	var (
		node   *NLP.EMorpheme
		arc    NLP.LabeledDepArc
		headID int
		depRel string
	)
	for _, arcID := range graph.GetEdges() {
		arc = graph.GetLabeledArc(arcID)
		if arc == nil {
			panic("Can't find arc")
		}
		arcIndex[arc.GetModifier()] = arc
	}
	for i, nodeID := range graph.GetVertices() {
		if nodeID == 0 {
			continue
		}
		node = graph.GetMorpheme(nodeID)

		if node == nil {
			panic("Can't find node")
		}

		arc, exists := arcIndex[i]
		if exists {
			headID = arc.GetHead()
			depRel = string(arc.GetRelation())
		} else {
			headID = 0
			depRel = "None"
		}
		row := Row{
			ID:      i,
			Form:    node.Form,
			CPosTag: node.CPOS,
			PosTag:  node.POS,
			Feats:   node.Features,
			Head:    headID,
			DepRel:  depRel,
		}
		sent[row.ID] = row
	}
	return sent
}

func MorphGraph2ConllCorpus(corpus []NLP.MorphDependencyGraph) []Sentence {
	sentCorpus := make([]Sentence, len(corpus))
	for i, graph := range corpus {
		sentCorpus[i] = MorphGraph2Conll(graph)
	}
	return sentCorpus
}
<|endoftext|>


<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package logger

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"

	logging "github.com/keybase/go-logging"
	"golang.org/x/net/context"

	keybase1 "github.com/keybase/client/go/protocol"
)

const permDir os.FileMode = 0700

// Map from module name to whether SetLevel() has been called for that
// module.
var initLoggingSetLevelCalled = map[string]struct{}{}

// Protects access to initLoggingSetLevelCalled and the actual
// SetLevel call.
var initLoggingSetLevelMutex sync.Mutex

// CtxStandardLoggerKey is a type defining context keys used by the
// Standard logger.
type CtxStandardLoggerKey int

const (
	// CtxLogTagsKey defines a context key that can associate with a map of
	// context keys (key -> descriptive-name), the mapped values of which should
	// be logged by a Standard logger if one of those keys is seen in a context
	// during a log call.
	CtxLogTagsKey CtxStandardLoggerKey = iota
)

type CtxLogTags map[interface{}]string

// NewContext returns a new Context that carries adds the given log
// tag mappings (context key -> display string).
func NewContextWithLogTags(
	ctx context.Context, logTagsToAdd CtxLogTags) context.Context {
	currTags, ok := LogTagsFromContext(ctx)
	if !ok {
		currTags = make(CtxLogTags)
	}
	for key, tag := range logTagsToAdd {
		currTags[key] = tag
	}
	return context.WithValue(ctx, CtxLogTagsKey, currTags)
}

// LogTagsFromContext returns the log tags being passed along with the
// given context.
func LogTagsFromContext(ctx context.Context) (CtxLogTags, bool) {
	logTags, ok := ctx.Value(CtxLogTagsKey).(CtxLogTags)
	return logTags, ok
}

type ExternalLogger interface {
	Log(level keybase1.LogLevel, format string, args []interface{})
}

type entry struct {
	level  keybase1.LogLevel
	format string
	args   []interface{}
}

type Standard struct {
	internal       *logging.Logger
	filename       string
	configureMutex sync.Mutex
	module         string

	externalHandler ExternalHandler
}

// Verify Standard fully implements the Logger interface.
var _ Logger = (*Standard)(nil)

// New creates a new Standard logger for module.
func New(module string) *Standard {
	return NewWithCallDepth(module, 0)
}

// NewWithCallDepth creates a new Standard logger for module, and when
// printing file names and line numbers, it goes extraCallDepth up the
// stack from where logger was invoked.
func NewWithCallDepth(module string, extraCallDepth int) *Standard {
	log := logging.MustGetLogger(module)
	log.ExtraCalldepth = 1 + extraCallDepth

	ret := &Standard{
		internal: log,
		module:   module,
	}
	ret.setLogLevelInfo()
	return ret
}

func (log *Standard) setLogLevelInfo() {
	initLoggingSetLevelMutex.Lock()
	defer initLoggingSetLevelMutex.Unlock()

	if _, found := initLoggingSetLevelCalled[log.module]; !found {
		logging.SetLevel(logging.INFO, log.module)
		initLoggingSetLevelCalled[log.module] = struct{}{}
	}
}

func prepareString(
	ctx context.Context, fmts string) string {
	if ctx == nil {
		return fmts
	}
	logTags, ok := LogTagsFromContext(ctx)
	if !ok || len(logTags) == 0 {
		return fmts
	}
	var tags []string
	for key, tag := range logTags {
		if v := ctx.Value(key); v != nil {
			tags = append(tags, fmt.Sprintf("%s=%s", tag, v))
		}
	}
	return fmts + " [tags:" + strings.Join(tags, ",") + "]"
}

func (log *Standard) Debug(fmt string, arg ...interface{}) {
	log.internal.Debugf(fmt, arg...)
	if log.externalHandler != nil {
		log.externalHandler.Log(keybase1.LogLevel_DEBUG, fmt, arg)
	}
}

func (log *Standard) CDebugf(ctx context.Context, fmt string,
	arg ...interface{}) {
	if log.internal.IsEnabledFor(logging.DEBUG) {
		log.Debug(prepareString(ctx, fmt), arg...)
	}
}

func (log *Standard) Info(fmt string, arg ...interface{}) {
	log.internal.Infof(fmt, arg...)
	if log.externalHandler != nil {
		log.externalHandler.Log(keybase1.LogLevel_INFO, fmt, arg)
	}
}

func (log *Standard) CInfof(ctx context.Context, fmt string,
	arg ...interface{}) {
	if log.internal.IsEnabledFor(logging.INFO) {
		log.Info(prepareString(ctx, fmt), arg...)
	}
}

func (log *Standard) Notice(fmt string, arg ...interface{}) {
	log.internal.Noticef(fmt, arg...)
	if log.externalHandler != nil {
		log.externalHandler.Log(keybase1.LogLevel_NOTICE, fmt, arg)
	}
}

func (log *Standard) CNoticef(ctx context.Context, fmt string,
	arg ...interface{}) {
	if log.internal.IsEnabledFor(logging.NOTICE) {
		log.Notice(prepareString(ctx, fmt), arg...)
	}
}

func (log *Standard) Warning(fmt string, arg ...interface{}) {
	log.internal.Warningf(fmt, arg...)
	if log.externalHandler != nil {
		log.externalHandler.Log(keybase1.LogLevel_WARN, fmt, arg)
	}
}

func (log *Standard) CWarningf(ctx context.Context, fmt string,
	arg ...interface{}) {
	if log.internal.IsEnabledFor(logging.WARNING) {
		log.Warning(prepareString(ctx, fmt), arg...)
	}
}

func (log *Standard) Error(fmt string, arg ...interface{}) {
	log.internal.Errorf(fmt, arg...)
	if log.externalHandler != nil {
		log.externalHandler.Log(keybase1.LogLevel_ERROR, fmt, arg)
	}
}

func (log *Standard) Errorf(fmt string, arg ...interface{}) {
	log.Error(fmt, arg...)
}

func (log *Standard) CErrorf(ctx context.Context, fmt string,
	arg ...interface{}) {
	if log.internal.IsEnabledFor(logging.ERROR) {
		log.Error(prepareString(ctx, fmt), arg...)
	}
}

func (log *Standard) Critical(fmt string, arg ...interface{}) {
	log.internal.Criticalf(fmt, arg...)
	if log.externalHandler != nil {
		log.externalHandler.Log(keybase1.LogLevel_CRITICAL, fmt, arg)
	}
}

func (log *Standard) CCriticalf(ctx context.Context, fmt string,
	arg ...interface{}) {
	if log.internal.IsEnabledFor(logging.CRITICAL) {
		log.Critical(prepareString(ctx, fmt), arg...)
	}
}

func (log *Standard) Fatalf(fmt string, arg ...interface{}) {
	log.internal.Fatalf(fmt, arg...)
	if log.externalHandler != nil {
		log.externalHandler.Log(keybase1.LogLevel_FATAL, fmt, arg)
	}
}

func (log *Standard) CFatalf(ctx context.Context, fmt string,
	arg ...interface{}) {
	log.Fatalf(prepareString(ctx, fmt), arg...)
}

func (log *Standard) Profile(fmts string, arg ...interface{}) {
	log.Debug(fmts, arg...)
}

// Configure sets the style of the log file, whether debugging (verbose)
// is enabled and a filename. If a filename is provided here it will
// be used for logging straight away (this is a new feature).
// SetLogFileConfig provides a way to set the log file with more control on rotation.
func (log *Standard) Configure(style string, debug bool, filename string) {
	log.configureMutex.Lock()
	defer log.configureMutex.Unlock()

	log.filename = filename

	var logfmt string

	globalLock.Lock()
	isTerm := stderrIsTerminal
	globalLock.Unlock()

	// TODO: how should setting the log file after a Configure be handled?
	if isTerm {
		if debug {
			logfmt = fancyFormat
		} else {
			logfmt = defaultFormat
		}
	} else {
		logfmt = fileFormat
	}

	// Override the format above if an explicit style was specified.
	switch style {
	case "default":
		logfmt = defaultFormat // Default
	case "plain":
		logfmt = plainFormat // Plain
	case "file":
		logfmt = fileFormat // Good for logging to files
	case "fancy":
		logfmt = fancyFormat // Fancy, good for terminals with color
	}

	if debug {
		logging.SetLevel(logging.DEBUG, log.module)
	}

	logging.SetFormatter(logging.MustStringFormatter(logfmt))

}

func OpenLogFile(filename string) (name string, file *os.File, err error) {
	name = filename
	if err = MakeParentDirs(name); err != nil {
		return
	}
	file, err = os.OpenFile(name, (os.O_APPEND | os.O_WRONLY | os.O_CREATE), 0600)
	if err != nil {
		return
	}
	return
}

func FileExists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}

func MakeParentDirs(filename string) error {
	dir, _ := filepath.Split(filename)
	// If passed a plain file name as a path
	if dir == "" {
		return nil
	}
	exists, err := FileExists(dir)
	if err != nil {
		return err
	}

	if !exists {
		err = os.MkdirAll(dir, permDir)
		if err != nil {
			return err
		}
	}
	return nil
}

func PickFirstError(errors ...error) error {
	for _, e := range errors {
		if e != nil {
			return e
		}
	}
	return nil
}

func (log *Standard) CloneWithAddedDepth(depth int) Logger {
	clone := *log
	cloneInternal := *log.internal
	cloneInternal.ExtraCalldepth = log.internal.ExtraCalldepth + depth
	clone.internal = &cloneInternal
	return &clone
}

func (log *Standard) SetExternalHandler(handler ExternalHandler) {
	log.externalHandler = handler
}

type UnforwardedLogger Standard

func (log *Standard) GetUnforwardedLogger() *UnforwardedLogger {
	return (*UnforwardedLogger)(log)
}

func (log *UnforwardedLogger) Debug(s string, args ...interface{})  { log.internal.Debugf(s, args...) }
func (log *UnforwardedLogger) Error(s string, args ...interface{})  { log.internal.Errorf(s, args...) }
func (log *UnforwardedLogger) Errorf(s string, args ...interface{}) { log.internal.Errorf(s, args...) }
func (log *UnforwardedLogger) Warning(s string, args ...interface{}) {
	log.internal.Warningf(s, args...)
}
func (log *UnforwardedLogger) Info(s string, args ...interface{})    { log.internal.Infof(s, args...) }
func (log *UnforwardedLogger) Profile(s string, args ...interface{}) { log.internal.Debugf(s, args...) }
<commit_msg>logger: copy log tags on update to avoid a race (#3412)<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package logger

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"

	logging "github.com/keybase/go-logging"
	"golang.org/x/net/context"

	keybase1 "github.com/keybase/client/go/protocol"
)

const permDir os.FileMode = 0700

// Map from module name to whether SetLevel() has been called for that
// module.
var initLoggingSetLevelCalled = map[string]struct{}{}

// Protects access to initLoggingSetLevelCalled and the actual
// SetLevel call.
var initLoggingSetLevelMutex sync.Mutex

// CtxStandardLoggerKey is a type defining context keys used by the
// Standard logger.
type CtxStandardLoggerKey int

const (
	// CtxLogTagsKey defines a context key that can associate with a map of
	// context keys (key -> descriptive-name), the mapped values of which should
	// be logged by a Standard logger if one of those keys is seen in a context
	// during a log call.
	CtxLogTagsKey CtxStandardLoggerKey = iota
)

type CtxLogTags map[interface{}]string

// NewContext returns a new Context that carries adds the given log
// tag mappings (context key -> display string).
func NewContextWithLogTags(
	ctx context.Context, logTagsToAdd CtxLogTags) context.Context {
	currTags, _ := LogTagsFromContext(ctx)
	newTags := make(CtxLogTags)
	// Make a copy to avoid races
	for key, tag := range currTags {
		newTags[key] = tag
	}
	for key, tag := range logTagsToAdd {
		newTags[key] = tag
	}
	return context.WithValue(ctx, CtxLogTagsKey, newTags)
}

// LogTagsFromContext returns the log tags being passed along with the
// given context.
func LogTagsFromContext(ctx context.Context) (CtxLogTags, bool) {
	logTags, ok := ctx.Value(CtxLogTagsKey).(CtxLogTags)
	return logTags, ok
}

type ExternalLogger interface {
	Log(level keybase1.LogLevel, format string, args []interface{})
}

type entry struct {
	level  keybase1.LogLevel
	format string
	args   []interface{}
}

type Standard struct {
	internal       *logging.Logger
	filename       string
	configureMutex sync.Mutex
	module         string

	externalHandler ExternalHandler
}

// Verify Standard fully implements the Logger interface.
var _ Logger = (*Standard)(nil)

// New creates a new Standard logger for module.
func New(module string) *Standard {
	return NewWithCallDepth(module, 0)
}

// NewWithCallDepth creates a new Standard logger for module, and when
// printing file names and line numbers, it goes extraCallDepth up the
// stack from where logger was invoked.
func NewWithCallDepth(module string, extraCallDepth int) *Standard {
	log := logging.MustGetLogger(module)
	log.ExtraCalldepth = 1 + extraCallDepth

	ret := &Standard{
		internal: log,
		module:   module,
	}
	ret.setLogLevelInfo()
	return ret
}

func (log *Standard) setLogLevelInfo() {
	initLoggingSetLevelMutex.Lock()
	defer initLoggingSetLevelMutex.Unlock()

	if _, found := initLoggingSetLevelCalled[log.module]; !found {
		logging.SetLevel(logging.INFO, log.module)
		initLoggingSetLevelCalled[log.module] = struct{}{}
	}
}

func prepareString(
	ctx context.Context, fmts string) string {
	if ctx == nil {
		return fmts
	}
	logTags, ok := LogTagsFromContext(ctx)
	if !ok || len(logTags) == 0 {
		return fmts
	}
	var tags []string
	for key, tag := range logTags {
		if v := ctx.Value(key); v != nil {
			tags = append(tags, fmt.Sprintf("%s=%s", tag, v))
		}
	}
	return fmts + " [tags:" + strings.Join(tags, ",") + "]"
}

func (log *Standard) Debug(fmt string, arg ...interface{}) {
	log.internal.Debugf(fmt, arg...)
	if log.externalHandler != nil {
		log.externalHandler.Log(keybase1.LogLevel_DEBUG, fmt, arg)
	}
}

func (log *Standard) CDebugf(ctx context.Context, fmt string,
	arg ...interface{}) {
	if log.internal.IsEnabledFor(logging.DEBUG) {
		log.Debug(prepareString(ctx, fmt), arg...)
	}
}

func (log *Standard) Info(fmt string, arg ...interface{}) {
	log.internal.Infof(fmt, arg...)
	if log.externalHandler != nil {
		log.externalHandler.Log(keybase1.LogLevel_INFO, fmt, arg)
	}
}

func (log *Standard) CInfof(ctx context.Context, fmt string,
	arg ...interface{}) {
	if log.internal.IsEnabledFor(logging.INFO) {
		log.Info(prepareString(ctx, fmt), arg...)
	}
}

func (log *Standard) Notice(fmt string, arg ...interface{}) {
	log.internal.Noticef(fmt, arg...)
	if log.externalHandler != nil {
		log.externalHandler.Log(keybase1.LogLevel_NOTICE, fmt, arg)
	}
}

func (log *Standard) CNoticef(ctx context.Context, fmt string,
	arg ...interface{}) {
	if log.internal.IsEnabledFor(logging.NOTICE) {
		log.Notice(prepareString(ctx, fmt), arg...)
	}
}

func (log *Standard) Warning(fmt string, arg ...interface{}) {
	log.internal.Warningf(fmt, arg...)
	if log.externalHandler != nil {
		log.externalHandler.Log(keybase1.LogLevel_WARN, fmt, arg)
	}
}

func (log *Standard) CWarningf(ctx context.Context, fmt string,
	arg ...interface{}) {
	if log.internal.IsEnabledFor(logging.WARNING) {
		log.Warning(prepareString(ctx, fmt), arg...)
	}
}

func (log *Standard) Error(fmt string, arg ...interface{}) {
	log.internal.Errorf(fmt, arg...)
	if log.externalHandler != nil {
		log.externalHandler.Log(keybase1.LogLevel_ERROR, fmt, arg)
	}
}

func (log *Standard) Errorf(fmt string, arg ...interface{}) {
	log.Error(fmt, arg...)
}

func (log *Standard) CErrorf(ctx context.Context, fmt string,
	arg ...interface{}) {
	if log.internal.IsEnabledFor(logging.ERROR) {
		log.Error(prepareString(ctx, fmt), arg...)
	}
}

func (log *Standard) Critical(fmt string, arg ...interface{}) {
	log.internal.Criticalf(fmt, arg...)
	if log.externalHandler != nil {
		log.externalHandler.Log(keybase1.LogLevel_CRITICAL, fmt, arg)
	}
}

func (log *Standard) CCriticalf(ctx context.Context, fmt string,
	arg ...interface{}) {
	if log.internal.IsEnabledFor(logging.CRITICAL) {
		log.Critical(prepareString(ctx, fmt), arg...)
	}
}

func (log *Standard) Fatalf(fmt string, arg ...interface{}) {
	log.internal.Fatalf(fmt, arg...)
	if log.externalHandler != nil {
		log.externalHandler.Log(keybase1.LogLevel_FATAL, fmt, arg)
	}
}

func (log *Standard) CFatalf(ctx context.Context, fmt string,
	arg ...interface{}) {
	log.Fatalf(prepareString(ctx, fmt), arg...)
}

func (log *Standard) Profile(fmts string, arg ...interface{}) {
	log.Debug(fmts, arg...)
}

// Configure sets the style of the log file, whether debugging (verbose)
// is enabled and a filename. If a filename is provided here it will
// be used for logging straight away (this is a new feature).
// SetLogFileConfig provides a way to set the log file with more control on rotation.
func (log *Standard) Configure(style string, debug bool, filename string) {
	log.configureMutex.Lock()
	defer log.configureMutex.Unlock()

	log.filename = filename

	var logfmt string

	globalLock.Lock()
	isTerm := stderrIsTerminal
	globalLock.Unlock()

	// TODO: how should setting the log file after a Configure be handled?
	if isTerm {
		if debug {
			logfmt = fancyFormat
		} else {
			logfmt = defaultFormat
		}
	} else {
		logfmt = fileFormat
	}

	// Override the format above if an explicit style was specified.
	switch style {
	case "default":
		logfmt = defaultFormat // Default
	case "plain":
		logfmt = plainFormat // Plain
	case "file":
		logfmt = fileFormat // Good for logging to files
	case "fancy":
		logfmt = fancyFormat // Fancy, good for terminals with color
	}

	if debug {
		logging.SetLevel(logging.DEBUG, log.module)
	}

	logging.SetFormatter(logging.MustStringFormatter(logfmt))

}

func OpenLogFile(filename string) (name string, file *os.File, err error) {
	name = filename
	if err = MakeParentDirs(name); err != nil {
		return
	}
	file, err = os.OpenFile(name, (os.O_APPEND | os.O_WRONLY | os.O_CREATE), 0600)
	if err != nil {
		return
	}
	return
}

func FileExists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}

func MakeParentDirs(filename string) error {
	dir, _ := filepath.Split(filename)
	// If passed a plain file name as a path
	if dir == "" {
		return nil
	}
	exists, err := FileExists(dir)
	if err != nil {
		return err
	}

	if !exists {
		err = os.MkdirAll(dir, permDir)
		if err != nil {
			return err
		}
	}
	return nil
}

func PickFirstError(errors ...error) error {
	for _, e := range errors {
		if e != nil {
			return e
		}
	}
	return nil
}

func (log *Standard) CloneWithAddedDepth(depth int) Logger {
	clone := *log
	cloneInternal := *log.internal
	cloneInternal.ExtraCalldepth = log.internal.ExtraCalldepth + depth
	clone.internal = &cloneInternal
	return &clone
}

func (log *Standard) SetExternalHandler(handler ExternalHandler) {
	log.externalHandler = handler
}

type UnforwardedLogger Standard

func (log *Standard) GetUnforwardedLogger() *UnforwardedLogger {
	return (*UnforwardedLogger)(log)
}

func (log *UnforwardedLogger) Debug(s string, args ...interface{})  { log.internal.Debugf(s, args...) }
func (log *UnforwardedLogger) Error(s string, args ...interface{})  { log.internal.Errorf(s, args...) }
func (log *UnforwardedLogger) Errorf(s string, args ...interface{}) { log.internal.Errorf(s, args...) }
func (log *UnforwardedLogger) Warning(s string, args ...interface{}) {
	log.internal.Warningf(s, args...)
}
func (log *UnforwardedLogger) Info(s string, args ...interface{})    { log.internal.Infof(s, args...) }
func (log *UnforwardedLogger) Profile(s string, args ...interface{}) { log.internal.Debugf(s, args...) }
<|endoftext|>


<commit_before>// Copyright 2012, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package rpcwrap

import (
	"bufio"
	"errors"
	"io"
	"net"
	"net/http"
	"time"

	log "github.com/golang/glog"
	rpc "github.com/youtube/vitess/go/rpcplus"
	"github.com/youtube/vitess/go/rpcwrap/auth"
	"github.com/youtube/vitess/go/rpcwrap/proto"
	"github.com/youtube/vitess/go/stats"
)

const (
	connected = "200 Connected to Go RPC"
)

var (
	connCount    = stats.NewInt("connection-count")
	connAccepted = stats.NewInt("connection-accepted")
)

type ClientCodecFactory func(conn io.ReadWriteCloser) rpc.ClientCodec

type BufferedConnection struct {
	isClosed bool
	*bufio.Reader
	io.WriteCloser
}

func NewBufferedConnection(conn io.ReadWriteCloser) *BufferedConnection {
	connCount.Add(1)
	connAccepted.Add(1)
	return &BufferedConnection{false, bufio.NewReader(conn), conn}
}

func (bc *BufferedConnection) Close() error {
	if !bc.isClosed {
		bc.isClosed = true
		connCount.Add(-1)
	}
	return bc.WriteCloser.Close()
}

// DialHTTP connects to a go HTTP RPC server using the specified codec.
// use 0 as connectTimeout for no timeout
func DialHTTP(network, address, codecName string, cFactory ClientCodecFactory, connectTimeout time.Duration) (*rpc.Client, error) {
	return dialHTTP(network, address, codecName, cFactory, false, connectTimeout)
}

// DialAuthHTTP connects to an authenticated go HTTP RPC server using
// the specified codec and credentials.
// use 0 as connectTimeout for no timeout
func DialAuthHTTP(network, address, user, password, codecName string, cFactory ClientCodecFactory, connectTimeout time.Duration) (conn *rpc.Client, err error) {
	if conn, err = dialHTTP(network, address, codecName, cFactory, true, connectTimeout); err != nil {
		return
	}
	reply := new(auth.GetNewChallengeReply)
	if err = conn.Call("AuthenticatorCRAMMD5.GetNewChallenge", "", reply); err != nil {
		return
	}
	proof := auth.CRAMMD5GetExpected(user, password, reply.Challenge)

	if err = conn.Call(
		"AuthenticatorCRAMMD5.Authenticate",
		auth.AuthenticateRequest{Proof: proof}, new(auth.AuthenticateReply)); err != nil {
		return
	}
	return
}

func dialHTTP(network, address, codecName string, cFactory ClientCodecFactory, auth bool, connectTimeout time.Duration) (*rpc.Client, error) {
	var err error
	var conn net.Conn
	if connectTimeout != 0 {
		conn, err = net.DialTimeout(network, address, connectTimeout)
	} else {
		conn, err = net.Dial(network, address)
	}
	if err != nil {
		return nil, err
	}

	io.WriteString(conn, "CONNECT "+GetRpcPath(codecName, auth)+" HTTP/1.0\n\n")

	// Require successful HTTP response
	// before switching to RPC protocol.
	buffered := NewBufferedConnection(conn)
	resp, err := http.ReadResponse(buffered.Reader, &http.Request{Method: "CONNECT"})
	if err == nil && resp.Status == connected {
		return rpc.NewClientWithCodec(cFactory(buffered)), nil
	}
	if err == nil {
		err = errors.New("unexpected HTTP response: " + resp.Status)
	}
	conn.Close()
	return nil, &net.OpError{"dial-http", network + " " + address, nil, err}
}

type ServerCodecFactory func(conn io.ReadWriteCloser) rpc.ServerCodec

// ServeRPC handles rpc requests using the hijack scheme of rpc
func ServeRPC(codecName string, cFactory ServerCodecFactory) {
	http.Handle(GetRpcPath(codecName, false), &rpcHandler{cFactory, false})
}

// ServeRPC handles rpc requests using the hijack scheme of rpc
func ServeAuthRPC(codecName string, cFactory ServerCodecFactory) {
	http.Handle(GetRpcPath(codecName, true), &rpcHandler{cFactory, true})
}

// ServeHTTP handles rpc requests in HTTP compliant POST form
func ServeHTTP(codecName string, cFactory ServerCodecFactory) {
	http.Handle(GetHttpPath(codecName), &httpHandler{cFactory})
}

// AuthenticatedServer is an rpc.Server instance that serves
// authenticated calls.
var AuthenticatedServer = rpc.NewServer()

// RegisterAuthenticated registers a receiver with the authenticated
// rpc server.
func RegisterAuthenticated(rcvr interface{}) error {
	// TODO(szopa): This should be removed after the transition
	// period, when all the clients know about authentication.
	if err := rpc.Register(rcvr); err != nil {
		return err
	}
	return AuthenticatedServer.Register(rcvr)
}

// ServeCodec calls ServeCodec for the appropriate server
// (authenticated or default).
func (h *rpcHandler) ServeCodecWithContext(c rpc.ServerCodec, context *proto.Context) {
	if h.useAuth {
		AuthenticatedServer.ServeCodecWithContext(c, context)
	} else {
		rpc.ServeCodecWithContext(c, context)
	}
}

type rpcHandler struct {
	cFactory ServerCodecFactory
	useAuth  bool
}

func (h *rpcHandler) ServeHTTP(c http.ResponseWriter, req *http.Request) {
	conn, _, err := c.(http.Hijacker).Hijack()
	if err != nil {
		log.Errorf("rpc hijacking %s: %v", req.RemoteAddr, err)
		return
	}
	io.WriteString(conn, "HTTP/1.0 "+connected+"\n\n")
	codec := h.cFactory(NewBufferedConnection(conn))
	context := &proto.Context{RemoteAddr: req.RemoteAddr}
	if h.useAuth {
		if authenticated, err := auth.Authenticate(codec, context); !authenticated {
			if err != nil {
				log.Errorf("authentication erred at %s: %v", req.RemoteAddr, err)
			}
			codec.Close()
			return
		}
	}
	h.ServeCodecWithContext(codec, context)
}

func GetRpcPath(codecName string, auth bool) string {
	path := "/_" + codecName + "_rpc_"
	if auth {
		path += "/auth"
	}
	return path
}

type httpHandler struct {
	cFactory ServerCodecFactory
}

func (hh *httpHandler) ServeHTTP(c http.ResponseWriter, req *http.Request) {
	conn := &httpConnectionBroker{c, req.Body}
	codec := hh.cFactory(conn)
	if err := rpc.ServeRequestWithContext(codec, &proto.Context{RemoteAddr: req.RemoteAddr}); err != nil {
		log.Errorf("rpcwrap: %v", err)
	}
}

// Emulate a read/write connection for the server codec
type httpConnectionBroker struct {
	http.ResponseWriter
	io.Reader
}

func (*httpConnectionBroker) Close() error {
	return nil
}

func GetHttpPath(codecName string) string {
	return "/_" + codecName + "_http_"
}
<commit_msg>add FIXME comment.<commit_after>// Copyright 2012, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package rpcwrap

import (
	"bufio"
	"errors"
	"io"
	"net"
	"net/http"
	"time"

	log "github.com/golang/glog"
	rpc "github.com/youtube/vitess/go/rpcplus"
	"github.com/youtube/vitess/go/rpcwrap/auth"
	"github.com/youtube/vitess/go/rpcwrap/proto"
	"github.com/youtube/vitess/go/stats"
)

const (
	connected = "200 Connected to Go RPC"
)

var (
	connCount    = stats.NewInt("connection-count")
	connAccepted = stats.NewInt("connection-accepted")
)

type ClientCodecFactory func(conn io.ReadWriteCloser) rpc.ClientCodec

type BufferedConnection struct {
	isClosed bool
	*bufio.Reader
	io.WriteCloser
}

func NewBufferedConnection(conn io.ReadWriteCloser) *BufferedConnection {
	connCount.Add(1)
	connAccepted.Add(1)
	return &BufferedConnection{false, bufio.NewReader(conn), conn}
}

// FIXME(sougou/szopa): Find a better way to track connection count.
func (bc *BufferedConnection) Close() error {
	if !bc.isClosed {
		bc.isClosed = true
		connCount.Add(-1)
	}
	return bc.WriteCloser.Close()
}

// DialHTTP connects to a go HTTP RPC server using the specified codec.
// use 0 as connectTimeout for no timeout
func DialHTTP(network, address, codecName string, cFactory ClientCodecFactory, connectTimeout time.Duration) (*rpc.Client, error) {
	return dialHTTP(network, address, codecName, cFactory, false, connectTimeout)
}

// DialAuthHTTP connects to an authenticated go HTTP RPC server using
// the specified codec and credentials.
// use 0 as connectTimeout for no timeout
func DialAuthHTTP(network, address, user, password, codecName string, cFactory ClientCodecFactory, connectTimeout time.Duration) (conn *rpc.Client, err error) {
	if conn, err = dialHTTP(network, address, codecName, cFactory, true, connectTimeout); err != nil {
		return
	}
	reply := new(auth.GetNewChallengeReply)
	if err = conn.Call("AuthenticatorCRAMMD5.GetNewChallenge", "", reply); err != nil {
		return
	}
	proof := auth.CRAMMD5GetExpected(user, password, reply.Challenge)

	if err = conn.Call(
		"AuthenticatorCRAMMD5.Authenticate",
		auth.AuthenticateRequest{Proof: proof}, new(auth.AuthenticateReply)); err != nil {
		return
	}
	return
}

func dialHTTP(network, address, codecName string, cFactory ClientCodecFactory, auth bool, connectTimeout time.Duration) (*rpc.Client, error) {
	var err error
	var conn net.Conn
	if connectTimeout != 0 {
		conn, err = net.DialTimeout(network, address, connectTimeout)
	} else {
		conn, err = net.Dial(network, address)
	}
	if err != nil {
		return nil, err
	}

	io.WriteString(conn, "CONNECT "+GetRpcPath(codecName, auth)+" HTTP/1.0\n\n")

	// Require successful HTTP response
	// before switching to RPC protocol.
	buffered := NewBufferedConnection(conn)
	resp, err := http.ReadResponse(buffered.Reader, &http.Request{Method: "CONNECT"})
	if err == nil && resp.Status == connected {
		return rpc.NewClientWithCodec(cFactory(buffered)), nil
	}
	if err == nil {
		err = errors.New("unexpected HTTP response: " + resp.Status)
	}
	conn.Close()
	return nil, &net.OpError{"dial-http", network + " " + address, nil, err}
}

type ServerCodecFactory func(conn io.ReadWriteCloser) rpc.ServerCodec

// ServeRPC handles rpc requests using the hijack scheme of rpc
func ServeRPC(codecName string, cFactory ServerCodecFactory) {
	http.Handle(GetRpcPath(codecName, false), &rpcHandler{cFactory, false})
}

// ServeRPC handles rpc requests using the hijack scheme of rpc
func ServeAuthRPC(codecName string, cFactory ServerCodecFactory) {
	http.Handle(GetRpcPath(codecName, true), &rpcHandler{cFactory, true})
}

// ServeHTTP handles rpc requests in HTTP compliant POST form
func ServeHTTP(codecName string, cFactory ServerCodecFactory) {
	http.Handle(GetHttpPath(codecName), &httpHandler{cFactory})
}

// AuthenticatedServer is an rpc.Server instance that serves
// authenticated calls.
var AuthenticatedServer = rpc.NewServer()

// RegisterAuthenticated registers a receiver with the authenticated
// rpc server.
func RegisterAuthenticated(rcvr interface{}) error {
	// TODO(szopa): This should be removed after the transition
	// period, when all the clients know about authentication.
	if err := rpc.Register(rcvr); err != nil {
		return err
	}
	return AuthenticatedServer.Register(rcvr)
}

// ServeCodec calls ServeCodec for the appropriate server
// (authenticated or default).
func (h *rpcHandler) ServeCodecWithContext(c rpc.ServerCodec, context *proto.Context) {
	if h.useAuth {
		AuthenticatedServer.ServeCodecWithContext(c, context)
	} else {
		rpc.ServeCodecWithContext(c, context)
	}
}

type rpcHandler struct {
	cFactory ServerCodecFactory
	useAuth  bool
}

func (h *rpcHandler) ServeHTTP(c http.ResponseWriter, req *http.Request) {
	conn, _, err := c.(http.Hijacker).Hijack()
	if err != nil {
		log.Errorf("rpc hijacking %s: %v", req.RemoteAddr, err)
		return
	}
	io.WriteString(conn, "HTTP/1.0 "+connected+"\n\n")
	codec := h.cFactory(NewBufferedConnection(conn))
	context := &proto.Context{RemoteAddr: req.RemoteAddr}
	if h.useAuth {
		if authenticated, err := auth.Authenticate(codec, context); !authenticated {
			if err != nil {
				log.Errorf("authentication erred at %s: %v", req.RemoteAddr, err)
			}
			codec.Close()
			return
		}
	}
	h.ServeCodecWithContext(codec, context)
}

func GetRpcPath(codecName string, auth bool) string {
	path := "/_" + codecName + "_rpc_"
	if auth {
		path += "/auth"
	}
	return path
}

type httpHandler struct {
	cFactory ServerCodecFactory
}

func (hh *httpHandler) ServeHTTP(c http.ResponseWriter, req *http.Request) {
	conn := &httpConnectionBroker{c, req.Body}
	codec := hh.cFactory(conn)
	if err := rpc.ServeRequestWithContext(codec, &proto.Context{RemoteAddr: req.RemoteAddr}); err != nil {
		log.Errorf("rpcwrap: %v", err)
	}
}

// Emulate a read/write connection for the server codec
type httpConnectionBroker struct {
	http.ResponseWriter
	io.Reader
}

func (*httpConnectionBroker) Close() error {
	return nil
}

func GetHttpPath(codecName string) string {
	return "/_" + codecName + "_http_"
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ssa_test

// This file runs the SSA builder in sanity-checking mode on all
// packages beneath $GOROOT and prints some summary information.
//
// Run test with GOMAXPROCS=8.

import (
	"go/token"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"testing"
	"time"

	"code.google.com/p/go.tools/go/loader"
	"code.google.com/p/go.tools/go/ssa"
	"code.google.com/p/go.tools/go/ssa/ssautil"
)

func allPackages() []string {
	var pkgs []string
	root := filepath.Join(runtime.GOROOT(), "src/pkg") + string(os.PathSeparator)
	filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
		// Prune the search if we encounter any of these names:
		switch filepath.Base(path) {
		case "testdata", ".hg":
			return filepath.SkipDir
		}
		if info.IsDir() {
			pkg := strings.TrimPrefix(path, root)
			switch pkg {
			case "builtin", "pkg", "code.google.com":
				return filepath.SkipDir // skip these subtrees
			case "":
				return nil // ignore root of tree
			}
			pkgs = append(pkgs, pkg)
		}

		return nil
	})
	return pkgs
}

func TestStdlib(t *testing.T) {
	// Load, parse and type-check the program.
	t0 := time.Now()

	var conf loader.Config
	if _, err := conf.FromArgs(allPackages()); err != nil {
		t.Errorf("FromArgs failed: %s", err)
		return
	}

	iprog, err := conf.Load()
	if err != nil {
		t.Errorf("Load failed: %s", err)
	}

	t1 := time.Now()

	runtime.GC()
	var memstats runtime.MemStats
	runtime.ReadMemStats(&memstats)
	alloc := memstats.Alloc

	// Create SSA packages.
	var mode ssa.BuilderMode
	// Comment out these lines during benchmarking.  Approx SSA build costs are noted.
	mode |= ssa.SanityCheckFunctions // + 2% space, + 4% time
	mode |= ssa.GlobalDebug          // +30% space, +18% time
	prog := ssa.Create(iprog, mode)

	t2 := time.Now()

	// Build SSA IR... if it's safe.
	prog.BuildAll()

	t3 := time.Now()

	runtime.GC()
	runtime.ReadMemStats(&memstats)

	numPkgs := len(prog.AllPackages())
	if want := 140; numPkgs < want {
		t.Errorf("Loaded only %d packages, want at least %d", numPkgs, want)
	}

	// Dump some statistics.
	allFuncs := ssautil.AllFunctions(prog)
	var numInstrs int
	for fn := range allFuncs {
		for _, b := range fn.Blocks {
			numInstrs += len(b.Instrs)
		}
	}

	// determine line count
	var lineCount int
	prog.Fset.Iterate(func(f *token.File) bool {
		lineCount += f.LineCount()
		return true
	})

	// NB: when benchmarking, don't forget to clear the debug +
	// sanity builder flags for better performance.

	t.Log("GOMAXPROCS:           ", runtime.GOMAXPROCS(0))
	t.Log("#Source lines:        ", lineCount)
	t.Log("Load/parse/typecheck: ", t1.Sub(t0))
	t.Log("SSA create:           ", t2.Sub(t1))
	t.Log("SSA build:            ", t3.Sub(t2))

	// SSA stats:
	t.Log("#Packages:            ", numPkgs)
	t.Log("#Functions:           ", len(allFuncs))
	t.Log("#Instructions:        ", numInstrs)
	t.Log("#MB:                  ", (memstats.Alloc-alloc)/1000000)
}
<commit_msg>go.tools/go/ssa: stop if error in TestStdlib<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ssa_test

// This file runs the SSA builder in sanity-checking mode on all
// packages beneath $GOROOT and prints some summary information.
//
// Run test with GOMAXPROCS=8.

import (
	"go/token"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"testing"
	"time"

	"code.google.com/p/go.tools/go/loader"
	"code.google.com/p/go.tools/go/ssa"
	"code.google.com/p/go.tools/go/ssa/ssautil"
)

func allPackages() []string {
	var pkgs []string
	root := filepath.Join(runtime.GOROOT(), "src/pkg") + string(os.PathSeparator)
	filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
		// Prune the search if we encounter any of these names:
		switch filepath.Base(path) {
		case "testdata", ".hg":
			return filepath.SkipDir
		}
		if info.IsDir() {
			pkg := strings.TrimPrefix(path, root)
			switch pkg {
			case "builtin", "pkg", "code.google.com":
				return filepath.SkipDir // skip these subtrees
			case "":
				return nil // ignore root of tree
			}
			pkgs = append(pkgs, pkg)
		}

		return nil
	})
	return pkgs
}

func TestStdlib(t *testing.T) {
	// Load, parse and type-check the program.
	t0 := time.Now()

	var conf loader.Config
	if _, err := conf.FromArgs(allPackages()); err != nil {
		t.Errorf("FromArgs failed: %v", err)
		return
	}

	iprog, err := conf.Load()
	if err != nil {
		t.Fatalf("Load failed: %v", err)
	}

	t1 := time.Now()

	runtime.GC()
	var memstats runtime.MemStats
	runtime.ReadMemStats(&memstats)
	alloc := memstats.Alloc

	// Create SSA packages.
	var mode ssa.BuilderMode
	// Comment out these lines during benchmarking.  Approx SSA build costs are noted.
	mode |= ssa.SanityCheckFunctions // + 2% space, + 4% time
	mode |= ssa.GlobalDebug          // +30% space, +18% time
	prog := ssa.Create(iprog, mode)

	t2 := time.Now()

	// Build SSA IR... if it's safe.
	prog.BuildAll()

	t3 := time.Now()

	runtime.GC()
	runtime.ReadMemStats(&memstats)

	numPkgs := len(prog.AllPackages())
	if want := 140; numPkgs < want {
		t.Errorf("Loaded only %d packages, want at least %d", numPkgs, want)
	}

	// Dump some statistics.
	allFuncs := ssautil.AllFunctions(prog)
	var numInstrs int
	for fn := range allFuncs {
		for _, b := range fn.Blocks {
			numInstrs += len(b.Instrs)
		}
	}

	// determine line count
	var lineCount int
	prog.Fset.Iterate(func(f *token.File) bool {
		lineCount += f.LineCount()
		return true
	})

	// NB: when benchmarking, don't forget to clear the debug +
	// sanity builder flags for better performance.

	t.Log("GOMAXPROCS:           ", runtime.GOMAXPROCS(0))
	t.Log("#Source lines:        ", lineCount)
	t.Log("Load/parse/typecheck: ", t1.Sub(t0))
	t.Log("SSA create:           ", t2.Sub(t1))
	t.Log("SSA build:            ", t3.Sub(t2))

	// SSA stats:
	t.Log("#Packages:            ", numPkgs)
	t.Log("#Functions:           ", len(allFuncs))
	t.Log("#Instructions:        ", numInstrs)
	t.Log("#MB:                  ", (memstats.Alloc-alloc)/1000000)
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package types

// A Type represents a type of Go.
// All types implement the Type interface.
type Type interface {
	// Underlying returns the underlying type of a type
	// w/o following forwarding chains. Only used by
	// client packages.
	Underlying() Type

	// String returns a string representation of a type.
	String() string
}

// under returns the true expanded underlying type.
// If it doesn't exist, the result is Typ[Invalid].
// under must only be called when a type is known
// to be fully set up.
func under(t Type) Type {
	if t, _ := t.(*Named); t != nil {
		return t.under()
	}
	return t.Underlying()
}

// If x and y are identical, match returns x.
// If x and y are identical channels but for their direction
// and one of them is unrestricted, match returns the channel
// with the restricted direction.
// In all other cases, match returns nil.
func match(x, y Type) Type {
	// Common case: we don't have channels.
	if Identical(x, y) {
		return x
	}

	// We may have channels that differ in direction only.
	if x, _ := x.(*Chan); x != nil {
		if y, _ := y.(*Chan); y != nil && Identical(x.elem, y.elem) {
			// We have channels that differ in direction only.
			// If there's an unrestricted channel, select the restricted one.
			switch {
			case x.dir == SendRecv:
				return y
			case y.dir == SendRecv:
				return x
			}
		}
	}

	// types are different
	return nil
}

// If typ is a type parameter, structuralType returns the single underlying
// type of all types in the corresponding type constraint if it exists, or
// nil otherwise. If the type set contains only unrestricted and restricted
// channel types (with identical element types), the single underlying type
// is the restricted channel type if the restrictions are always the same.
// If typ is not a type parameter, structuralType returns the underlying type.
func structuralType(typ Type) Type {
	var su Type
	if underIs(typ, func(u Type) bool {
		if u == nil {
			return false
		}
		if su != nil {
			u = match(su, u)
			if u == nil {
				return false
			}
		}
		// su == nil || match(su, u) != nil
		su = u
		return true
	}) {
		return su
	}
	return nil
}
<commit_msg>go/types: optimize common case in structuralType<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package types

// A Type represents a type of Go.
// All types implement the Type interface.
type Type interface {
	// Underlying returns the underlying type of a type
	// w/o following forwarding chains. Only used by
	// client packages.
	Underlying() Type

	// String returns a string representation of a type.
	String() string
}

// under returns the true expanded underlying type.
// If it doesn't exist, the result is Typ[Invalid].
// under must only be called when a type is known
// to be fully set up.
func under(t Type) Type {
	if t, _ := t.(*Named); t != nil {
		return t.under()
	}
	return t.Underlying()
}

// If x and y are identical, match returns x.
// If x and y are identical channels but for their direction
// and one of them is unrestricted, match returns the channel
// with the restricted direction.
// In all other cases, match returns nil.
func match(x, y Type) Type {
	// Common case: we don't have channels.
	if Identical(x, y) {
		return x
	}

	// We may have channels that differ in direction only.
	if x, _ := x.(*Chan); x != nil {
		if y, _ := y.(*Chan); y != nil && Identical(x.elem, y.elem) {
			// We have channels that differ in direction only.
			// If there's an unrestricted channel, select the restricted one.
			switch {
			case x.dir == SendRecv:
				return y
			case y.dir == SendRecv:
				return x
			}
		}
	}

	// types are different
	return nil
}

// If t is not a type parameter, structuralType returns the underlying type.
// If t is a type parameter, structuralType returns the single underlying
// type of all types in its type set if it exists, or nil otherwise. If the
// type set contains only unrestricted and restricted channel types (with
// identical element types), the single underlying type is the restricted
// channel type if the restrictions are always the same, or nil otherwise.
func structuralType(t Type) Type {
	tpar, _ := t.(*TypeParam)
	if tpar == nil {
		return under(t)
	}

	var su Type
	if tpar.underIs(func(u Type) bool {
		if u == nil {
			return false
		}
		if su != nil {
			u = match(su, u)
			if u == nil {
				return false
			}
		}
		// su == nil || match(su, u) != nil
		su = u
		return true
	}) {
		return su
	}
	return nil
}
<|endoftext|>