<commit_before>package openjpeg

import (
	"color-assert"
	"image"
	"os"
	"testing"
)

func jp2i() *JP2Image {
	dir, _ := os.Getwd()
	jp2, err := NewJP2Image(dir + "/../testfile/test-world.jp2")
	if err != nil {
		panic("Error reading JP2 for testing!")
	}
	return jp2
}

func TestNewJP2Image(t *testing.T) {
	jp2 := jp2i()

	if jp2 == nil {
		t.Error("No JP2 object!")
	}

	t.Log(jp2.image)
}

func TestDimensions(t *testing.T) {
	jp2 := jp2i()
	jp2.ReadHeader()
	assert.Equal(800, jp2.GetWidth(), "jp2 width is 800px", t)
	assert.Equal(400, jp2.GetHeight(), "jp2 height is 400px", t)
}

func TestDirectConversion(t *testing.T) {
	jp2 := jp2i()
	i, err := jp2.DecodeImage()
	assert.Equal(err, nil, "No error decoding jp2", t)
	assert.Equal(0, i.Bounds().Min.X, "Min.X should be 0", t)
	assert.Equal(0, i.Bounds().Min.Y, "Min.Y should be 0", t)
	assert.Equal(800, i.Bounds().Max.X, "Max.X should be 800", t)
	assert.Equal(400, i.Bounds().Max.Y, "Max.Y should be 400", t)
}

func TestCrop(t *testing.T) {
	jp2 := jp2i()
	jp2.SetCrop(image.Rect(200, 100, 500, 400))
	i, err := jp2.DecodeImage()
	assert.Equal(err, nil, "No error decoding jp2", t)
	assert.Equal(0, i.Bounds().Min.X, "Min.X should be 0", t)
	assert.Equal(0, i.Bounds().Min.Y, "Min.Y should be 0", t)
	assert.Equal(300, i.Bounds().Max.X, "Max.X should be 300 (cropped X from 200 - 500)", t)
	assert.Equal(300, i.Bounds().Max.Y, "Max.Y should be 300 (cropped Y from 100 - 400)", t)
}

// This serves as a resize test as well as a test that we properly check
// maximum resolution factor
func TestResizeWH(t *testing.T) {
	jp2 := jp2i()
	jp2.SetResizeWH(50, 50)
	i, err := jp2.DecodeImage()
	assert.Equal(err, nil, "No error decoding jp2", t)
	assert.Equal(0, i.Bounds().Min.X, "Min.X should be 0", t)
	assert.Equal(0, i.Bounds().Min.Y, "Min.Y should be 0", t)
	assert.Equal(50, i.Bounds().Max.X, "Max.X should be 50", t)
	assert.Equal(50, i.Bounds().Max.Y, "Max.Y should be 50", t)
}

func TestResizeWHAndCrop(t *testing.T) {
	jp2 := jp2i()
	jp2.SetCrop(image.Rect(200, 100, 500, 400))
	jp2.SetResizeWH(125, 125)
	i, err := jp2.DecodeImage()
	assert.Equal(err, nil, "No error decoding jp2", t)
	assert.Equal(0, i.Bounds().Min.X, "Min.X should be 0", t)
	assert.Equal(0, i.Bounds().Min.Y, "Min.Y should be 0", t)
	assert.Equal(125, i.Bounds().Max.X, "Max.X should be 125", t)
	assert.Equal(125, i.Bounds().Max.Y, "Max.Y should be 125", t)
}
<commit_msg>Fix JP2 Image Test path to test-world.jp2 file<commit_after>package openjpeg

import (
	"color-assert"
	"image"
	"os"
	"testing"
)

func jp2i() *JP2Image {
	dir, _ := os.Getwd()
	jp2, err := NewJP2Image(dir + "/../../testfile/test-world.jp2")
	if err != nil {
		panic("Error reading JP2 for testing!")
	}
	return jp2
}

func TestNewJP2Image(t *testing.T) {
	jp2 := jp2i()

	if jp2 == nil {
		t.Error("No JP2 object!")
	}

	t.Log(jp2.image)
}

func TestDimensions(t *testing.T) {
	jp2 := jp2i()
	jp2.ReadHeader()
	assert.Equal(800, jp2.GetWidth(), "jp2 width is 800px", t)
	assert.Equal(400, jp2.GetHeight(), "jp2 height is 400px", t)
}

func TestDirectConversion(t *testing.T) {
	jp2 := jp2i()
	i, err := jp2.DecodeImage()
	assert.Equal(err, nil, "No error decoding jp2", t)
	assert.Equal(0, i.Bounds().Min.X, "Min.X should be 0", t)
	assert.Equal(0, i.Bounds().Min.Y, "Min.Y should be 0", t)
	assert.Equal(800, i.Bounds().Max.X, "Max.X should be 800", t)
	assert.Equal(400, i.Bounds().Max.Y, "Max.Y should be 400", t)
}

func TestCrop(t *testing.T) {
	jp2 := jp2i()
	jp2.SetCrop(image.Rect(200, 100, 500, 400))
	i, err := jp2.DecodeImage()
	assert.Equal(err, nil, "No error decoding jp2", t)
	assert.Equal(0, i.Bounds().Min.X, "Min.X should be 0", t)
	assert.Equal(0, i.Bounds().Min.Y, "Min.Y should be 0", t)
	assert.Equal(300, i.Bounds().Max.X, "Max.X should be 300 (cropped X from 200 - 500)", t)
	assert.Equal(300, i.Bounds().Max.Y, "Max.Y should be 300 (cropped Y from 100 - 400)", t)
}

// This serves as a resize test as well as a test that we properly check
// maximum resolution factor
func TestResizeWH(t *testing.T) {
	jp2 := jp2i()
	jp2.SetResizeWH(50, 50)
	i, err := jp2.DecodeImage()
	assert.Equal(err, nil, "No error decoding jp2", t)
	assert.Equal(0, i.Bounds().Min.X, "Min.X should be 0", t)
	assert.Equal(0, i.Bounds().Min.Y, "Min.Y should be 0", t)
	assert.Equal(50, i.Bounds().Max.X, "Max.X should be 50", t)
	assert.Equal(50, i.Bounds().Max.Y, "Max.Y should be 50", t)
}

func TestResizeWHAndCrop(t *testing.T) {
	jp2 := jp2i()
	jp2.SetCrop(image.Rect(200, 100, 500, 400))
	jp2.SetResizeWH(125, 125)
	i, err := jp2.DecodeImage()
	assert.Equal(err, nil, "No error decoding jp2", t)
	assert.Equal(0, i.Bounds().Min.X, "Min.X should be 0", t)
	assert.Equal(0, i.Bounds().Min.Y, "Min.Y should be 0", t)
	assert.Equal(125, i.Bounds().Max.X, "Max.X should be 125", t)
	assert.Equal(125, i.Bounds().Max.Y, "Max.Y should be 125", t)
}
<|endoftext|>


<commit_before>package ghwebhookauth
<commit_msg>Add tests<commit_after>package ghwebhookauth

import (
	"bytes"
	"crypto/hmac"
	"crypto/sha1"
	"encoding/hex"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
)

func dummyHandler(w http.ResponseWriter, r *http.Request) {
	w.Write([]byte("OK"))
}

func TestNew(t *testing.T) {
	assert := assert.New(t)
	g := New("mys3cr3t")
	assert.NotNil(g)
}

func TestDisallowedMethods(t *testing.T) {
	assert := assert.New(t)
	g := New("mys3cr3t")
	assert.NotNil(g)

	disallow := []string{"GET", "PUT", "DELETE", "PATCH", "HEAD"}

	for _, m := range disallow {
		w := httptest.NewRecorder()
		req, err := http.NewRequest(m, "http://localhost/foobar", nil)
		if err != nil {
			t.Fatal(err)
		}
		g.ServeHTTP(w, req, dummyHandler)
		assert.Equal(w.Code, 405, "Disallowed methods should return 405")
	}

}

func TestEmptyPost(t *testing.T) {
	assert := assert.New(t)
	g := New("mys3cr3t")
	assert.NotNil(g)

	w := httptest.NewRecorder()
	req, err := http.NewRequest("POST", "http://localhost/foobar", nil)
	if err != nil {
		t.Fatal(err)
	}
	req.Header.Set(HeaderName, "dummyvalue")
	g.ServeHTTP(w, req, dummyHandler)
	assert.Equal(w.Code, 400, "Empty body returns 400")
}

func TestMissingHeader(t *testing.T) {
	assert := assert.New(t)
	g := New("mys3cr3t")
	assert.NotNil(g)

	w := httptest.NewRecorder()
	body := strings.NewReader("body")
	req, err := http.NewRequest("POST", "http://localhost/foobar", body)
	if err != nil {
		t.Fatal(err)
	}
	g.ServeHTTP(w, req, dummyHandler)
	assert.Equal(w.Code, 400, "Missing header returns 400")
	assert.Equal(string(w.Body.Bytes()), ErrMissingHeader.Error()+"\n", "Should equal missing header")
}

func TestEmptyHeader(t *testing.T) {
	assert := assert.New(t)
	g := New("mys3cr3t")
	assert.NotNil(g)

	w := httptest.NewRecorder()
	body := strings.NewReader("body")
	req, err := http.NewRequest("POST", "http://localhost/foobar", body)
	if err != nil {
		t.Fatal(err)
	}
	req.Header.Set(HeaderName, "")
	g.ServeHTTP(w, req, dummyHandler)
	assert.Equal(w.Code, 400, "Missing header returns 400")
	assert.Equal(string(w.Body.Bytes()), ErrMissingHeader.Error()+"\n", "Should equal missing header")
}

func TestInvalidSignature(t *testing.T) {
	assert := assert.New(t)
	g := New("mys3cr3t")
	assert.NotNil(g)

	w := httptest.NewRecorder()
	body := strings.NewReader("body")
	req, err := http.NewRequest("POST", "http://localhost/foobar", body)
	if err != nil {
		t.Fatal(err)
	}
	req.Header.Set(HeaderName, "dummyvalue")
	g.ServeHTTP(w, req, dummyHandler)
	assert.Equal(w.Code, 400, "Empty body returns 400")
	assert.Equal(string(w.Body.Bytes()), ErrInvalidSignature.Error()+"\n", "Should equal invalid signature")
}

func TestValidRequest(t *testing.T) {
	assert := assert.New(t)
	g := New("mys3cr3t")
	assert.NotNil(g)

	w := httptest.NewRecorder()
	body := strings.NewReader("body")
	req, err := http.NewRequest("POST", "http://localhost/foobar", body)
	if err != nil {
		t.Fatal(err)
	}

	digest := hmac.New(sha1.New, []byte("mys3cr3t"))
	digest.Write([]byte("body"))
	if err != nil {
		t.Fatal(err)
	}
	b := bytes.NewBufferString("sha1=" + hex.EncodeToString(digest.Sum(nil)))

	req.Header.Set(HeaderName, string(b.Bytes()))
	g.ServeHTTP(w, req, dummyHandler)
	assert.Equal(w.Code, 200, "Empty body returns 200")
	assert.Equal(string(w.Body.Bytes()), "OK", "Should equal invalid signature")
}
<|endoftext|>


<commit_before>package main

import (
	"bitbucket.org/sinbad/git-lob/util"
	"fmt"
	"os"
	"path/filepath"
	"runtime/debug"
)

func main() {
	// Need to send the result code to the OS but also need to support 'defer'
	// os.Exit would finish before any defers, so wrap everything in mainImpl()
	os.Exit(MainImpl())

}

func MainImpl() int {

	// Generic panic handler so we get stack trace
	defer func() {
		if e := recover(); e != nil {
			fmt.Fprintf(os.Stderr, "git-lob-serve panic: %v\n", e)
			fmt.Fprint(os.Stderr, string(debug.Stack()))
			os.Exit(99)
		}

	}()

	// Get set up
	cfg := LoadConfig()

	if cfg.BasePath == "" {
		fmt.Fprintf(os.Stderr, "Missing required configuration setting: base-path\n")
		return 12
	}
	if util.DirExists(cfg.BasePath) {
		fmt.Fprintf(os.Stderr, "Invalid value for base-path: %v\nDirectory must exist.\n", cfg.BasePath)
		return 14
	}
	// Change to the base path directory so filepath.Clean() can work with relative dirs
	os.Chdir(cfg.BasePath)

	if cfg.DeltaCachePath != "" && !util.DirExists(cfg.DeltaCachePath) {
		// Create delta cache if doesn't exist, use same permissions as base path
		s, err := os.Stat(cfg.BasePath)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Invalid value for base-path: %v\nCannot stat: %v\n", cfg.BasePath, err.Error())
			return 16
		}
		err = os.MkdirAll(cfg.DeltaCachePath, s.Mode())
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error creating delta cache path %v: %v\n", cfg.DeltaCachePath, err.Error())
			return 16
		}
	}

	// Get path argument
	if len(os.Args) < 2 {
		fmt.Fprintf(os.Stderr, "Path argument missing, cannot continue\n")
		return 18
	}
	path := filepath.Clean(os.Args[1])
	if filepath.IsAbs(path) && !cfg.AllowAbsolutePaths {
		fmt.Fprintf(os.Stderr, "Path argument %v invalid, absolute paths are not allowed by this server\n", path)
		return 18
	}

	return Serve(os.Stdin, os.Stdout, os.Stderr, cfg, path)
}
<commit_msg>Fix base path existence check in git-lob-serve<commit_after>package main

import (
	"bitbucket.org/sinbad/git-lob/util"
	"fmt"
	"os"
	"path/filepath"
	"runtime/debug"
)

func main() {
	// Need to send the result code to the OS but also need to support 'defer'
	// os.Exit would finish before any defers, so wrap everything in mainImpl()
	os.Exit(MainImpl())

}

func MainImpl() int {

	// Generic panic handler so we get stack trace
	defer func() {
		if e := recover(); e != nil {
			fmt.Fprintf(os.Stderr, "git-lob-serve panic: %v\n", e)
			fmt.Fprint(os.Stderr, string(debug.Stack()))
			os.Exit(99)
		}

	}()

	// Get set up
	cfg := LoadConfig()

	if cfg.BasePath == "" {
		fmt.Fprintf(os.Stderr, "Missing required configuration setting: base-path\n")
		return 12
	}
	if !util.DirExists(cfg.BasePath) {
		fmt.Fprintf(os.Stderr, "Invalid value for base-path: %v\nDirectory must exist.\n", cfg.BasePath)
		return 14
	}
	// Change to the base path directory so filepath.Clean() can work with relative dirs
	os.Chdir(cfg.BasePath)

	if cfg.DeltaCachePath != "" && !util.DirExists(cfg.DeltaCachePath) {
		// Create delta cache if doesn't exist, use same permissions as base path
		s, err := os.Stat(cfg.BasePath)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Invalid value for base-path: %v\nCannot stat: %v\n", cfg.BasePath, err.Error())
			return 16
		}
		err = os.MkdirAll(cfg.DeltaCachePath, s.Mode())
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error creating delta cache path %v: %v\n", cfg.DeltaCachePath, err.Error())
			return 16
		}
	}

	// Get path argument
	if len(os.Args) < 2 {
		fmt.Fprintf(os.Stderr, "Path argument missing, cannot continue\n")
		return 18
	}
	path := filepath.Clean(os.Args[1])
	if filepath.IsAbs(path) && !cfg.AllowAbsolutePaths {
		fmt.Fprintf(os.Stderr, "Path argument %v invalid, absolute paths are not allowed by this server\n", path)
		return 18
	}

	return Serve(os.Stdin, os.Stdout, os.Stderr, cfg, path)
}
<|endoftext|>


<commit_before>// +build !linux

package selinux

import (
	"errors"
)

const (
	// Enforcing constant indicate SELinux is in enforcing mode
	Enforcing = 1
	// Permissive constant to indicate SELinux is in permissive mode
	Permissive = 0
	// Disabled constant to indicate SELinux is disabled
	Disabled = -1
)

var (
	// ErrMCSAlreadyExists is returned when trying to allocate a duplicate MCS.
	ErrMCSAlreadyExists = errors.New("MCS label already exists")
	// ErrEmptyPath is returned when an empty path has been specified.
	ErrEmptyPath = errors.New("empty path")
)

// Context is a representation of the SELinux label broken into 4 parts
type Context map[string]string

// SetDisabled disables selinux support for the package
func SetDisabled() {
	return
}

// SetFileLabel sets the SELinux label for this path or returns an error.
func SetFileLabel(fpath string, label string) error {
	return nil
}

// FileLabel returns the SELinux label for this path or returns an error.
func FileLabel(fpath string) (string, error) {
	return "", nil
}

/*
SetFSCreateLabel tells kernel the label to create all file system objects
created by this task. Setting label="" to return to default.
*/
func SetFSCreateLabel(label string) error {
	return nil
}

/*
FSCreateLabel returns the default label the kernel which the kernel is using
for file system objects created by this task. "" indicates default.
*/
func FSCreateLabel() (string, error) {
	return nil
}

// CurrentLabel returns the SELinux label of the current process thread, or an error.
func CurrentLabel() (string, error) {
	return "", nil
}

// PidLabel returns the SELinux label of the given pid, or an error.
func PidLabel(pid int) (string, error) {
	return "", nil
}

/*
ExecLabel returns the SELinux label that the kernel will use for any programs
that are executed by the current process thread, or an error.
*/
func ExecLabel() (string, error) {
	return "", nil
}

/*
CanonicalizeContext takes a context string and writes it to the kernel
the function then returns the context that the kernel will use.  This function
can be used to see if two contexts are equivalent
*/
func CanonicalizeContext(val string) (string, error) {
	return "", nil
}

/*
SetExecLabel sets the SELinux label that the kernel will use for any programs
that are executed by the current process thread, or an error.
*/
func SetExecLabel(label string) error {
	return nil
}

// Get returns the Context as a string
func (c Context) Get() string {
	return ""
}

// NewContext creates a new Context struct from the specified label
func NewContext(label string) Context {
	c := make(Context)
	return c
}

// ReserveLabel reserves the MLS/MCS level component of the specified label
func ReserveLabel(label string) {
	return
}

// EnforceMode returns the current SELinux mode Enforcing, Permissive, Disabled
func EnforceMode() int {
	return Disabled
}

/*
SetEnforceMode sets the current SELinux mode Enforcing, Permissive.
Disabled is not valid, since this needs to be set at boot time.
*/
func SetEnforceMode(mode int) error {
	return nil
}

/*
DefaultEnforceMode returns the systems default SELinux mode Enforcing,
Permissive or Disabled. Note this is is just the default at boot time.
EnforceMode tells you the systems current mode.
*/
func DefaultEnforceMode() int {
	return Disabled
}

/*
ReleaseLabel will unreserve the MLS/MCS Level field of the specified label.
Allowing it to be used by another process.
*/
func ReleaseLabel(label string) {
	return
}

// ROFileLabel returns the specified SELinux readonly file label
func ROFileLabel() string {
	return ""
}

/*
ContainerLabels returns an allocated processLabel and fileLabel to be used for
container labeling by the calling process.
*/
func ContainerLabels() (processLabel string, fileLabel string) {
	return "", ""
}

// SecurityCheckContext validates that the SELinux label is understood by the kernel
func SecurityCheckContext(val string) error {
	return nil
}

/*
CopyLevel returns a label with the MLS/MCS level from src label replaced on
the dest label.
*/
func CopyLevel(src, dest string) (string, error) {
	return "", nil
}

// Chcon changes the `fpath` file object to the SELinux label `label`.
// If `fpath` is a directory and `recurse`` is true, Chcon will walk the
// directory tree setting the label.
func Chcon(fpath string, label string, recurse bool) error {
	return nil
}

// DupSecOpt takes an SELinux process label and returns security options that
// can be used to set the SELinux Type and Level for future container processes.
func DupSecOpt(src string) []string {
	return nil
}

// DisableSecOpt returns a security opt that can be used to disable SELinux
// labeling support for future container processes.
func DisableSecOpt() []string {
	return []string{"disable"}
}
<commit_msg>go-selinux/selinux.go: fix return value for FSCreateLabel<commit_after>// +build !linux

package selinux

import (
	"errors"
)

const (
	// Enforcing constant indicate SELinux is in enforcing mode
	Enforcing = 1
	// Permissive constant to indicate SELinux is in permissive mode
	Permissive = 0
	// Disabled constant to indicate SELinux is disabled
	Disabled = -1
)

var (
	// ErrMCSAlreadyExists is returned when trying to allocate a duplicate MCS.
	ErrMCSAlreadyExists = errors.New("MCS label already exists")
	// ErrEmptyPath is returned when an empty path has been specified.
	ErrEmptyPath = errors.New("empty path")
)

// Context is a representation of the SELinux label broken into 4 parts
type Context map[string]string

// SetDisabled disables selinux support for the package
func SetDisabled() {
	return
}

// SetFileLabel sets the SELinux label for this path or returns an error.
func SetFileLabel(fpath string, label string) error {
	return nil
}

// FileLabel returns the SELinux label for this path or returns an error.
func FileLabel(fpath string) (string, error) {
	return "", nil
}

/*
SetFSCreateLabel tells kernel the label to create all file system objects
created by this task. Setting label="" to return to default.
*/
func SetFSCreateLabel(label string) error {
	return nil
}

/*
FSCreateLabel returns the default label the kernel which the kernel is using
for file system objects created by this task. "" indicates default.
*/
func FSCreateLabel() (string, error) {
	return "", nil
}

// CurrentLabel returns the SELinux label of the current process thread, or an error.
func CurrentLabel() (string, error) {
	return "", nil
}

// PidLabel returns the SELinux label of the given pid, or an error.
func PidLabel(pid int) (string, error) {
	return "", nil
}

/*
ExecLabel returns the SELinux label that the kernel will use for any programs
that are executed by the current process thread, or an error.
*/
func ExecLabel() (string, error) {
	return "", nil
}

/*
CanonicalizeContext takes a context string and writes it to the kernel
the function then returns the context that the kernel will use.  This function
can be used to see if two contexts are equivalent
*/
func CanonicalizeContext(val string) (string, error) {
	return "", nil
}

/*
SetExecLabel sets the SELinux label that the kernel will use for any programs
that are executed by the current process thread, or an error.
*/
func SetExecLabel(label string) error {
	return nil
}

// Get returns the Context as a string
func (c Context) Get() string {
	return ""
}

// NewContext creates a new Context struct from the specified label
func NewContext(label string) Context {
	c := make(Context)
	return c
}

// ReserveLabel reserves the MLS/MCS level component of the specified label
func ReserveLabel(label string) {
	return
}

// EnforceMode returns the current SELinux mode Enforcing, Permissive, Disabled
func EnforceMode() int {
	return Disabled
}

/*
SetEnforceMode sets the current SELinux mode Enforcing, Permissive.
Disabled is not valid, since this needs to be set at boot time.
*/
func SetEnforceMode(mode int) error {
	return nil
}

/*
DefaultEnforceMode returns the systems default SELinux mode Enforcing,
Permissive or Disabled. Note this is is just the default at boot time.
EnforceMode tells you the systems current mode.
*/
func DefaultEnforceMode() int {
	return Disabled
}

/*
ReleaseLabel will unreserve the MLS/MCS Level field of the specified label.
Allowing it to be used by another process.
*/
func ReleaseLabel(label string) {
	return
}

// ROFileLabel returns the specified SELinux readonly file label
func ROFileLabel() string {
	return ""
}

/*
ContainerLabels returns an allocated processLabel and fileLabel to be used for
container labeling by the calling process.
*/
func ContainerLabels() (processLabel string, fileLabel string) {
	return "", ""
}

// SecurityCheckContext validates that the SELinux label is understood by the kernel
func SecurityCheckContext(val string) error {
	return nil
}

/*
CopyLevel returns a label with the MLS/MCS level from src label replaced on
the dest label.
*/
func CopyLevel(src, dest string) (string, error) {
	return "", nil
}

// Chcon changes the `fpath` file object to the SELinux label `label`.
// If `fpath` is a directory and `recurse`` is true, Chcon will walk the
// directory tree setting the label.
func Chcon(fpath string, label string, recurse bool) error {
	return nil
}

// DupSecOpt takes an SELinux process label and returns security options that
// can be used to set the SELinux Type and Level for future container processes.
func DupSecOpt(src string) []string {
	return nil
}

// DisableSecOpt returns a security opt that can be used to disable SELinux
// labeling support for future container processes.
func DisableSecOpt() []string {
	return []string{"disable"}
}
<|endoftext|>


<commit_before>// Convert Wikipedia XML dump to JSON or extract categories
package main

import (
	"encoding/json"
	"encoding/xml"
	"flag"
	"fmt"
	"net/url"
	"os"
	"regexp"
	"strings"
)

const AppVersion = "1.0.5"

// Here is an example article from the Wikipedia XML dump
//
// <page>
// 	<title>Apollo 11</title>
//      <redirect title="Foo bar" />
// 	...
// 	<revision>
// 	...
// 	  <text xml:space="preserve">
// 	  {{Infobox Space mission
// 	  |mission_name=<!--See above-->
// 	  |insignia=Apollo_11_insignia.png
// 	...
// 	  </text>
// 	</revision>
// </page>
//
// Note how the tags on the fields of Page and Redirect below
// describe the XML schema structure.

type Redirect struct {
	Title string `xml:"title,attr" json:"title"`
}

type Page struct {
	Title          string   `xml:"title" json:"title"`
	CanonicalTitle string   `xml:"ctitle" json:"ctitle"`
	Redir          Redirect `xml:"redirect" json:"redirect"`
	Text           string   `xml:"revision>text" json:"text"`
}

func CanonicalizeTitle(title string) string {
	can := strings.ToLower(title)
	can = strings.Replace(can, " ", "_", -1)
	can = url.QueryEscape(can)
	return can
}

func main() {
	version := flag.Bool("v", false, "prints current version and exits")
	extractCategories := flag.String("c", "", "only extract categories TSV(page, category")
	extractAuthorityData := flag.Bool("a", false, "only extract authority data (Normdaten)")
	filter, _ := regexp.Compile("^file:.*|^talk:.*|^special:.*|^wikipedia:.*|^wiktionary:.*|^user:.*|^user_talk:.*")

	flag.Parse()

	if *extractCategories != "" && *extractAuthorityData {
		fmt.Println("It's either -a or -c")
		os.Exit(1)
	}

	if *version {
		fmt.Println(AppVersion)
		os.Exit(0)
	}

	if flag.NArg() < 1 {
		fmt.Println("Usage: wptojson WIKIPEDIA-XML-DUMP")
		os.Exit(1)
	}
	inputFile := flag.Args()[0]

	xmlFile, err := os.Open(inputFile)
	if err != nil {
		fmt.Println("Error opening file:", err)
		return
	}
	defer xmlFile.Close()

	decoder := xml.NewDecoder(xmlFile)
	var inElement string
	categoryPattern := regexp.MustCompile(`\[\[` + *extractCategories + `:([^\[]+)\]\]`)
	authorityDataPattern := regexp.MustCompile(`(?mi){{Normdaten[^}]*}}`)

	for {
		// Read tokens from the XML document in a stream.
		t, _ := decoder.Token()
		if t == nil {
			break
		}
		// Inspect the type of the token just read.
		switch se := t.(type) {
		case xml.StartElement:
			// If we just read a StartElement token
			inElement = se.Name.Local
			// ...and its name is "page"
			if inElement == "page" {
				var p Page
				// decode a whole chunk of following XML into the
				// variable p which is a Page (se above)
				decoder.DecodeElement(&p, &se)

				// Do some stuff with the page.
				p.CanonicalTitle = CanonicalizeTitle(p.Title)
				m := filter.MatchString(p.CanonicalTitle)
				if !m && p.Redir.Title == "" {
					if *extractCategories != "" {
						result := categoryPattern.FindAllStringSubmatch(p.Text, -1)
						for _, value := range result {
							// replace anything after a |
							category := strings.TrimSpace(value[1])
							firstIndex := strings.Index(category, "|")
							if firstIndex != -1 {
								category = category[0:firstIndex]
							}
							fmt.Printf("%s\t%s\n", p.Title, category)
						}
					} else if *extractAuthorityData {
						result := authorityDataPattern.FindString(p.Text)
						if result != "" {
							// https://cdn.mediacru.sh/JsdjtGoLZBcR.png
							result = strings.Replace(result, "\t", "", -1)
							fmt.Printf("%s\t%s\n", p.Title, result)
						}
					} else {
						b, err := json.Marshal(p)
						if err != nil {
							os.Exit(2)
						}
						fmt.Println(string(b))
					}
				}
			}
		default:
		}
	}
}
<commit_msg>added -d for wikidata inner json decoding<commit_after>// Convert Wikipedia XML dump to JSON or extract categories
package main

import (
	"encoding/json"
	"encoding/xml"
	"flag"
	"fmt"
	"net/url"
	"os"
	"regexp"
	"strings"
)

const AppVersion = "1.0.6"

// Here is an example article from the Wikipedia XML dump
//
// <page>
// 	<title>Apollo 11</title>
//      <redirect title="Foo bar" />
// 	...
// 	<revision>
// 	...
// 	  <text xml:space="preserve">
// 	  {{Infobox Space mission
// 	  |mission_name=<!--See above-->
// 	  |insignia=Apollo_11_insignia.png
// 	...
// 	  </text>
// 	</revision>
// </page>
//
// Note how the tags on the fields of Page and Redirect below
// describe the XML schema structure.

type Redirect struct {
	Title string `xml:"title,attr" json:"title"`
}

type Page struct {
	Title          string   `xml:"title" json:"title"`
	CanonicalTitle string   `xml:"ctitle" json:"ctitle"`
	Redir          Redirect `xml:"redirect" json:"redirect"`
	Text           string   `xml:"revision>text" json:"text"`
}

type WikidataPage struct {
	Title          string      `xml:"title" json:"title"`
	CanonicalTitle string      `xml:"ctitle" json:"ctitle"`
	Redir          Redirect    `xml:"redirect" json:"redirect"`
	Content        interface{} `json:"content"`
}

func CanonicalizeTitle(title string) string {
	can := strings.ToLower(title)
	can = strings.Replace(can, " ", "_", -1)
	can = url.QueryEscape(can)
	return can
}

func main() {
	version := flag.Bool("v", false, "prints current version and exits")
	extractCategories := flag.String("c", "", "only extract categories TSV(page, category")
	extractAuthorityData := flag.Bool("a", false, "only extract authority data (Normdaten)")
	decodeWikiData := flag.Bool("d", false, "decode the text key value")
	filter, _ := regexp.Compile("^file:.*|^talk:.*|^special:.*|^wikipedia:.*|^wiktionary:.*|^user:.*|^user_talk:.*")

	flag.Parse()

	if *extractCategories != "" && *extractAuthorityData {
		fmt.Println("It's either -a or -c")
		os.Exit(1)
	}

	if *version {
		fmt.Println(AppVersion)
		os.Exit(0)
	}

	if flag.NArg() < 1 {
		fmt.Println("Usage: wptojson WIKIPEDIA-XML-DUMP")
		os.Exit(1)
	}
	inputFile := flag.Args()[0]

	xmlFile, err := os.Open(inputFile)
	if err != nil {
		fmt.Println("Error opening file:", err)
		return
	}
	defer xmlFile.Close()

	decoder := xml.NewDecoder(xmlFile)
	var inElement string
	categoryPattern := regexp.MustCompile(`\[\[` + *extractCategories + `:([^\[]+)\]\]`)
	authorityDataPattern := regexp.MustCompile(`(?mi){{Normdaten[^}]*}}`)

	// for wikidata
	var container interface{}

	for {
		// Read tokens from the XML document in a stream.
		t, _ := decoder.Token()
		if t == nil {
			break
		}
		// Inspect the type of the token just read.
		switch se := t.(type) {
		case xml.StartElement:
			// If we just read a StartElement token
			inElement = se.Name.Local
			// ...and its name is "page"
			if inElement == "page" {
				var p Page
				// decode a whole chunk of following XML into the
				// variable p which is a Page (se above)
				decoder.DecodeElement(&p, &se)

				// Do some stuff with the page.
				p.CanonicalTitle = CanonicalizeTitle(p.Title)
				m := filter.MatchString(p.CanonicalTitle)
				if !m && p.Redir.Title == "" {
					if *extractCategories != "" {
						result := categoryPattern.FindAllStringSubmatch(p.Text, -1)
						for _, value := range result {
							// replace anything after a |
							category := strings.TrimSpace(value[1])
							firstIndex := strings.Index(category, "|")
							if firstIndex != -1 {
								category = category[0:firstIndex]
							}
							fmt.Printf("%s\t%s\n", p.Title, category)
						}
					} else if *extractAuthorityData {
						result := authorityDataPattern.FindString(p.Text)
						if result != "" {
							// https://cdn.mediacru.sh/JsdjtGoLZBcR.png
							result = strings.Replace(result, "\t", "", -1)
							fmt.Printf("%s\t%s\n", p.Title, result)
						}
					} else if *decodeWikiData {
						json.Unmarshal([]byte(p.Text), &container)
						parsed := WikidataPage{Title: p.Title,
							CanonicalTitle: p.CanonicalTitle,
							Content:        container,
							Redir:          p.Redir}
						b, err := json.Marshal(parsed)
						if err != nil {
							os.Exit(2)
						}
						fmt.Println(string(b))
					} else {
						b, err := json.Marshal(p)
						if err != nil {
							os.Exit(2)
						}
						fmt.Println(string(b))
					}
				}
			}
		default:
		}
	}
}
<|endoftext|>


<commit_before>package logger

import (
	"fmt"
	"os"
	"path"
	"strings"
	"sync"
	"syscall"

	keybase1 "github.com/keybase/client/protocol/go"
	logging "github.com/op/go-logging"
	"golang.org/x/net/context"
)

const (
	fancyFormat   = "%{color}%{time:15:04:05.000000} ▶ [%{level:.4s} %{module} %{shortfile}] %{id:03x}%{color:reset} %{message}"
	plainFormat   = "[%{level:.4s}] %{id:03x} %{message}"
	fileFormat    = "%{time:15:04:05.000000} ▶ [%{level:.4s} %{module} %{shortfile}] %{id:03x} %{message}"
	defaultFormat = "%{color}%{message}%{color:reset}"
)

const permDir os.FileMode = 0700

var initLoggingBackendOnce sync.Once
var logRotateMutex sync.Mutex

// CtxStandardLoggerKey is a type defining context keys used by the
// Standard logger.
type CtxStandardLoggerKey int

const (
	// CtxLogTags defines a context key that can hold a slice of context
	// keys, the value of which should be logged by a Standard logger if
	// one of those keys is seen in a context during a log call.
	CtxLogTagsKey CtxStandardLoggerKey = iota
)

type CtxLogTags map[interface{}]string

// NewContext returns a new Context that carries adds the given log
// tag mappings (context key -> display string).
func NewContextWithLogTags(
	ctx context.Context, logTagsToAdd CtxLogTags) context.Context {
	currTags, ok := LogTagsFromContext(ctx)
	if !ok {
		currTags = make(CtxLogTags)
	}
	for key, tag := range logTagsToAdd {
		currTags[key] = tag
	}
	return context.WithValue(ctx, CtxLogTagsKey, currTags)
}

// LogTagsFromContext returns the log tags being passed along with the
// given context.
func LogTagsFromContext(ctx context.Context) (CtxLogTags, bool) {
	logTags, ok := ctx.Value(CtxLogTagsKey).(CtxLogTags)
	return logTags, ok
}

type ExternalLogger interface {
	Log(level keybase1.LogLevel, format string, args []interface{})
}

type Standard struct {
	internal       *logging.Logger
	filename       string
	configureMutex sync.Mutex
	module         string

	// External loggers are a hack to allow the calls to G.Log.* in the daemon
	// to be forwarded to the client. Loggers are registered here with
	// AddExternalLogger when connections are started, and every log that's
	// done gets replayed for each external logger registered at the time. That
	// will cause some duplication when multiple clients are connected, but
	// it's a hack. Ideally in the future every function that needs to log will
	// have a context.
	//
	// Because external loggers are intended to be talking over the RPC
	// connection, we don't want to push all the voluminous debug logs unless
	// the client actually wants them. Thus we keep a log level here, and we
	// drop any logs that are below that level. Clients will set this over RPC
	// when they connect.
	externalLoggers      map[uint64]ExternalLogger
	externalLoggersCount uint64
	externalLogLevel     keybase1.LogLevel
	externalLoggersMutex sync.RWMutex
}

// New creates a new Standard logger for module.
func New(module string) *Standard {
	return NewWithCallDepth(module, 0)
}

// NewWithCallDepth creates a new Standard logger for module, and when
// printing file names and line numbers, it goes extraCallDepth up the
// stack from where logger was invoked.
func NewWithCallDepth(module string, extraCallDepth int) *Standard {
	log := logging.MustGetLogger(module)
	log.ExtraCalldepth = 1 + extraCallDepth
	ret := &Standard{
		internal:             log,
		module:               module,
		externalLoggers:      make(map[uint64]ExternalLogger),
		externalLoggersCount: 0,
		externalLogLevel:     keybase1.LogLevel_INFO,
	}
	ret.initLogging()
	return ret
}

func (log *Standard) initLogging() {
	// Logging is always done to stderr. It's the responsibility of the
	// launcher (like launchd on OSX, or the autoforking code) to set up stderr
	// to point to the appropriate log file.
	initLoggingBackendOnce.Do(func() {
		logBackend := logging.NewLogBackend(os.Stderr, "", 0)
		logging.SetBackend(logBackend)
	})
	logging.SetLevel(logging.INFO, log.module)
}

func (log *Standard) prepareString(
	ctx context.Context, fmts string) string {
	if ctx == nil {
		return fmts
	}
	logTags, ok := LogTagsFromContext(ctx)
	if !ok || len(logTags) == 0 {
		return fmts
	}
	var tags []string
	for key, tag := range logTags {
		if v := ctx.Value(key); v != nil {
			tags = append(tags, fmt.Sprintf("%s=%s", tag, v))
		}
	}
	return fmts + " [tags:" + strings.Join(tags, ",") + "]"
}

func (log *Standard) Debug(fmt string, arg ...interface{}) {
	log.internal.Debug(fmt, arg...)
	log.logToExternalLoggers(keybase1.LogLevel_DEBUG, fmt, arg)
}

func (log *Standard) CDebugf(ctx context.Context, fmt string,
	arg ...interface{}) {
	if log.internal.IsEnabledFor(logging.DEBUG) {
		log.Debug(log.prepareString(ctx, fmt), arg...)
	}
}

func (log *Standard) Info(fmt string, arg ...interface{}) {
	log.internal.Info(fmt, arg...)
	log.logToExternalLoggers(keybase1.LogLevel_INFO, fmt, arg)
}

func (log *Standard) CInfof(ctx context.Context, fmt string,
	arg ...interface{}) {
	if log.internal.IsEnabledFor(logging.INFO) {
		log.Info(log.prepareString(ctx, fmt), arg...)
	}
}

func (log *Standard) Notice(fmt string, arg ...interface{}) {
	log.internal.Notice(fmt, arg...)
	log.logToExternalLoggers(keybase1.LogLevel_NOTICE, fmt, arg)
}

func (log *Standard) CNoticef(ctx context.Context, fmt string,
	arg ...interface{}) {
	if log.internal.IsEnabledFor(logging.NOTICE) {
		log.Notice(log.prepareString(ctx, fmt), arg...)
	}
}

func (log *Standard) Warning(fmt string, arg ...interface{}) {
	log.internal.Warning(fmt, arg...)
	log.logToExternalLoggers(keybase1.LogLevel_WARN, fmt, arg)
}

func (log *Standard) CWarningf(ctx context.Context, fmt string,
	arg ...interface{}) {
	if log.internal.IsEnabledFor(logging.WARNING) {
		log.Warning(log.prepareString(ctx, fmt), arg...)
	}
}

func (log *Standard) Error(fmt string, arg ...interface{}) {
	log.internal.Error(fmt, arg...)
	log.logToExternalLoggers(keybase1.LogLevel_ERROR, fmt, arg)
}

func (log *Standard) Errorf(fmt string, arg ...interface{}) {
	log.Error(fmt, arg...)
}

func (log *Standard) CErrorf(ctx context.Context, fmt string,
	arg ...interface{}) {
	if log.internal.IsEnabledFor(logging.ERROR) {
		log.Error(log.prepareString(ctx, fmt), arg...)
	}
}

func (log *Standard) Critical(fmt string, arg ...interface{}) {
	log.internal.Critical(fmt, arg...)
	log.logToExternalLoggers(keybase1.LogLevel_CRITICAL, fmt, arg)
}

func (log *Standard) CCriticalf(ctx context.Context, fmt string,
	arg ...interface{}) {
	if log.internal.IsEnabledFor(logging.CRITICAL) {
		log.Critical(log.prepareString(ctx, fmt), arg...)
	}
}

func (log *Standard) Fatalf(fmt string, arg ...interface{}) {
	log.internal.Fatalf(fmt, arg...)
	log.logToExternalLoggers(keybase1.LogLevel_FATAL, fmt, arg)
}

func (log *Standard) CFatalf(ctx context.Context, fmt string,
	arg ...interface{}) {
	log.Fatalf(log.prepareString(ctx, fmt), arg...)
}

func (log *Standard) Profile(fmts string, arg ...interface{}) {
	log.Debug(fmts, arg...)
}

func (log *Standard) Configure(style string, debug bool, filename string) {
	log.configureMutex.Lock()
	defer log.configureMutex.Unlock()

	log.filename = filename

	var logfmt string
	if debug {
		logfmt = fancyFormat
	} else {
		logfmt = defaultFormat
	}

	// Override the format above if an explicit style was specified.
	switch style {
	case "default":
		logfmt = defaultFormat // Default
	case "plain":
		logfmt = plainFormat // Plain
	case "file":
		logfmt = fileFormat // Good for logging to files
	case "fancy":
		logfmt = fancyFormat // Fancy, good for terminals with color
	}

	if debug {
		logging.SetLevel(logging.DEBUG, log.module)
	}

	logging.SetFormatter(logging.MustStringFormatter(logfmt))
}

func (log *Standard) RotateLogFile() error {
	logRotateMutex.Lock()
	defer logRotateMutex.Unlock()
	log.internal.Info("Rotating log file; closing down old file")
	_, file, err := OpenLogFile(log.filename)
	if err != nil {
		return err
	}
	err = PickFirstError(
		syscall.Close(1),
		syscall.Close(2),
		syscall.Dup2(int(file.Fd()), 1),
		syscall.Dup2(int(file.Fd()), 2),
		file.Close(),
	)
	if err != nil {
		log.internal.Warning("Couldn't rotate file: %v", err)
	}
	log.internal.Info("Rotated log file; opening up new file")
	return nil
}

func OpenLogFile(filename string) (name string, file *os.File, err error) {
	name = filename
	if err = MakeParentDirs(name); err != nil {
		return
	}
	file, err = os.OpenFile(name, (os.O_APPEND | os.O_WRONLY | os.O_CREATE), 0600)
	if err != nil {
		return
	}
	return
}

func FileExists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}

func MakeParentDirs(filename string) error {
	dir, _ := path.Split(filename)
	exists, err := FileExists(dir)
	if err != nil {
		return err
	}

	if !exists {
		err = os.MkdirAll(dir, permDir)
		if err != nil {
			return err
		}
	}
	return nil
}

func PickFirstError(errors ...error) error {
	for _, e := range errors {
		if e != nil {
			return e
		}
	}
	return nil
}

func (log *Standard) AddExternalLogger(externalLogger ExternalLogger) uint64 {
	log.externalLoggersMutex.Lock()
	defer log.externalLoggersMutex.Unlock()

	handle := log.externalLoggersCount
	log.externalLoggersCount++
	log.externalLoggers[handle] = externalLogger
	return handle
}

func (log *Standard) RemoveExternalLogger(handle uint64) {
	log.externalLoggersMutex.Lock()
	defer log.externalLoggersMutex.Unlock()

	delete(log.externalLoggers, handle)
}

func (log *Standard) logToExternalLoggers(level keybase1.LogLevel, format string, args ...interface{}) {
	log.externalLoggersMutex.RLock()
	defer log.externalLoggersMutex.RUnlock()

	// Short circuit logs that are more verbose than the current external log
	// level.
	if level < log.externalLogLevel {
		return
	}

	for _, externalLogger := range log.externalLoggers {
		go externalLogger.Log(level, format, args)
	}
}

func (log *Standard) SetLogLevel(level keybase1.LogLevel) {
	log.externalLoggersMutex.Lock()
	defer log.externalLoggersMutex.Unlock()

	log.externalLogLevel = level
}
<commit_msg>dropping a duplicate block comment<commit_after>package logger

import (
	"fmt"
	"os"
	"path"
	"strings"
	"sync"
	"syscall"

	keybase1 "github.com/keybase/client/protocol/go"
	logging "github.com/op/go-logging"
	"golang.org/x/net/context"
)

const (
	fancyFormat   = "%{color}%{time:15:04:05.000000} ▶ [%{level:.4s} %{module} %{shortfile}] %{id:03x}%{color:reset} %{message}"
	plainFormat   = "[%{level:.4s}] %{id:03x} %{message}"
	fileFormat    = "%{time:15:04:05.000000} ▶ [%{level:.4s} %{module} %{shortfile}] %{id:03x} %{message}"
	defaultFormat = "%{color}%{message}%{color:reset}"
)

const permDir os.FileMode = 0700

var initLoggingBackendOnce sync.Once
var logRotateMutex sync.Mutex

// CtxStandardLoggerKey is a type defining context keys used by the
// Standard logger.
type CtxStandardLoggerKey int

const (
	// CtxLogTags defines a context key that can hold a slice of context
	// keys, the value of which should be logged by a Standard logger if
	// one of those keys is seen in a context during a log call.
	CtxLogTagsKey CtxStandardLoggerKey = iota
)

type CtxLogTags map[interface{}]string

// NewContext returns a new Context that carries adds the given log
// tag mappings (context key -> display string).
func NewContextWithLogTags(
	ctx context.Context, logTagsToAdd CtxLogTags) context.Context {
	currTags, ok := LogTagsFromContext(ctx)
	if !ok {
		currTags = make(CtxLogTags)
	}
	for key, tag := range logTagsToAdd {
		currTags[key] = tag
	}
	return context.WithValue(ctx, CtxLogTagsKey, currTags)
}

// LogTagsFromContext returns the log tags being passed along with the
// given context.
func LogTagsFromContext(ctx context.Context) (CtxLogTags, bool) {
	logTags, ok := ctx.Value(CtxLogTagsKey).(CtxLogTags)
	return logTags, ok
}

type ExternalLogger interface {
	Log(level keybase1.LogLevel, format string, args []interface{})
}

type Standard struct {
	internal       *logging.Logger
	filename       string
	configureMutex sync.Mutex
	module         string

	externalLoggers      map[uint64]ExternalLogger
	externalLoggersCount uint64
	externalLogLevel     keybase1.LogLevel
	externalLoggersMutex sync.RWMutex
}

// New creates a new Standard logger for module.
func New(module string) *Standard {
	return NewWithCallDepth(module, 0)
}

// NewWithCallDepth creates a new Standard logger for module, and when
// printing file names and line numbers, it goes extraCallDepth up the
// stack from where logger was invoked.
func NewWithCallDepth(module string, extraCallDepth int) *Standard {
	log := logging.MustGetLogger(module)
	log.ExtraCalldepth = 1 + extraCallDepth
	ret := &Standard{
		internal:             log,
		module:               module,
		externalLoggers:      make(map[uint64]ExternalLogger),
		externalLoggersCount: 0,
		externalLogLevel:     keybase1.LogLevel_INFO,
	}
	ret.initLogging()
	return ret
}

func (log *Standard) initLogging() {
	// Logging is always done to stderr. It's the responsibility of the
	// launcher (like launchd on OSX, or the autoforking code) to set up stderr
	// to point to the appropriate log file.
	initLoggingBackendOnce.Do(func() {
		logBackend := logging.NewLogBackend(os.Stderr, "", 0)
		logging.SetBackend(logBackend)
	})
	logging.SetLevel(logging.INFO, log.module)
}

func (log *Standard) prepareString(
	ctx context.Context, fmts string) string {
	if ctx == nil {
		return fmts
	}
	logTags, ok := LogTagsFromContext(ctx)
	if !ok || len(logTags) == 0 {
		return fmts
	}
	var tags []string
	for key, tag := range logTags {
		if v := ctx.Value(key); v != nil {
			tags = append(tags, fmt.Sprintf("%s=%s", tag, v))
		}
	}
	return fmts + " [tags:" + strings.Join(tags, ",") + "]"
}

func (log *Standard) Debug(fmt string, arg ...interface{}) {
	log.internal.Debug(fmt, arg...)
	log.logToExternalLoggers(keybase1.LogLevel_DEBUG, fmt, arg)
}

func (log *Standard) CDebugf(ctx context.Context, fmt string,
	arg ...interface{}) {
	if log.internal.IsEnabledFor(logging.DEBUG) {
		log.Debug(log.prepareString(ctx, fmt), arg...)
	}
}

func (log *Standard) Info(fmt string, arg ...interface{}) {
	log.internal.Info(fmt, arg...)
	log.logToExternalLoggers(keybase1.LogLevel_INFO, fmt, arg)
}

func (log *Standard) CInfof(ctx context.Context, fmt string,
	arg ...interface{}) {
	if log.internal.IsEnabledFor(logging.INFO) {
		log.Info(log.prepareString(ctx, fmt), arg...)
	}
}

func (log *Standard) Notice(fmt string, arg ...interface{}) {
	log.internal.Notice(fmt, arg...)
	log.logToExternalLoggers(keybase1.LogLevel_NOTICE, fmt, arg)
}

func (log *Standard) CNoticef(ctx context.Context, fmt string,
	arg ...interface{}) {
	if log.internal.IsEnabledFor(logging.NOTICE) {
		log.Notice(log.prepareString(ctx, fmt), arg...)
	}
}

func (log *Standard) Warning(fmt string, arg ...interface{}) {
	log.internal.Warning(fmt, arg...)
	log.logToExternalLoggers(keybase1.LogLevel_WARN, fmt, arg)
}

func (log *Standard) CWarningf(ctx context.Context, fmt string,
	arg ...interface{}) {
	if log.internal.IsEnabledFor(logging.WARNING) {
		log.Warning(log.prepareString(ctx, fmt), arg...)
	}
}

func (log *Standard) Error(fmt string, arg ...interface{}) {
	log.internal.Error(fmt, arg...)
	log.logToExternalLoggers(keybase1.LogLevel_ERROR, fmt, arg)
}

func (log *Standard) Errorf(fmt string, arg ...interface{}) {
	log.Error(fmt, arg...)
}

func (log *Standard) CErrorf(ctx context.Context, fmt string,
	arg ...interface{}) {
	if log.internal.IsEnabledFor(logging.ERROR) {
		log.Error(log.prepareString(ctx, fmt), arg...)
	}
}

func (log *Standard) Critical(fmt string, arg ...interface{}) {
	log.internal.Critical(fmt, arg...)
	log.logToExternalLoggers(keybase1.LogLevel_CRITICAL, fmt, arg)
}

func (log *Standard) CCriticalf(ctx context.Context, fmt string,
	arg ...interface{}) {
	if log.internal.IsEnabledFor(logging.CRITICAL) {
		log.Critical(log.prepareString(ctx, fmt), arg...)
	}
}

func (log *Standard) Fatalf(fmt string, arg ...interface{}) {
	log.internal.Fatalf(fmt, arg...)
	log.logToExternalLoggers(keybase1.LogLevel_FATAL, fmt, arg)
}

func (log *Standard) CFatalf(ctx context.Context, fmt string,
	arg ...interface{}) {
	log.Fatalf(log.prepareString(ctx, fmt), arg...)
}

func (log *Standard) Profile(fmts string, arg ...interface{}) {
	log.Debug(fmts, arg...)
}

func (log *Standard) Configure(style string, debug bool, filename string) {
	log.configureMutex.Lock()
	defer log.configureMutex.Unlock()

	log.filename = filename

	var logfmt string
	if debug {
		logfmt = fancyFormat
	} else {
		logfmt = defaultFormat
	}

	// Override the format above if an explicit style was specified.
	switch style {
	case "default":
		logfmt = defaultFormat // Default
	case "plain":
		logfmt = plainFormat // Plain
	case "file":
		logfmt = fileFormat // Good for logging to files
	case "fancy":
		logfmt = fancyFormat // Fancy, good for terminals with color
	}

	if debug {
		logging.SetLevel(logging.DEBUG, log.module)
	}

	logging.SetFormatter(logging.MustStringFormatter(logfmt))
}

func (log *Standard) RotateLogFile() error {
	logRotateMutex.Lock()
	defer logRotateMutex.Unlock()
	log.internal.Info("Rotating log file; closing down old file")
	_, file, err := OpenLogFile(log.filename)
	if err != nil {
		return err
	}
	err = PickFirstError(
		syscall.Close(1),
		syscall.Close(2),
		syscall.Dup2(int(file.Fd()), 1),
		syscall.Dup2(int(file.Fd()), 2),
		file.Close(),
	)
	if err != nil {
		log.internal.Warning("Couldn't rotate file: %v", err)
	}
	log.internal.Info("Rotated log file; opening up new file")
	return nil
}

func OpenLogFile(filename string) (name string, file *os.File, err error) {
	name = filename
	if err = MakeParentDirs(name); err != nil {
		return
	}
	file, err = os.OpenFile(name, (os.O_APPEND | os.O_WRONLY | os.O_CREATE), 0600)
	if err != nil {
		return
	}
	return
}

func FileExists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}

func MakeParentDirs(filename string) error {
	dir, _ := path.Split(filename)
	exists, err := FileExists(dir)
	if err != nil {
		return err
	}

	if !exists {
		err = os.MkdirAll(dir, permDir)
		if err != nil {
			return err
		}
	}
	return nil
}

func PickFirstError(errors ...error) error {
	for _, e := range errors {
		if e != nil {
			return e
		}
	}
	return nil
}

func (log *Standard) AddExternalLogger(externalLogger ExternalLogger) uint64 {
	log.externalLoggersMutex.Lock()
	defer log.externalLoggersMutex.Unlock()

	handle := log.externalLoggersCount
	log.externalLoggersCount++
	log.externalLoggers[handle] = externalLogger
	return handle
}

func (log *Standard) RemoveExternalLogger(handle uint64) {
	log.externalLoggersMutex.Lock()
	defer log.externalLoggersMutex.Unlock()

	delete(log.externalLoggers, handle)
}

func (log *Standard) logToExternalLoggers(level keybase1.LogLevel, format string, args ...interface{}) {
	log.externalLoggersMutex.RLock()
	defer log.externalLoggersMutex.RUnlock()

	// Short circuit logs that are more verbose than the current external log
	// level.
	if level < log.externalLogLevel {
		return
	}

	for _, externalLogger := range log.externalLoggers {
		go externalLogger.Log(level, format, args)
	}
}

func (log *Standard) SetLogLevel(level keybase1.LogLevel) {
	log.externalLoggersMutex.Lock()
	defer log.externalLoggersMutex.Unlock()

	log.externalLogLevel = level
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Incomplete source tree on Android.

// +build !android

package ssa_test

// This file runs the SSA builder in sanity-checking mode on all
// packages beneath $GOROOT and prints some summary information.
//
// Run with "go test -cpu=8 to" set GOMAXPROCS.

import (
	"go/ast"
	"go/build"
	"go/token"
	"runtime"
	"testing"
	"time"

	"golang.org/x/tools/go/buildutil"
	"golang.org/x/tools/go/loader"
	"golang.org/x/tools/go/ssa"
	"golang.org/x/tools/go/ssa/ssautil"
	"golang.org/x/tools/internal/testenv"
)

// Skip the set of packages that transitively depend on
// cmd/internal/objfile, which uses vendoring,
// which go/loader does not yet support.
// TODO(adonovan): add support for vendoring and delete this.
var skip = map[string]bool{
	"cmd/addr2line":        true,
	"cmd/internal/objfile": true,
	"cmd/nm":               true,
	"cmd/objdump":          true,
	"cmd/pprof":            true,
}

func bytesAllocated() uint64 {
	runtime.GC()
	var stats runtime.MemStats
	runtime.ReadMemStats(&stats)
	return stats.Alloc
}

func TestStdlib(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping in short mode; too slow (https://golang.org/issue/14113)")
	}
	testenv.NeedsTool(t, "go")

	// Load, parse and type-check the program.
	t0 := time.Now()
	alloc0 := bytesAllocated()

	// Load, parse and type-check the program.
	ctxt := build.Default // copy
	ctxt.GOPATH = ""      // disable GOPATH
	conf := loader.Config{Build: &ctxt}
	for _, path := range buildutil.AllPackages(conf.Build) {
		if skip[path] {
			continue
		}
		conf.ImportWithTests(path)
	}

	iprog, err := conf.Load()
	if err != nil {
		t.Fatalf("Load failed: %v", err)
	}

	t1 := time.Now()
	alloc1 := bytesAllocated()

	// Create SSA packages.
	var mode ssa.BuilderMode
	// Comment out these lines during benchmarking.  Approx SSA build costs are noted.
	mode |= ssa.SanityCheckFunctions // + 2% space, + 4% time
	mode |= ssa.GlobalDebug          // +30% space, +18% time
	prog := ssautil.CreateProgram(iprog, mode)

	t2 := time.Now()

	// Build SSA.
	prog.Build()

	t3 := time.Now()
	alloc3 := bytesAllocated()

	numPkgs := len(prog.AllPackages())
	if want := 140; numPkgs < want {
		t.Errorf("Loaded only %d packages, want at least %d", numPkgs, want)
	}

	// Keep iprog reachable until after we've measured memory usage.
	if len(iprog.AllPackages) == 0 {
		panic("unreachable")
	}

	allFuncs := ssautil.AllFunctions(prog)

	// Check that all non-synthetic functions have distinct names.
	// Synthetic wrappers for exported methods should be distinct too,
	// except for unexported ones (explained at (*Function).RelString).
	byName := make(map[string]*ssa.Function)
	for fn := range allFuncs {
		if fn.Synthetic == "" || ast.IsExported(fn.Name()) {
			str := fn.String()
			prev := byName[str]
			byName[str] = fn
			if prev != nil {
				t.Errorf("%s: duplicate function named %s",
					prog.Fset.Position(fn.Pos()), str)
				t.Errorf("%s:   (previously defined here)",
					prog.Fset.Position(prev.Pos()))
			}
		}
	}

	// Dump some statistics.
	var numInstrs int
	for fn := range allFuncs {
		for _, b := range fn.Blocks {
			numInstrs += len(b.Instrs)
		}
	}

	// determine line count
	var lineCount int
	prog.Fset.Iterate(func(f *token.File) bool {
		lineCount += f.LineCount()
		return true
	})

	// NB: when benchmarking, don't forget to clear the debug +
	// sanity builder flags for better performance.

	t.Log("GOMAXPROCS:           ", runtime.GOMAXPROCS(0))
	t.Log("#Source lines:        ", lineCount)
	t.Log("Load/parse/typecheck: ", t1.Sub(t0))
	t.Log("SSA create:           ", t2.Sub(t1))
	t.Log("SSA build:            ", t3.Sub(t2))

	// SSA stats:
	t.Log("#Packages:            ", numPkgs)
	t.Log("#Functions:           ", len(allFuncs))
	t.Log("#Instructions:        ", numInstrs)
	t.Log("#MB AST+types:        ", int64(alloc1-alloc0)/1e6)
	t.Log("#MB SSA:              ", int64(alloc3-alloc1)/1e6)
}
<commit_msg>go/ssa: remove workaround for lack of vendoring in go/loader<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Incomplete source tree on Android.

// +build !android

package ssa_test

// This file runs the SSA builder in sanity-checking mode on all
// packages beneath $GOROOT and prints some summary information.
//
// Run with "go test -cpu=8 to" set GOMAXPROCS.

import (
	"go/ast"
	"go/build"
	"go/token"
	"runtime"
	"testing"
	"time"

	"golang.org/x/tools/go/buildutil"
	"golang.org/x/tools/go/loader"
	"golang.org/x/tools/go/ssa"
	"golang.org/x/tools/go/ssa/ssautil"
	"golang.org/x/tools/internal/testenv"
)

func bytesAllocated() uint64 {
	runtime.GC()
	var stats runtime.MemStats
	runtime.ReadMemStats(&stats)
	return stats.Alloc
}

func TestStdlib(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping in short mode; too slow (https://golang.org/issue/14113)")
	}
	testenv.NeedsTool(t, "go")

	// Load, parse and type-check the program.
	t0 := time.Now()
	alloc0 := bytesAllocated()

	// Load, parse and type-check the program.
	ctxt := build.Default // copy
	ctxt.GOPATH = ""      // disable GOPATH
	conf := loader.Config{Build: &ctxt}
	for _, path := range buildutil.AllPackages(conf.Build) {
		conf.ImportWithTests(path)
	}

	iprog, err := conf.Load()
	if err != nil {
		t.Fatalf("Load failed: %v", err)
	}

	t1 := time.Now()
	alloc1 := bytesAllocated()

	// Create SSA packages.
	var mode ssa.BuilderMode
	// Comment out these lines during benchmarking.  Approx SSA build costs are noted.
	mode |= ssa.SanityCheckFunctions // + 2% space, + 4% time
	mode |= ssa.GlobalDebug          // +30% space, +18% time
	prog := ssautil.CreateProgram(iprog, mode)

	t2 := time.Now()

	// Build SSA.
	prog.Build()

	t3 := time.Now()
	alloc3 := bytesAllocated()

	numPkgs := len(prog.AllPackages())
	if want := 140; numPkgs < want {
		t.Errorf("Loaded only %d packages, want at least %d", numPkgs, want)
	}

	// Keep iprog reachable until after we've measured memory usage.
	if len(iprog.AllPackages) == 0 {
		panic("unreachable")
	}

	allFuncs := ssautil.AllFunctions(prog)

	// Check that all non-synthetic functions have distinct names.
	// Synthetic wrappers for exported methods should be distinct too,
	// except for unexported ones (explained at (*Function).RelString).
	byName := make(map[string]*ssa.Function)
	for fn := range allFuncs {
		if fn.Synthetic == "" || ast.IsExported(fn.Name()) {
			str := fn.String()
			prev := byName[str]
			byName[str] = fn
			if prev != nil {
				t.Errorf("%s: duplicate function named %s",
					prog.Fset.Position(fn.Pos()), str)
				t.Errorf("%s:   (previously defined here)",
					prog.Fset.Position(prev.Pos()))
			}
		}
	}

	// Dump some statistics.
	var numInstrs int
	for fn := range allFuncs {
		for _, b := range fn.Blocks {
			numInstrs += len(b.Instrs)
		}
	}

	// determine line count
	var lineCount int
	prog.Fset.Iterate(func(f *token.File) bool {
		lineCount += f.LineCount()
		return true
	})

	// NB: when benchmarking, don't forget to clear the debug +
	// sanity builder flags for better performance.

	t.Log("GOMAXPROCS:           ", runtime.GOMAXPROCS(0))
	t.Log("#Source lines:        ", lineCount)
	t.Log("Load/parse/typecheck: ", t1.Sub(t0))
	t.Log("SSA create:           ", t2.Sub(t1))
	t.Log("SSA build:            ", t3.Sub(t2))

	// SSA stats:
	t.Log("#Packages:            ", numPkgs)
	t.Log("#Functions:           ", len(allFuncs))
	t.Log("#Instructions:        ", numInstrs)
	t.Log("#MB AST+types:        ", int64(alloc1-alloc0)/1e6)
	t.Log("#MB SSA:              ", int64(alloc3-alloc1)/1e6)
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file implements Selections.

package types

import (
	"bytes"
	"fmt"
)

// SelectionKind describes the kind of a selector expression x.f.
type SelectionKind int

const (
	FieldVal   SelectionKind = iota // x.f is a struct field selector
	MethodVal                       // x.f is a method selector
	MethodExpr                      // x.f is a method expression
	PackageObj                      // x.f is a qualified identifier
)

// A Selection describes a selector expression x.f.
// For the declarations:
//
// 	type T struct{ x int; E }
// 	type E struct{}
// 	func (e E) m() {}
// 	var p *T
//
// the following relations exist:
//
//	Selector    Kind          Recv    Obj    Type               Index     Indirect
//
//      p.x         FieldVal      T       x      int                {0}       true
//      p.m         MethodVal     *T      m      func (e *T) m()    {1, 0}    true
//      T.m         MethodExpr    T       m      func m(_ T)        {1, 0}    false
//      math.Pi     PackageObj    nil     Pi     untyped numeric    nil       false
//
type Selection struct {
	kind     SelectionKind
	recv     Type   // type of x, nil if kind == PackageObj
	obj      Object // object denoted by x.f
	index    []int  // path from x to x.f, nil if kind == PackageObj
	indirect bool   // set if there was any pointer indirection on the path, false if kind == PackageObj
}

// Kind returns the selection kind.
func (s *Selection) Kind() SelectionKind { return s.kind }

// Recv returns the type of x in x.f.
// The result is nil if x.f is a qualified identifier (PackageObj).
func (s *Selection) Recv() Type { return s.recv }

// Obj returns the object denoted by x.f.
// The following object types may appear:
//
//	Kind          Object
//
//	FieldVal      *Var                          field
//	MethodVal     *Func                         method
//	MethodExpr    *Func                         method
//	PackageObj    *Const, *Type, *Var, *Func    imported const, type, var, or func
//
func (s *Selection) Obj() Object { return s.obj }

// Type returns the type of x.f, which may be different from the type of f.
// See Selection for more information.
func (s *Selection) Type() Type {
	switch s.kind {
	case MethodVal:
		// The type of x.f is a method with its receiver type set
		// to the type of x.
		sig := *s.obj.(*Func).typ.(*Signature)
		recv := *sig.recv
		recv.typ = s.recv
		sig.recv = &recv
		return &sig

	case MethodExpr:
		// The type of x.f is a function (without receiver)
		// and an additional first argument with the same type as x.
		// TODO(gri) Similar code is already in call.go - factor!
		sig := *s.obj.(*Func).typ.(*Signature)
		arg0 := *sig.recv
		arg0.typ = s.recv
		var params []*Var
		if sig.params != nil {
			params = sig.params.vars
		}
		sig.params = NewTuple(append([]*Var{&arg0}, params...)...)
		return &sig
	}

	// In all other cases, the type of x.f is the type of x.
	return s.obj.Type()
}

// Index describes the path from x to f in x.f.
// The result is nil if x.f is a qualified identifier (PackageObj).
//
// The last index entry is the field or method index of the type declaring f;
// either:
//
//	1) the list of declared methods of a named type; or
//	2) the list of methods of an interface type; or
//	3) the list of fields of a struct type.
//
// The earlier index entries are the indices of the embedded fields implicitly
// traversed to get from (the type of) x to f, starting at embedding depth 0.
func (s *Selection) Index() []int { return s.index }

// Indirect reports whether any pointer indirection was required to get from
// x to f in x.f.
// The result is false if x.f is a qualified identifier (PackageObj).
func (s *Selection) Indirect() bool { return s.indirect }

func (s *Selection) String() string {
	var k string
	switch s.kind {
	case FieldVal:
		k = "field"
	case MethodVal:
		k = "method"
	case MethodExpr:
		k = "method expr"
	case PackageObj:
		return fmt.Sprintf("qualified ident %s", s.obj)
	default:
		unreachable()
	}
	var buf bytes.Buffer
	fmt.Fprintf(&buf, "%s (%s) %s", k, s.Recv(), s.obj.Name())
	writeSignature(&buf, nil, s.Type().(*Signature))
	return buf.String()
}
<commit_msg>go.tools/go/types: SelectionString: print method with selective package-qualification.<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file implements Selections.

package types

import (
	"bytes"
	"fmt"
)

// SelectionKind describes the kind of a selector expression x.f.
type SelectionKind int

const (
	FieldVal   SelectionKind = iota // x.f is a struct field selector
	MethodVal                       // x.f is a method selector
	MethodExpr                      // x.f is a method expression
	PackageObj                      // x.f is a qualified identifier
)

// A Selection describes a selector expression x.f.
// For the declarations:
//
// 	type T struct{ x int; E }
// 	type E struct{}
// 	func (e E) m() {}
// 	var p *T
//
// the following relations exist:
//
//	Selector    Kind          Recv    Obj    Type               Index     Indirect
//
//      p.x         FieldVal      T       x      int                {0}       true
//      p.m         MethodVal     *T      m      func (e *T) m()    {1, 0}    true
//      T.m         MethodExpr    T       m      func m(_ T)        {1, 0}    false
//      math.Pi     PackageObj    nil     Pi     untyped numeric    nil       false
//
type Selection struct {
	kind     SelectionKind
	recv     Type   // type of x, nil if kind == PackageObj
	obj      Object // object denoted by x.f
	index    []int  // path from x to x.f, nil if kind == PackageObj
	indirect bool   // set if there was any pointer indirection on the path, false if kind == PackageObj
}

// Kind returns the selection kind.
func (s *Selection) Kind() SelectionKind { return s.kind }

// Recv returns the type of x in x.f.
// The result is nil if x.f is a qualified identifier (PackageObj).
func (s *Selection) Recv() Type { return s.recv }

// Obj returns the object denoted by x.f.
// The following object types may appear:
//
//	Kind          Object
//
//	FieldVal      *Var                          field
//	MethodVal     *Func                         method
//	MethodExpr    *Func                         method
//	PackageObj    *Const, *Type, *Var, *Func    imported const, type, var, or func
//
func (s *Selection) Obj() Object { return s.obj }

// Type returns the type of x.f, which may be different from the type of f.
// See Selection for more information.
func (s *Selection) Type() Type {
	switch s.kind {
	case MethodVal:
		// The type of x.f is a method with its receiver type set
		// to the type of x.
		sig := *s.obj.(*Func).typ.(*Signature)
		recv := *sig.recv
		recv.typ = s.recv
		sig.recv = &recv
		return &sig

	case MethodExpr:
		// The type of x.f is a function (without receiver)
		// and an additional first argument with the same type as x.
		// TODO(gri) Similar code is already in call.go - factor!
		sig := *s.obj.(*Func).typ.(*Signature)
		arg0 := *sig.recv
		arg0.typ = s.recv
		var params []*Var
		if sig.params != nil {
			params = sig.params.vars
		}
		sig.params = NewTuple(append([]*Var{&arg0}, params...)...)
		return &sig
	}

	// In all other cases, the type of x.f is the type of x.
	return s.obj.Type()
}

// Index describes the path from x to f in x.f.
// The result is nil if x.f is a qualified identifier (PackageObj).
//
// The last index entry is the field or method index of the type declaring f;
// either:
//
//	1) the list of declared methods of a named type; or
//	2) the list of methods of an interface type; or
//	3) the list of fields of a struct type.
//
// The earlier index entries are the indices of the embedded fields implicitly
// traversed to get from (the type of) x to f, starting at embedding depth 0.
func (s *Selection) Index() []int { return s.index }

// Indirect reports whether any pointer indirection was required to get from
// x to f in x.f.
// The result is false if x.f is a qualified identifier (PackageObj).
func (s *Selection) Indirect() bool { return s.indirect }

func (s *Selection) String() string { return SelectionString(nil, s) }

// SelectionString returns the string form of s.
// Type names are printed package-qualified
// only if they do not belong to this package.
//
func SelectionString(this *Package, s *Selection) string {
	var k string
	switch s.kind {
	case FieldVal:
		k = "field "
	case MethodVal:
		k = "method "
	case MethodExpr:
		k = "method expr "
	case PackageObj:
		return fmt.Sprintf("qualified ident %s", s.obj)
	default:
		unreachable()
	}
	var buf bytes.Buffer
	buf.WriteString(k)
	buf.WriteByte('(')
	writeType(&buf, this, s.Recv())
	fmt.Fprintf(&buf, ") %s", s.obj.Name())
	writeSignature(&buf, this, s.Type().(*Signature))
	return buf.String()
}
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime_test

import (
	. "runtime"
	"testing"
	"unsafe"
)

// Simple serial sanity test for parallelfor.
func TestParFor(t *testing.T) {
	const P = 1
	const N = 20
	data := make([]uint64, N)
	for i := uint64(0); i < N; i++ {
		data[i] = i
	}
	desc := NewParFor(P)
	ParForSetup(desc, P, N, nil, true, func(desc *ParFor, i uint32) {
		data[i] = data[i]*data[i] + 1
	})
	ParForDo(desc)
	for i := uint64(0); i < N; i++ {
		if data[i] != i*i+1 {
			t.Fatalf("Wrong element %d: %d", i, data[i])
		}
	}
}

// Test that nonblocking parallelfor does not block.
func TestParFor2(t *testing.T) {
	const P = 7
	const N = 1003
	data := make([]uint64, N)
	for i := uint64(0); i < N; i++ {
		data[i] = i
	}
	desc := NewParFor(P)
	ParForSetup(desc, P, N, (*byte)(unsafe.Pointer(&data)), false, func(desc *ParFor, i uint32) {
		d := *(*[]uint64)(unsafe.Pointer(desc.Ctx))
		d[i] = d[i]*d[i] + 1
	})
	for p := 0; p < P; p++ {
		ParForDo(desc)
	}
	for i := uint64(0); i < N; i++ {
		if data[i] != i*i+1 {
			t.Fatalf("Wrong element %d: %d", i, data[i])
		}
	}
}

// Test that iterations are properly distributed.
func TestParForSetup(t *testing.T) {
	const P = 11
	const N = 101
	desc := NewParFor(P)
	for n := uint32(0); n < N; n++ {
		for p := uint32(1); p <= P; p++ {
			ParForSetup(desc, p, n, nil, true, func(desc *ParFor, i uint32) {})
			sum := uint32(0)
			size0 := uint32(0)
			end0 := uint32(0)
			for i := uint32(0); i < p; i++ {
				begin, end := ParForIters(desc, i)
				size := end - begin
				sum += size
				if i == 0 {
					size0 = size
					if begin != 0 {
						t.Fatalf("incorrect begin: %d (n=%d, p=%d)", begin, n, p)
					}
				} else {
					if size != size0 && size != size0+1 {
						t.Fatalf("incorrect size: %d/%d (n=%d, p=%d)", size, size0, n, p)
					}
					if begin != end0 {
						t.Fatalf("incorrect begin/end: %d/%d (n=%d, p=%d)", begin, end0, n, p)
					}
				}
				end0 = end
			}
			if sum != n {
				t.Fatalf("incorrect sum: %d/%d (p=%d)", sum, n, p)
			}
		}
	}
}

// Test parallel parallelfor.
func TestParForParallel(t *testing.T) {
	if GOARCH != "amd64" {
		t.Log("temporarily disabled, see http://golang.org/issue/4155")
		return
	}

	N := uint64(1e7)
	if testing.Short() {
		N /= 10
	}
	data := make([]uint64, N)
	for i := uint64(0); i < N; i++ {
		data[i] = i
	}
	P := GOMAXPROCS(-1)
	desc := NewParFor(uint32(P))
	ParForSetup(desc, uint32(P), uint32(N), nil, true, func(desc *ParFor, i uint32) {
		data[i] = data[i]*data[i] + 1
	})
	for p := 1; p < P; p++ {
		go ParForDo(desc)
	}
	ParForDo(desc)
	for i := uint64(0); i < N; i++ {
		if data[i] != i*i+1 {
			t.Fatalf("Wrong element %d: %d", i, data[i])
		}
	}

	data, desc = nil, nil
	GC()
}
<commit_msg>runtime: disable parallel for tests under race detector. The race detector does not understand ParFor synchronization, because it's implemented in C. If run with -cpu=2 currently race detector says:  WARNING: DATA RACE  Read by goroutine 5:   runtime_test.TestParForParallel()       src/pkg/runtime/parfor_test.go:118 +0x2e0   testing.tRunner()       src/pkg/testing/testing.go:301 +0x8f  Previous write by goroutine 6:   runtime_test.func?024()       src/pkg/runtime/parfor_test.go:111 +0x52<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The race detector does not understand ParFor synchronization.
// +build !race

package runtime_test

import (
	. "runtime"
	"testing"
	"unsafe"
)

// Simple serial sanity test for parallelfor.
func TestParFor(t *testing.T) {
	const P = 1
	const N = 20
	data := make([]uint64, N)
	for i := uint64(0); i < N; i++ {
		data[i] = i
	}
	desc := NewParFor(P)
	ParForSetup(desc, P, N, nil, true, func(desc *ParFor, i uint32) {
		data[i] = data[i]*data[i] + 1
	})
	ParForDo(desc)
	for i := uint64(0); i < N; i++ {
		if data[i] != i*i+1 {
			t.Fatalf("Wrong element %d: %d", i, data[i])
		}
	}
}

// Test that nonblocking parallelfor does not block.
func TestParFor2(t *testing.T) {
	const P = 7
	const N = 1003
	data := make([]uint64, N)
	for i := uint64(0); i < N; i++ {
		data[i] = i
	}
	desc := NewParFor(P)
	ParForSetup(desc, P, N, (*byte)(unsafe.Pointer(&data)), false, func(desc *ParFor, i uint32) {
		d := *(*[]uint64)(unsafe.Pointer(desc.Ctx))
		d[i] = d[i]*d[i] + 1
	})
	for p := 0; p < P; p++ {
		ParForDo(desc)
	}
	for i := uint64(0); i < N; i++ {
		if data[i] != i*i+1 {
			t.Fatalf("Wrong element %d: %d", i, data[i])
		}
	}
}

// Test that iterations are properly distributed.
func TestParForSetup(t *testing.T) {
	const P = 11
	const N = 101
	desc := NewParFor(P)
	for n := uint32(0); n < N; n++ {
		for p := uint32(1); p <= P; p++ {
			ParForSetup(desc, p, n, nil, true, func(desc *ParFor, i uint32) {})
			sum := uint32(0)
			size0 := uint32(0)
			end0 := uint32(0)
			for i := uint32(0); i < p; i++ {
				begin, end := ParForIters(desc, i)
				size := end - begin
				sum += size
				if i == 0 {
					size0 = size
					if begin != 0 {
						t.Fatalf("incorrect begin: %d (n=%d, p=%d)", begin, n, p)
					}
				} else {
					if size != size0 && size != size0+1 {
						t.Fatalf("incorrect size: %d/%d (n=%d, p=%d)", size, size0, n, p)
					}
					if begin != end0 {
						t.Fatalf("incorrect begin/end: %d/%d (n=%d, p=%d)", begin, end0, n, p)
					}
				}
				end0 = end
			}
			if sum != n {
				t.Fatalf("incorrect sum: %d/%d (p=%d)", sum, n, p)
			}
		}
	}
}

// Test parallel parallelfor.
func TestParForParallel(t *testing.T) {
	if GOARCH != "amd64" {
		t.Log("temporarily disabled, see http://golang.org/issue/4155")
		return
	}

	N := uint64(1e7)
	if testing.Short() {
		N /= 10
	}
	data := make([]uint64, N)
	for i := uint64(0); i < N; i++ {
		data[i] = i
	}
	P := GOMAXPROCS(-1)
	desc := NewParFor(uint32(P))
	ParForSetup(desc, uint32(P), uint32(N), nil, true, func(desc *ParFor, i uint32) {
		data[i] = data[i]*data[i] + 1
	})
	for p := 1; p < P; p++ {
		go ParForDo(desc)
	}
	ParForDo(desc)
	for i := uint64(0); i < N; i++ {
		if data[i] != i*i+1 {
			t.Fatalf("Wrong element %d: %d", i, data[i])
		}
	}

	data, desc = nil, nil
	GC()
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package dog adds dog images to issues in response to a /woof comment
package dog

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"regexp"
	"strings"

	"github.com/sirupsen/logrus"

	"k8s.io/test-infra/prow/github"
	"k8s.io/test-infra/prow/pluginhelp"
	"k8s.io/test-infra/prow/plugins"
)

var (
	match = regexp.MustCompile(`(?mi)^/(woof|bark)\s*$`)
)

const (
	dogURL     = realPack("https://random.dog/woof.json")
	pluginName = "dog"
)

func init() {
	plugins.RegisterGenericCommentHandler(pluginName, handleGenericComment, helpProvider)
}

func helpProvider(config *plugins.Configuration, enabledRepos []string) (*pluginhelp.PluginHelp, error) {
	// The Config field is omitted because this plugin is not configurable.
	// TODO(qhuynh96): Removes all the fields of pluginHelp except Description.
	pluginHelp := &pluginhelp.PluginHelp{
		Description: "The dog plugin adds a dog image to an issue in response to the `/woof` command.",
		WhoCanUse:   "Anyone",
		Usage:       "/woof | /bark",
		Examples:    []string{"/woof", "/bark"},
	}
	pluginHelp.AddCommand(pluginhelp.Command{
		Usage:       "/woof | /bark",
		Description: "Add a dog image to the issue",
		Featured:    false,
		WhoCanUse:   "Anyone",
		Examples:    []string{"/woof", "/bark"},
	})
	return pluginHelp, nil
}

type githubClient interface {
	CreateComment(owner, repo string, number int, comment string) error
}

type pack interface {
	readDog() (string, error)
}

type realPack string

var client = http.Client{}

type dogResult struct {
	URL string `json:"url"`
}

func (dr dogResult) Format() (string, error) {
	if dr.URL == "" {
		return "", errors.New("empty url")
	}
	src, err := url.ParseRequestURI(dr.URL)
	if err != nil {
		return "", fmt.Errorf("invalid url %s: %v", dr.URL, err)
	}
	return fmt.Sprintf("[![dog image](%s)](%s)", src, src), nil
}

func (u realPack) readDog() (string, error) {
	uri := string(u)
	req, err := http.NewRequest("GET", uri, nil)
	if err != nil {
		return "", fmt.Errorf("could not create request %s: %v", uri, err)
	}
	req.Header.Add("Accept", "application/json")
	resp, err := client.Do(req)
	if err != nil {
		return "", fmt.Errorf("could not read dog from %s: %v", uri, err)
	}
	defer resp.Body.Close()
	var a dogResult
	if err = json.NewDecoder(resp.Body).Decode(&a); err != nil {
		return "", err
	}
	// GitHub doesn't support videos :(
	if strings.HasSuffix(a.URL, ".mp4") {
		return "", errors.New("unsupported doggo :( github doesn't support .mp4")
	}
	return a.Format()
}

func handleGenericComment(pc plugins.PluginClient, e github.GenericCommentEvent) error {
	return handle(pc.GitHubClient, pc.Logger, &e, dogURL)
}

func handle(gc githubClient, log *logrus.Entry, e *github.GenericCommentEvent, p pack) error {
	// Only consider new comments.
	if e.Action != github.GenericCommentActionCreated {
		return nil
	}
	// Make sure they are requesting a dog
	mat := match.FindStringSubmatch(e.Body)
	if mat == nil {
		return nil
	}

	org := e.Repo.Owner.Login
	repo := e.Repo.Name
	number := e.Number

	for i := 0; i < 5; i++ {
		resp, err := p.readDog()
		if err != nil {
			log.WithError(err).Println("Failed to get dog img")
			continue
		}
		return gc.CreateComment(org, repo, number, plugins.FormatResponseRaw(e.Body, e.HTMLURL, e.User.Login, resp))
	}

	return errors.New("could not find a valid dog image")
}
<commit_msg>fix dog plugin usage string<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package dog adds dog images to issues in response to a /woof comment
package dog

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"regexp"
	"strings"

	"github.com/sirupsen/logrus"

	"k8s.io/test-infra/prow/github"
	"k8s.io/test-infra/prow/pluginhelp"
	"k8s.io/test-infra/prow/plugins"
)

var (
	match = regexp.MustCompile(`(?mi)^/(woof|bark)\s*$`)
)

const (
	dogURL     = realPack("https://random.dog/woof.json")
	pluginName = "dog"
)

func init() {
	plugins.RegisterGenericCommentHandler(pluginName, handleGenericComment, helpProvider)
}

func helpProvider(config *plugins.Configuration, enabledRepos []string) (*pluginhelp.PluginHelp, error) {
	// The Config field is omitted because this plugin is not configurable.
	// TODO(qhuynh96): Removes all the fields of pluginHelp except Description.
	pluginHelp := &pluginhelp.PluginHelp{
		Description: "The dog plugin adds a dog image to an issue in response to the `/woof` command.",
		WhoCanUse:   "Anyone",
		Usage:       "/(woof|bark)",
		Examples:    []string{"/woof", "/bark"},
	}
	pluginHelp.AddCommand(pluginhelp.Command{
		Usage:       "/woof | /bark",
		Description: "Add a dog image to the issue",
		Featured:    false,
		WhoCanUse:   "Anyone",
		Examples:    []string{"/woof", "/bark"},
	})
	return pluginHelp, nil
}

type githubClient interface {
	CreateComment(owner, repo string, number int, comment string) error
}

type pack interface {
	readDog() (string, error)
}

type realPack string

var client = http.Client{}

type dogResult struct {
	URL string `json:"url"`
}

func (dr dogResult) Format() (string, error) {
	if dr.URL == "" {
		return "", errors.New("empty url")
	}
	src, err := url.ParseRequestURI(dr.URL)
	if err != nil {
		return "", fmt.Errorf("invalid url %s: %v", dr.URL, err)
	}
	return fmt.Sprintf("[![dog image](%s)](%s)", src, src), nil
}

func (u realPack) readDog() (string, error) {
	uri := string(u)
	req, err := http.NewRequest("GET", uri, nil)
	if err != nil {
		return "", fmt.Errorf("could not create request %s: %v", uri, err)
	}
	req.Header.Add("Accept", "application/json")
	resp, err := client.Do(req)
	if err != nil {
		return "", fmt.Errorf("could not read dog from %s: %v", uri, err)
	}
	defer resp.Body.Close()
	var a dogResult
	if err = json.NewDecoder(resp.Body).Decode(&a); err != nil {
		return "", err
	}
	// GitHub doesn't support videos :(
	if strings.HasSuffix(a.URL, ".mp4") {
		return "", errors.New("unsupported doggo :( github doesn't support .mp4")
	}
	return a.Format()
}

func handleGenericComment(pc plugins.PluginClient, e github.GenericCommentEvent) error {
	return handle(pc.GitHubClient, pc.Logger, &e, dogURL)
}

func handle(gc githubClient, log *logrus.Entry, e *github.GenericCommentEvent, p pack) error {
	// Only consider new comments.
	if e.Action != github.GenericCommentActionCreated {
		return nil
	}
	// Make sure they are requesting a dog
	mat := match.FindStringSubmatch(e.Body)
	if mat == nil {
		return nil
	}

	org := e.Repo.Owner.Login
	repo := e.Repo.Name
	number := e.Number

	for i := 0; i < 5; i++ {
		resp, err := p.readDog()
		if err != nil {
			log.WithError(err).Println("Failed to get dog img")
			continue
		}
		return gc.CreateComment(org, repo, number, plugins.FormatResponseRaw(e.Body, e.HTMLURL, e.User.Login, resp))
	}

	return errors.New("could not find a valid dog image")
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"code.google.com/p/rog-go/exp/go/ast"
	"code.google.com/p/rog-go/exp/go/parser"
	"code.google.com/p/rog-go/exp/go/printer"
	"code.google.com/p/rog-go/exp/go/types"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"strings"
)

var readStdin = flag.Bool("i", false, "read file from stdin")
var offset = flag.Int("o", -1, "file offset of identifier in stdin")
var debug = flag.Bool("debug", false, "debug mode")
var tflag = flag.Bool("t", false, "print type information")
var aflag = flag.Bool("a", false, "print public type and member information")
var Aflag = flag.Bool("A", false, "print all type and members information")
var fflag = flag.String("f", "", "Go source filename")
var acmeFlag = flag.Bool("acme", false, "use current acme window")

func fail(s string, a ...interface{}) {
	fmt.Fprint(os.Stderr, "godef: "+fmt.Sprintf(s, a...)+"\n")
	os.Exit(2)
}

func init() {
	// take GOPATH, set types.GoPath to it if it's not empty.
	p := os.Getenv("GOPATH")
	if p == "" {
		return
	}
	gopath := strings.Split(p, ":")
	for i, d := range gopath {
		gopath[i] = filepath.Join(d, "src")
	}
	r := os.Getenv("GOROOT")
	if r != "" {
		gopath = append(gopath, r+"/src/pkg")
	}
	types.GoPath = gopath
}

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "usage: godef [flags] [expr]\n")
		flag.PrintDefaults()
	}
	flag.Parse()
	if flag.NArg() > 1 {
		flag.Usage()
		os.Exit(2)
	}
	types.Debug = *debug
	*tflag = *tflag || *aflag || *Aflag
	searchpos := *offset
	filename := *fflag

	var afile *acmeFile
	var src []byte
	if *acmeFlag {
		var err error
		if afile, err = acmeCurrentFile(); err != nil {
			fail("%v", err)
		}
		filename, src, searchpos = afile.name, afile.body, afile.offset
	} else if *readStdin {
		src, _ = ioutil.ReadAll(os.Stdin)
	} else {
		// TODO if there's no filename, look in the current
		// directory and do something plausible.
		b, err := ioutil.ReadFile(filename)
		if err != nil {
			fail("cannot read %s: %v", filename, err)
		}
		src = b
	}
	pkgScope := ast.NewScope(parser.Universe)
	f, err := parser.ParseFile(types.FileSet, filename, src, 0, pkgScope)
	if f == nil {
		fail("cannot parse %s: %v", filename, err)
	}

	var e ast.Expr
	switch {
	case flag.NArg() > 1:
		e = parseExpr(f.Scope, flag.Arg(1))

	case searchpos >= 0:
		e = findIdentifier(f, searchpos)

	default:
		fmt.Fprintf(os.Stderr, "no expression or offset specified\n")
		flag.Usage()
		os.Exit(2)
	}
	// print old source location to facilitate backtracking
	if *acmeFlag {
		fmt.Printf("\t%s:#%d\n", afile.name, afile.runeOffset)
	}
	if !*tflag {
		// try local declarations only
		if obj, typ := types.ExprType(e, types.DefaultImporter); obj != nil {
			done(obj, typ)
		}
	}
	// add declarations from other files in the local package and try again
	pkg, _ := parseLocalPackage(filename, f, pkgScope)
	if pkg == nil && !*tflag {
		fmt.Printf("parseLocalPackage error: %v\n", err)
	}
	if obj, typ := types.ExprType(e, types.DefaultImporter); obj != nil {
		done(obj, typ)
	}
	fail("no declaration found for %v", pretty{e})
}

// findIdentifier looks for an identifier at byte-offset searchpos
// inside the parsed source represented by node.
// If it is part of a selector expression, it returns
// that expression rather than the identifier itself.
//
func findIdentifier(f *ast.File, searchpos int) ast.Expr {
	ec := make(chan ast.Expr)
	go func() {
		var visit FVisitor = func(n ast.Node) bool {
			var id *ast.Ident
			switch n := n.(type) {
			case *ast.Ident:
				id = n
			case *ast.SelectorExpr:
				id = n.Sel
			default:
				return true
			}

			pos := types.FileSet.Position(id.NamePos)
			if pos.Offset <= searchpos && pos.Offset+len(id.Name) >= searchpos {
				ec <- n.(ast.Expr)
				runtime.Goexit()
			}
			return true
		}
		ast.Walk(visit, f)
		ec <- nil
	}()
	ev := <-ec
	if ev == nil {
		fail("no identifier found")
	}
	return ev
}

type orderedObjects []*ast.Object

func (o orderedObjects) Less(i, j int) bool { return o[i].Name < o[j].Name }
func (o orderedObjects) Len() int           { return len(o) }
func (o orderedObjects) Swap(i, j int)      { o[i], o[j] = o[j], o[i] }

func done(obj *ast.Object, typ types.Type) {
	defer os.Exit(0)
	pos := types.FileSet.Position(types.DeclPos(obj))
	if pos.Column > 0 {
		pos.Column--
	}
	fmt.Printf("%v\n", pos)
	if typ.Kind == ast.Bad || !*tflag {
		return
	}
	fmt.Printf("%s\n", strings.Replace(typeStr(obj, typ), "\n", "\n\t", -1))
	if *aflag || *Aflag {
		var m orderedObjects
		for obj := range typ.Iter(types.DefaultImporter) {
			m = append(m, obj)
		}
		sort.Sort(m)
		for _, obj := range m {
			// Ignore unexported members unless Aflag is set.
			if !*Aflag && (typ.Pkg != "" || !ast.IsExported(obj.Name)) {
				continue
			}
			id := ast.NewIdent(obj.Name)
			id.Obj = obj
			_, mt := types.ExprType(id, types.DefaultImporter)
			fmt.Printf("\t%s\n", strings.Replace(typeStr(obj, mt), "\n", "\n\t\t", -1))
			fmt.Printf("\t\t%v\n", types.FileSet.Position(types.DeclPos(obj)))
		}
	}
}

func typeStr(obj *ast.Object, typ types.Type) string {
	switch typ.Kind {
	case ast.Fun, ast.Var:
		return fmt.Sprintf("%s %v", obj.Name, pretty{typ.Node})
	case ast.Pkg:
		return fmt.Sprintf("import (%s %s)", obj.Name, typ.Node.(*ast.ImportSpec).Path.Value)
	case ast.Con:
		return fmt.Sprintf("const %s %v", obj.Name, pretty{typ.Node})
	case ast.Lbl:
		return fmt.Sprintf("label %s", obj.Name)
	case ast.Typ:
		typ = typ.Underlying(false, types.DefaultImporter)
		return fmt.Sprintf("type %s %v", obj.Name, pretty{typ.Node})
	}
	return fmt.Sprintf("unknown %s %v", obj.Name, typ.Kind)
}

func parseExpr(s *ast.Scope, expr string) ast.Expr {
	n, err := parser.ParseExpr(types.FileSet, "<arg>", expr, s)
	if err != nil {
		fail("cannot parse expression: %v", err)
	}
	switch n := n.(type) {
	case *ast.Ident, *ast.SelectorExpr:
		return n
	}
	fail("no identifier found in expression")
	return nil
}

type FVisitor func(n ast.Node) bool

func (f FVisitor) Visit(n ast.Node) ast.Visitor {
	if f(n) {
		return f
	}
	return nil
}

var errNoPkgFiles = errors.New("no more package files found")

// parseLocalPackage reads and parses all go files from the
// current directory that implement the same package name
// the principal source file, except the original source file
// itself, which will already have been parsed.
//
func parseLocalPackage(filename string, src *ast.File, pkgScope *ast.Scope) (*ast.Package, error) {
	pkg := &ast.Package{src.Name.Name, pkgScope, nil, map[string]*ast.File{filename: src}}
	d, f := filepath.Split(filename)
	if d == "" {
		d = "./"
	}
	fd, err := os.Open(d)
	if err != nil {
		return nil, errNoPkgFiles
	}
	defer fd.Close()

	list, err := fd.Readdirnames(-1)
	if err != nil {
		return nil, errNoPkgFiles
	}

	for _, pf := range list {
		file := filepath.Join(d, pf)
		if !strings.HasSuffix(pf, ".go") ||
			pf == f ||
			pkgName(file) != pkg.Name {
			continue
		}
		src, err := parser.ParseFile(types.FileSet, file, nil, 0, pkg.Scope)
		if err == nil {
			pkg.Files[file] = src
		}
	}
	if len(pkg.Files) == 1 {
		return nil, errNoPkgFiles
	}
	return pkg, nil
}

// pkgName returns the package name implemented by the
// go source filename.
//
func pkgName(filename string) string {
	prog, _ := parser.ParseFile(types.FileSet, filename, nil, parser.PackageClauseOnly, nil)
	if prog != nil {
		return prog.Name.Name
	}
	return ""
}

func hasSuffix(s, suff string) bool {
	return len(s) >= len(suff) && s[len(s)-len(suff):] == suff
}

type pretty struct {
	n interface{}
}

func (p pretty) String() string {
	var b bytes.Buffer
	printer.Fprint(&b, types.FileSet, p.n)
	return b.String()
}
<commit_msg>godef: fix with expression<commit_after>package main

import (
	"bytes"
	"code.google.com/p/rog-go/exp/go/ast"
	"code.google.com/p/rog-go/exp/go/parser"
	"code.google.com/p/rog-go/exp/go/printer"
	"code.google.com/p/rog-go/exp/go/types"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"strings"
)

var readStdin = flag.Bool("i", false, "read file from stdin")
var offset = flag.Int("o", -1, "file offset of identifier in stdin")
var debug = flag.Bool("debug", false, "debug mode")
var tflag = flag.Bool("t", false, "print type information")
var aflag = flag.Bool("a", false, "print public type and member information")
var Aflag = flag.Bool("A", false, "print all type and members information")
var fflag = flag.String("f", "", "Go source filename")
var acmeFlag = flag.Bool("acme", false, "use current acme window")

func fail(s string, a ...interface{}) {
	fmt.Fprint(os.Stderr, "godef: "+fmt.Sprintf(s, a...)+"\n")
	os.Exit(2)
}

func init() {
	// take GOPATH, set types.GoPath to it if it's not empty.
	p := os.Getenv("GOPATH")
	if p == "" {
		return
	}
	gopath := strings.Split(p, ":")
	for i, d := range gopath {
		gopath[i] = filepath.Join(d, "src")
	}
	r := os.Getenv("GOROOT")
	if r != "" {
		gopath = append(gopath, r+"/src/pkg")
	}
	types.GoPath = gopath
}

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "usage: godef [flags] [expr]\n")
		flag.PrintDefaults()
	}
	flag.Parse()
	if flag.NArg() > 1 {
		flag.Usage()
		os.Exit(2)
	}
	types.Debug = *debug
	*tflag = *tflag || *aflag || *Aflag
	searchpos := *offset
	filename := *fflag

	var afile *acmeFile
	var src []byte
	if *acmeFlag {
		var err error
		if afile, err = acmeCurrentFile(); err != nil {
			fail("%v", err)
		}
		filename, src, searchpos = afile.name, afile.body, afile.offset
	} else if *readStdin {
		src, _ = ioutil.ReadAll(os.Stdin)
	} else {
		// TODO if there's no filename, look in the current
		// directory and do something plausible.
		b, err := ioutil.ReadFile(filename)
		if err != nil {
			fail("cannot read %s: %v", filename, err)
		}
		src = b
	}
	pkgScope := ast.NewScope(parser.Universe)
	f, err := parser.ParseFile(types.FileSet, filename, src, 0, pkgScope)
	if f == nil {
		fail("cannot parse %s: %v", filename, err)
	}

	var e ast.Expr
	switch {
	case flag.NArg() > 0:
		e = parseExpr(f.Scope, flag.Arg(0))

	case searchpos >= 0:
		e = findIdentifier(f, searchpos)

	default:
		fmt.Fprintf(os.Stderr, "no expression or offset specified\n")
		flag.Usage()
		os.Exit(2)
	}
	// print old source location to facilitate backtracking
	if *acmeFlag {
		fmt.Printf("\t%s:#%d\n", afile.name, afile.runeOffset)
	}
	if !*tflag {
		// try local declarations only
		if obj, typ := types.ExprType(e, types.DefaultImporter); obj != nil {
			done(obj, typ)
		}
	}
	// add declarations from other files in the local package and try again
	pkg, _ := parseLocalPackage(filename, f, pkgScope)
	if pkg == nil && !*tflag {
		fmt.Printf("parseLocalPackage error: %v\n", err)
	}
	if obj, typ := types.ExprType(e, types.DefaultImporter); obj != nil {
		done(obj, typ)
	}
	fail("no declaration found for %v", pretty{e})
}

// findIdentifier looks for an identifier at byte-offset searchpos
// inside the parsed source represented by node.
// If it is part of a selector expression, it returns
// that expression rather than the identifier itself.
//
func findIdentifier(f *ast.File, searchpos int) ast.Expr {
	ec := make(chan ast.Expr)
	go func() {
		var visit FVisitor = func(n ast.Node) bool {
			var id *ast.Ident
			switch n := n.(type) {
			case *ast.Ident:
				id = n
			case *ast.SelectorExpr:
				id = n.Sel
			default:
				return true
			}

			pos := types.FileSet.Position(id.NamePos)
			if pos.Offset <= searchpos && pos.Offset+len(id.Name) >= searchpos {
				ec <- n.(ast.Expr)
				runtime.Goexit()
			}
			return true
		}
		ast.Walk(visit, f)
		ec <- nil
	}()
	ev := <-ec
	if ev == nil {
		fail("no identifier found")
	}
	return ev
}

type orderedObjects []*ast.Object

func (o orderedObjects) Less(i, j int) bool { return o[i].Name < o[j].Name }
func (o orderedObjects) Len() int           { return len(o) }
func (o orderedObjects) Swap(i, j int)      { o[i], o[j] = o[j], o[i] }

func done(obj *ast.Object, typ types.Type) {
	defer os.Exit(0)
	pos := types.FileSet.Position(types.DeclPos(obj))
	if pos.Column > 0 {
		pos.Column--
	}
	fmt.Printf("%v\n", pos)
	if typ.Kind == ast.Bad || !*tflag {
		return
	}
	fmt.Printf("%s\n", strings.Replace(typeStr(obj, typ), "\n", "\n\t", -1))
	if *aflag || *Aflag {
		var m orderedObjects
		for obj := range typ.Iter(types.DefaultImporter) {
			m = append(m, obj)
		}
		sort.Sort(m)
		for _, obj := range m {
			// Ignore unexported members unless Aflag is set.
			if !*Aflag && (typ.Pkg != "" || !ast.IsExported(obj.Name)) {
				continue
			}
			id := ast.NewIdent(obj.Name)
			id.Obj = obj
			_, mt := types.ExprType(id, types.DefaultImporter)
			fmt.Printf("\t%s\n", strings.Replace(typeStr(obj, mt), "\n", "\n\t\t", -1))
			fmt.Printf("\t\t%v\n", types.FileSet.Position(types.DeclPos(obj)))
		}
	}
}

func typeStr(obj *ast.Object, typ types.Type) string {
	switch typ.Kind {
	case ast.Fun, ast.Var:
		return fmt.Sprintf("%s %v", obj.Name, pretty{typ.Node})
	case ast.Pkg:
		return fmt.Sprintf("import (%s %s)", obj.Name, typ.Node.(*ast.ImportSpec).Path.Value)
	case ast.Con:
		return fmt.Sprintf("const %s %v", obj.Name, pretty{typ.Node})
	case ast.Lbl:
		return fmt.Sprintf("label %s", obj.Name)
	case ast.Typ:
		typ = typ.Underlying(false, types.DefaultImporter)
		return fmt.Sprintf("type %s %v", obj.Name, pretty{typ.Node})
	}
	return fmt.Sprintf("unknown %s %v", obj.Name, typ.Kind)
}

func parseExpr(s *ast.Scope, expr string) ast.Expr {
	n, err := parser.ParseExpr(types.FileSet, "<arg>", expr, s)
	if err != nil {
		fail("cannot parse expression: %v", err)
	}
	switch n := n.(type) {
	case *ast.Ident, *ast.SelectorExpr:
		return n
	}
	fail("no identifier found in expression")
	return nil
}

type FVisitor func(n ast.Node) bool

func (f FVisitor) Visit(n ast.Node) ast.Visitor {
	if f(n) {
		return f
	}
	return nil
}

var errNoPkgFiles = errors.New("no more package files found")

// parseLocalPackage reads and parses all go files from the
// current directory that implement the same package name
// the principal source file, except the original source file
// itself, which will already have been parsed.
//
func parseLocalPackage(filename string, src *ast.File, pkgScope *ast.Scope) (*ast.Package, error) {
	pkg := &ast.Package{src.Name.Name, pkgScope, nil, map[string]*ast.File{filename: src}}
	d, f := filepath.Split(filename)
	if d == "" {
		d = "./"
	}
	fd, err := os.Open(d)
	if err != nil {
		return nil, errNoPkgFiles
	}
	defer fd.Close()

	list, err := fd.Readdirnames(-1)
	if err != nil {
		return nil, errNoPkgFiles
	}

	for _, pf := range list {
		file := filepath.Join(d, pf)
		if !strings.HasSuffix(pf, ".go") ||
			pf == f ||
			pkgName(file) != pkg.Name {
			continue
		}
		src, err := parser.ParseFile(types.FileSet, file, nil, 0, pkg.Scope)
		if err == nil {
			pkg.Files[file] = src
		}
	}
	if len(pkg.Files) == 1 {
		return nil, errNoPkgFiles
	}
	return pkg, nil
}

// pkgName returns the package name implemented by the
// go source filename.
//
func pkgName(filename string) string {
	prog, _ := parser.ParseFile(types.FileSet, filename, nil, parser.PackageClauseOnly, nil)
	if prog != nil {
		return prog.Name.Name
	}
	return ""
}

func hasSuffix(s, suff string) bool {
	return len(s) >= len(suff) && s[len(s)-len(suff):] == suff
}

type pretty struct {
	n interface{}
}

func (p pretty) String() string {
	var b bytes.Buffer
	printer.Fprint(&b, types.FileSet, p.n)
	return b.String()
}
<|endoftext|>