<commit_before>// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime_test

import (
	. "runtime"
	"testing"
)

func TestMemmove(t *testing.T) {
	size := 256
	if testing.Short() {
		size = 128 + 16
	}
	src := make([]byte, size)
	dst := make([]byte, size)
	for i := 0; i < size; i++ {
		src[i] = byte(128 + (i & 127))
	}
	for i := 0; i < size; i++ {
		dst[i] = byte(i & 127)
	}
	for n := 0; n <= size; n++ {
		for x := 0; x <= size-n; x++ { // offset in src
			for y := 0; y <= size-n; y++ { // offset in dst
				copy(dst[y:y+n], src[x:x+n])
				for i := 0; i < y; i++ {
					if dst[i] != byte(i&127) {
						t.Fatalf("prefix dst[%d] = %d", i, dst[i])
					}
				}
				for i := y; i < y+n; i++ {
					if dst[i] != byte(128+((i-y+x)&127)) {
						t.Fatalf("copied dst[%d] = %d", i, dst[i])
					}
					dst[i] = byte(i & 127) // reset dst
				}
				for i := y + n; i < size; i++ {
					if dst[i] != byte(i&127) {
						t.Fatalf("suffix dst[%d] = %d", i, dst[i])
					}
				}
			}
		}
	}
}

func TestMemmoveAlias(t *testing.T) {
	size := 256
	if testing.Short() {
		size = 128 + 16
	}
	buf := make([]byte, size)
	for i := 0; i < size; i++ {
		buf[i] = byte(i)
	}
	for n := 0; n <= size; n++ {
		for x := 0; x <= size-n; x++ { // src offset
			for y := 0; y <= size-n; y++ { // dst offset
				copy(buf[y:y+n], buf[x:x+n])
				for i := 0; i < y; i++ {
					if buf[i] != byte(i) {
						t.Fatalf("prefix buf[%d] = %d", i, buf[i])
					}
				}
				for i := y; i < y+n; i++ {
					if buf[i] != byte(i-y+x) {
						t.Fatalf("copied buf[%d] = %d", i, buf[i])
					}
					buf[i] = byte(i) // reset buf
				}
				for i := y + n; i < size; i++ {
					if buf[i] != byte(i) {
						t.Fatalf("suffix buf[%d] = %d", i, buf[i])
					}
				}
			}
		}
	}
}

func bmMemmove(b *testing.B, n int) {
	x := make([]byte, n)
	y := make([]byte, n)
	b.SetBytes(int64(n))
	for i := 0; i < b.N; i++ {
		copy(x, y)
	}
}

func BenchmarkMemmove0(b *testing.B)    { bmMemmove(b, 0) }
func BenchmarkMemmove1(b *testing.B)    { bmMemmove(b, 1) }
func BenchmarkMemmove2(b *testing.B)    { bmMemmove(b, 2) }
func BenchmarkMemmove3(b *testing.B)    { bmMemmove(b, 3) }
func BenchmarkMemmove4(b *testing.B)    { bmMemmove(b, 4) }
func BenchmarkMemmove5(b *testing.B)    { bmMemmove(b, 5) }
func BenchmarkMemmove6(b *testing.B)    { bmMemmove(b, 6) }
func BenchmarkMemmove7(b *testing.B)    { bmMemmove(b, 7) }
func BenchmarkMemmove8(b *testing.B)    { bmMemmove(b, 8) }
func BenchmarkMemmove9(b *testing.B)    { bmMemmove(b, 9) }
func BenchmarkMemmove10(b *testing.B)   { bmMemmove(b, 10) }
func BenchmarkMemmove11(b *testing.B)   { bmMemmove(b, 11) }
func BenchmarkMemmove12(b *testing.B)   { bmMemmove(b, 12) }
func BenchmarkMemmove13(b *testing.B)   { bmMemmove(b, 13) }
func BenchmarkMemmove14(b *testing.B)   { bmMemmove(b, 14) }
func BenchmarkMemmove15(b *testing.B)   { bmMemmove(b, 15) }
func BenchmarkMemmove16(b *testing.B)   { bmMemmove(b, 16) }
func BenchmarkMemmove32(b *testing.B)   { bmMemmove(b, 32) }
func BenchmarkMemmove64(b *testing.B)   { bmMemmove(b, 64) }
func BenchmarkMemmove128(b *testing.B)  { bmMemmove(b, 128) }
func BenchmarkMemmove256(b *testing.B)  { bmMemmove(b, 256) }
func BenchmarkMemmove512(b *testing.B)  { bmMemmove(b, 512) }
func BenchmarkMemmove1024(b *testing.B) { bmMemmove(b, 1024) }
func BenchmarkMemmove2048(b *testing.B) { bmMemmove(b, 2048) }
func BenchmarkMemmove4096(b *testing.B) { bmMemmove(b, 4096) }

func TestMemclr(t *testing.T) {
	size := 512
	if testing.Short() {
		size = 128 + 16
	}
	mem := make([]byte, size)
	for i := 0; i < size; i++ {
		mem[i] = 0xee
	}
	for n := 0; n < size; n++ {
		for x := 0; x <= size-n; x++ { // offset in mem
			MemclrBytes(mem[x : x+n])
			for i := 0; i < x; i++ {
				if mem[i] != 0xee {
					t.Fatalf("overwrite prefix mem[%d] = %d", i, mem[i])
				}
			}
			for i := x; i < x+n; i++ {
				if mem[i] != 0 {
					t.Fatalf("failed clear mem[%d] = %d", i, mem[i])
				}
				mem[i] = 0xee
			}
			for i := x + n; i < size; i++ {
				if mem[i] != 0xee {
					t.Fatalf("overwrite suffix mem[%d] = %d", i, mem[i])
				}
			}
		}
	}
}

func bmMemclr(b *testing.B, n int) {
	x := make([]byte, n)
	b.SetBytes(int64(n))
	for i := 0; i < b.N; i++ {
		MemclrBytes(x)
	}
}
func BenchmarkMemclr5(b *testing.B)     { bmMemclr(b, 5) }
func BenchmarkMemclr16(b *testing.B)    { bmMemclr(b, 16) }
func BenchmarkMemclr64(b *testing.B)    { bmMemclr(b, 64) }
func BenchmarkMemclr256(b *testing.B)   { bmMemclr(b, 256) }
func BenchmarkMemclr4096(b *testing.B)  { bmMemclr(b, 4096) }
func BenchmarkMemclr65536(b *testing.B) { bmMemclr(b, 65536) }

func BenchmarkClearFat32(b *testing.B) {
	for i := 0; i < b.N; i++ {
		var x [32]byte
		_ = x
	}
}
func BenchmarkClearFat64(b *testing.B) {
	for i := 0; i < b.N; i++ {
		var x [64]byte
		_ = x
	}
}
func BenchmarkClearFat128(b *testing.B) {
	for i := 0; i < b.N; i++ {
		var x [128]byte
		_ = x
	}
}
func BenchmarkClearFat256(b *testing.B) {
	for i := 0; i < b.N; i++ {
		var x [256]byte
		_ = x
	}
}
func BenchmarkClearFat512(b *testing.B) {
	for i := 0; i < b.N; i++ {
		var x [512]byte
		_ = x
	}
}
func BenchmarkClearFat1024(b *testing.B) {
	for i := 0; i < b.N; i++ {
		var x [1024]byte
		_ = x
	}
}

func BenchmarkCopyFat32(b *testing.B) {
	var x [32 / 4]uint32
	for i := 0; i < b.N; i++ {
		y := x
		_ = y
	}
}
func BenchmarkCopyFat64(b *testing.B) {
	var x [64 / 4]uint32
	for i := 0; i < b.N; i++ {
		y := x
		_ = y
	}
}
func BenchmarkCopyFat128(b *testing.B) {
	var x [128 / 4]uint32
	for i := 0; i < b.N; i++ {
		y := x
		_ = y
	}
}
func BenchmarkCopyFat256(b *testing.B) {
	var x [256 / 4]uint32
	for i := 0; i < b.N; i++ {
		y := x
		_ = y
	}
}
func BenchmarkCopyFat512(b *testing.B) {
	var x [512 / 4]uint32
	for i := 0; i < b.N; i++ {
		y := x
		_ = y
	}
}
func BenchmarkCopyFat1024(b *testing.B) {
	var x [1024 / 4]uint32
	for i := 0; i < b.N; i++ {
		y := x
		_ = y
	}
}
<commit_msg>runtime: add Benchmark[Clear|Copy]Fat[16|24]<commit_after>// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime_test

import (
	. "runtime"
	"testing"
)

func TestMemmove(t *testing.T) {
	size := 256
	if testing.Short() {
		size = 128 + 16
	}
	src := make([]byte, size)
	dst := make([]byte, size)
	for i := 0; i < size; i++ {
		src[i] = byte(128 + (i & 127))
	}
	for i := 0; i < size; i++ {
		dst[i] = byte(i & 127)
	}
	for n := 0; n <= size; n++ {
		for x := 0; x <= size-n; x++ { // offset in src
			for y := 0; y <= size-n; y++ { // offset in dst
				copy(dst[y:y+n], src[x:x+n])
				for i := 0; i < y; i++ {
					if dst[i] != byte(i&127) {
						t.Fatalf("prefix dst[%d] = %d", i, dst[i])
					}
				}
				for i := y; i < y+n; i++ {
					if dst[i] != byte(128+((i-y+x)&127)) {
						t.Fatalf("copied dst[%d] = %d", i, dst[i])
					}
					dst[i] = byte(i & 127) // reset dst
				}
				for i := y + n; i < size; i++ {
					if dst[i] != byte(i&127) {
						t.Fatalf("suffix dst[%d] = %d", i, dst[i])
					}
				}
			}
		}
	}
}

func TestMemmoveAlias(t *testing.T) {
	size := 256
	if testing.Short() {
		size = 128 + 16
	}
	buf := make([]byte, size)
	for i := 0; i < size; i++ {
		buf[i] = byte(i)
	}
	for n := 0; n <= size; n++ {
		for x := 0; x <= size-n; x++ { // src offset
			for y := 0; y <= size-n; y++ { // dst offset
				copy(buf[y:y+n], buf[x:x+n])
				for i := 0; i < y; i++ {
					if buf[i] != byte(i) {
						t.Fatalf("prefix buf[%d] = %d", i, buf[i])
					}
				}
				for i := y; i < y+n; i++ {
					if buf[i] != byte(i-y+x) {
						t.Fatalf("copied buf[%d] = %d", i, buf[i])
					}
					buf[i] = byte(i) // reset buf
				}
				for i := y + n; i < size; i++ {
					if buf[i] != byte(i) {
						t.Fatalf("suffix buf[%d] = %d", i, buf[i])
					}
				}
			}
		}
	}
}

func bmMemmove(b *testing.B, n int) {
	x := make([]byte, n)
	y := make([]byte, n)
	b.SetBytes(int64(n))
	for i := 0; i < b.N; i++ {
		copy(x, y)
	}
}

func BenchmarkMemmove0(b *testing.B)    { bmMemmove(b, 0) }
func BenchmarkMemmove1(b *testing.B)    { bmMemmove(b, 1) }
func BenchmarkMemmove2(b *testing.B)    { bmMemmove(b, 2) }
func BenchmarkMemmove3(b *testing.B)    { bmMemmove(b, 3) }
func BenchmarkMemmove4(b *testing.B)    { bmMemmove(b, 4) }
func BenchmarkMemmove5(b *testing.B)    { bmMemmove(b, 5) }
func BenchmarkMemmove6(b *testing.B)    { bmMemmove(b, 6) }
func BenchmarkMemmove7(b *testing.B)    { bmMemmove(b, 7) }
func BenchmarkMemmove8(b *testing.B)    { bmMemmove(b, 8) }
func BenchmarkMemmove9(b *testing.B)    { bmMemmove(b, 9) }
func BenchmarkMemmove10(b *testing.B)   { bmMemmove(b, 10) }
func BenchmarkMemmove11(b *testing.B)   { bmMemmove(b, 11) }
func BenchmarkMemmove12(b *testing.B)   { bmMemmove(b, 12) }
func BenchmarkMemmove13(b *testing.B)   { bmMemmove(b, 13) }
func BenchmarkMemmove14(b *testing.B)   { bmMemmove(b, 14) }
func BenchmarkMemmove15(b *testing.B)   { bmMemmove(b, 15) }
func BenchmarkMemmove16(b *testing.B)   { bmMemmove(b, 16) }
func BenchmarkMemmove32(b *testing.B)   { bmMemmove(b, 32) }
func BenchmarkMemmove64(b *testing.B)   { bmMemmove(b, 64) }
func BenchmarkMemmove128(b *testing.B)  { bmMemmove(b, 128) }
func BenchmarkMemmove256(b *testing.B)  { bmMemmove(b, 256) }
func BenchmarkMemmove512(b *testing.B)  { bmMemmove(b, 512) }
func BenchmarkMemmove1024(b *testing.B) { bmMemmove(b, 1024) }
func BenchmarkMemmove2048(b *testing.B) { bmMemmove(b, 2048) }
func BenchmarkMemmove4096(b *testing.B) { bmMemmove(b, 4096) }

func TestMemclr(t *testing.T) {
	size := 512
	if testing.Short() {
		size = 128 + 16
	}
	mem := make([]byte, size)
	for i := 0; i < size; i++ {
		mem[i] = 0xee
	}
	for n := 0; n < size; n++ {
		for x := 0; x <= size-n; x++ { // offset in mem
			MemclrBytes(mem[x : x+n])
			for i := 0; i < x; i++ {
				if mem[i] != 0xee {
					t.Fatalf("overwrite prefix mem[%d] = %d", i, mem[i])
				}
			}
			for i := x; i < x+n; i++ {
				if mem[i] != 0 {
					t.Fatalf("failed clear mem[%d] = %d", i, mem[i])
				}
				mem[i] = 0xee
			}
			for i := x + n; i < size; i++ {
				if mem[i] != 0xee {
					t.Fatalf("overwrite suffix mem[%d] = %d", i, mem[i])
				}
			}
		}
	}
}

func bmMemclr(b *testing.B, n int) {
	x := make([]byte, n)
	b.SetBytes(int64(n))
	for i := 0; i < b.N; i++ {
		MemclrBytes(x)
	}
}
func BenchmarkMemclr5(b *testing.B)     { bmMemclr(b, 5) }
func BenchmarkMemclr16(b *testing.B)    { bmMemclr(b, 16) }
func BenchmarkMemclr64(b *testing.B)    { bmMemclr(b, 64) }
func BenchmarkMemclr256(b *testing.B)   { bmMemclr(b, 256) }
func BenchmarkMemclr4096(b *testing.B)  { bmMemclr(b, 4096) }
func BenchmarkMemclr65536(b *testing.B) { bmMemclr(b, 65536) }

func BenchmarkClearFat16(b *testing.B) {
	for i := 0; i < b.N; i++ {
		var x [16]byte
		_ = x
	}
}
func BenchmarkClearFat24(b *testing.B) {
	for i := 0; i < b.N; i++ {
		var x [24]byte
		_ = x
	}
}
func BenchmarkClearFat32(b *testing.B) {
	for i := 0; i < b.N; i++ {
		var x [32]byte
		_ = x
	}
}
func BenchmarkClearFat64(b *testing.B) {
	for i := 0; i < b.N; i++ {
		var x [64]byte
		_ = x
	}
}
func BenchmarkClearFat128(b *testing.B) {
	for i := 0; i < b.N; i++ {
		var x [128]byte
		_ = x
	}
}
func BenchmarkClearFat256(b *testing.B) {
	for i := 0; i < b.N; i++ {
		var x [256]byte
		_ = x
	}
}
func BenchmarkClearFat512(b *testing.B) {
	for i := 0; i < b.N; i++ {
		var x [512]byte
		_ = x
	}
}
func BenchmarkClearFat1024(b *testing.B) {
	for i := 0; i < b.N; i++ {
		var x [1024]byte
		_ = x
	}
}

func BenchmarkCopyFat16(b *testing.B) {
	var x [16 / 4]uint32
	for i := 0; i < b.N; i++ {
		y := x
		_ = y
	}
}
func BenchmarkCopyFat24(b *testing.B) {
	var x [24 / 4]uint32
	for i := 0; i < b.N; i++ {
		y := x
		_ = y
	}
}
func BenchmarkCopyFat32(b *testing.B) {
	var x [32 / 4]uint32
	for i := 0; i < b.N; i++ {
		y := x
		_ = y
	}
}
func BenchmarkCopyFat64(b *testing.B) {
	var x [64 / 4]uint32
	for i := 0; i < b.N; i++ {
		y := x
		_ = y
	}
}
func BenchmarkCopyFat128(b *testing.B) {
	var x [128 / 4]uint32
	for i := 0; i < b.N; i++ {
		y := x
		_ = y
	}
}
func BenchmarkCopyFat256(b *testing.B) {
	var x [256 / 4]uint32
	for i := 0; i < b.N; i++ {
		y := x
		_ = y
	}
}
func BenchmarkCopyFat512(b *testing.B) {
	var x [512 / 4]uint32
	for i := 0; i < b.N; i++ {
		y := x
		_ = y
	}
}
func BenchmarkCopyFat1024(b *testing.B) {
	var x [1024 / 4]uint32
	for i := 0; i < b.N; i++ {
		y := x
		_ = y
	}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Fork, exec, wait, etc.

package syscall

import (
	"sync"
	"unsafe"
	"utf16"
)

var ForkLock sync.RWMutex

// escape rewrites command line argument s as prescribed
// in http://msdn.microsoft.com/en-us/library/ms880421.
// This function returns "" (2 double quotes) if s is empty.
// Alternatively, these transformations are done:
// - every back slash (\) is doubled, but only if immediately
//   followed by double quote (");
// - every double quote (") is escaped by back slash (\);
// - finally, s is wrapped with double quotes (arg -> "arg"),
//   but only if there is space or tab inside s.
func escape(s string) string {
	if len(s) == 0 {
		return "\"\""
	}
	n := len(s)
	hasSpace := false
	for i := 0; i < len(s); i++ {
		switch s[i] {
		case '"', '\\':
			n++
		case ' ', '\t':
			hasSpace = true
		}
	}
	if hasSpace {
		n += 2
	}
	if n == len(s) {
		return s
	}

	qs := make([]byte, n)
	j := 0
	if hasSpace {
		qs[j] = '"'
		j++
	}
	slashes := 0
	for i := 0; i < len(s); i++ {
		switch s[i] {
		default:
			slashes = 0
			qs[j] = s[i]
		case '\\':
			slashes++
			qs[j] = s[i]
		case '"':
			for ; slashes > 0; slashes-- {
				qs[j] = '\\'
				j++
			}
			qs[j] = '\\'
			j++
			qs[j] = s[i]
		}
		j++
	}
	if hasSpace {
		for ; slashes > 0; slashes-- {
			qs[j] = '\\'
			j++
		}
		qs[j] = '"'
		j++
	}
	return string(qs[:j])
}

// makeCmdLine builds a command line out of args by escaping "special"
// characters and joining the arguments with spaces.
func makeCmdLine(args []string) string {
	var s string
	for _, v := range args {
		if s != "" {
			s += " "
		}
		s += escape(v)
	}
	return s
}

// createEnvBlock converts an array of environment strings into
// the representation required by CreateProcess: a sequence of NUL
// terminated strings followed by a nil.
// Last bytes are two UCS-2 NULs, or four NUL bytes.
func createEnvBlock(envv []string) *uint16 {
	if len(envv) == 0 {
		return &utf16.Encode([]int("\x00\x00"))[0]
	}
	length := 0
	for _, s := range envv {
		length += len(s) + 1
	}
	length += 1

	b := make([]byte, length)
	i := 0
	for _, s := range envv {
		l := len(s)
		copy(b[i:i+l], []byte(s))
		copy(b[i+l:i+l+1], []byte{0})
		i = i + l + 1
	}
	copy(b[i:i+1], []byte{0})

	return &utf16.Encode([]int(string(b)))[0]
}

func CloseOnExec(fd int) {
	SetHandleInformation(int32(fd), HANDLE_FLAG_INHERIT, 0)
}

func SetNonblock(fd int, nonblocking bool) (errno int) {
	return 0
}

// getFullPath retrieves the full path of the specified file.
// Just a wrapper for Windows GetFullPathName api.
func getFullPath(name string) (path string, err int) {
	p := StringToUTF16Ptr(name)
	buf := make([]uint16, 100)
	n, err := GetFullPathName(p, uint32(len(buf)), &buf[0], nil)
	if err != 0 {
		return "", err
	}
	if n > uint32(len(buf)) {
		// Windows is asking for bigger buffer.
		buf = make([]uint16, n)
		n, err = GetFullPathName(p, uint32(len(buf)), &buf[0], nil)
		if err != 0 {
			return "", err
		}
		if n > uint32(len(buf)) {
			return "", EINVAL
		}
	}
	return UTF16ToString(buf[:n]), 0
}

func isSlash(c uint8) bool {
	return c == '\\' || c == '/'
}

func normalizeDir(dir string) (name string, err int) {
	ndir, err := getFullPath(dir)
	if err != 0 {
		return "", err
	}
	if len(ndir) > 2 && isSlash(ndir[0]) && isSlash(ndir[1]) {
		// dir cannot have \\server\share\path form
		return "", EINVAL
	}
	return ndir, 0
}

func volToUpper(ch int) int {
	if 'a' <= ch && ch <= 'z' {
		ch += 'A' - 'a'
	}
	return ch
}

func joinExeDirAndFName(dir, p string) (name string, err int) {
	if len(p) == 0 {
		return "", EINVAL
	}
	if len(p) > 2 && isSlash(p[0]) && isSlash(p[1]) {
		// \\server\share\path form
		return p, 0
	}
	if len(p) > 1 && p[1] == ':' {
		// has drive letter
		if len(p) == 2 {
			return "", EINVAL
		}
		if isSlash(p[2]) {
			return p, 0
		} else {
			d, err := normalizeDir(dir)
			if err != 0 {
				return "", err
			}
			if volToUpper(int(p[0])) == volToUpper(int(d[0])) {
				return getFullPath(d + "\\" + p[2:])
			} else {
				return getFullPath(p)
			}
		}
	} else {
		// no drive letter
		d, err := normalizeDir(dir)
		if err != 0 {
			return "", err
		}
		if isSlash(p[0]) {
			return getFullPath(d[:2] + p)
		} else {
			return getFullPath(d + "\\" + p)
		}
	}
	// we shouldn't be here
	return "", EINVAL
}

type ProcAttr struct {
	Dir        string
	Env        []string
	Files      []int
	HideWindow bool
}

var zeroAttributes ProcAttr

func StartProcess(argv0 string, argv []string, attr *ProcAttr) (pid, handle int, err int) {
	if len(argv0) == 0 {
		return 0, 0, EWINDOWS
	}
	if attr == nil {
		attr = &zeroAttributes
	}
	if len(attr.Files) > 3 {
		return 0, 0, EWINDOWS
	}

	if len(attr.Dir) != 0 {
		// StartProcess assumes that argv0 is relative to attr.Dir,
		// because it implies Chdir(attr.Dir) before executing argv0.
		// Windows CreateProcess assumes the opposite: it looks for
		// argv0 relative to the current directory, and, only once the new
		// process is started, it does Chdir(attr.Dir). We are adjusting
		// for that difference here by making argv0 absolute.
		var err int
		argv0, err = joinExeDirAndFName(attr.Dir, argv0)
		if err != 0 {
			return 0, 0, err
		}
	}
	argv0p := StringToUTF16Ptr(argv0)

	var argvp *uint16
	s := makeCmdLine(argv)
	if len(s) != 0 {
		argvp = StringToUTF16Ptr(s)
	}

	var dirp *uint16
	if len(attr.Dir) != 0 {
		dirp = StringToUTF16Ptr(attr.Dir)
	}

	// Acquire the fork lock so that no other threads
	// create new fds that are not yet close-on-exec
	// before we fork.
	ForkLock.Lock()
	defer ForkLock.Unlock()

	p, _ := GetCurrentProcess()
	fd := make([]int32, len(attr.Files))
	for i := range attr.Files {
		if attr.Files[i] > 0 {
			err := DuplicateHandle(p, int32(attr.Files[i]), p, &fd[i], 0, true, DUPLICATE_SAME_ACCESS)
			if err != 0 {
				return 0, 0, err
			}
			defer CloseHandle(int32(fd[i]))
		}
	}
	si := new(StartupInfo)
	si.Cb = uint32(unsafe.Sizeof(*si))
	si.Flags = STARTF_USESTDHANDLES
	if attr.HideWindow {
		si.Flags |= STARTF_USESHOWWINDOW
		si.ShowWindow = SW_HIDE
	}
	si.StdInput = fd[0]
	si.StdOutput = fd[1]
	si.StdErr = fd[2]

	pi := new(ProcessInformation)

	err = CreateProcess(argv0p, argvp, nil, nil, true, CREATE_UNICODE_ENVIRONMENT, createEnvBlock(attr.Env), dirp, si, pi)
	if err != 0 {
		return 0, 0, err
	}
	defer CloseHandle(pi.Thread)

	return int(pi.ProcessId), int(pi.Process), 0
}

func Exec(argv0 string, argv []string, envv []string) (err int) {
	return EWINDOWS
}
<commit_msg>syscall : add ProcAttr field to pass an unescaped command line on windows<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Fork, exec, wait, etc.

package syscall

import (
	"sync"
	"unsafe"
	"utf16"
)

var ForkLock sync.RWMutex

// EscapeArg rewrites command line argument s as prescribed
// in http://msdn.microsoft.com/en-us/library/ms880421.
// This function returns "" (2 double quotes) if s is empty.
// Alternatively, these transformations are done:
// - every back slash (\) is doubled, but only if immediately
//   followed by double quote (");
// - every double quote (") is escaped by back slash (\);
// - finally, s is wrapped with double quotes (arg -> "arg"),
//   but only if there is space or tab inside s.
func EscapeArg(s string) string {
	if len(s) == 0 {
		return "\"\""
	}
	n := len(s)
	hasSpace := false
	for i := 0; i < len(s); i++ {
		switch s[i] {
		case '"', '\\':
			n++
		case ' ', '\t':
			hasSpace = true
		}
	}
	if hasSpace {
		n += 2
	}
	if n == len(s) {
		return s
	}

	qs := make([]byte, n)
	j := 0
	if hasSpace {
		qs[j] = '"'
		j++
	}
	slashes := 0
	for i := 0; i < len(s); i++ {
		switch s[i] {
		default:
			slashes = 0
			qs[j] = s[i]
		case '\\':
			slashes++
			qs[j] = s[i]
		case '"':
			for ; slashes > 0; slashes-- {
				qs[j] = '\\'
				j++
			}
			qs[j] = '\\'
			j++
			qs[j] = s[i]
		}
		j++
	}
	if hasSpace {
		for ; slashes > 0; slashes-- {
			qs[j] = '\\'
			j++
		}
		qs[j] = '"'
		j++
	}
	return string(qs[:j])
}

// makeCmdLine builds a command line out of args by escaping "special"
// characters and joining the arguments with spaces.
func makeCmdLine(args []string) string {
	var s string
	for _, v := range args {
		if s != "" {
			s += " "
		}
		s += EscapeArg(v)
	}
	return s
}

// createEnvBlock converts an array of environment strings into
// the representation required by CreateProcess: a sequence of NUL
// terminated strings followed by a nil.
// Last bytes are two UCS-2 NULs, or four NUL bytes.
func createEnvBlock(envv []string) *uint16 {
	if len(envv) == 0 {
		return &utf16.Encode([]int("\x00\x00"))[0]
	}
	length := 0
	for _, s := range envv {
		length += len(s) + 1
	}
	length += 1

	b := make([]byte, length)
	i := 0
	for _, s := range envv {
		l := len(s)
		copy(b[i:i+l], []byte(s))
		copy(b[i+l:i+l+1], []byte{0})
		i = i + l + 1
	}
	copy(b[i:i+1], []byte{0})

	return &utf16.Encode([]int(string(b)))[0]
}

func CloseOnExec(fd int) {
	SetHandleInformation(int32(fd), HANDLE_FLAG_INHERIT, 0)
}

func SetNonblock(fd int, nonblocking bool) (errno int) {
	return 0
}

// getFullPath retrieves the full path of the specified file.
// Just a wrapper for Windows GetFullPathName api.
func getFullPath(name string) (path string, err int) {
	p := StringToUTF16Ptr(name)
	buf := make([]uint16, 100)
	n, err := GetFullPathName(p, uint32(len(buf)), &buf[0], nil)
	if err != 0 {
		return "", err
	}
	if n > uint32(len(buf)) {
		// Windows is asking for bigger buffer.
		buf = make([]uint16, n)
		n, err = GetFullPathName(p, uint32(len(buf)), &buf[0], nil)
		if err != 0 {
			return "", err
		}
		if n > uint32(len(buf)) {
			return "", EINVAL
		}
	}
	return UTF16ToString(buf[:n]), 0
}

func isSlash(c uint8) bool {
	return c == '\\' || c == '/'
}

func normalizeDir(dir string) (name string, err int) {
	ndir, err := getFullPath(dir)
	if err != 0 {
		return "", err
	}
	if len(ndir) > 2 && isSlash(ndir[0]) && isSlash(ndir[1]) {
		// dir cannot have \\server\share\path form
		return "", EINVAL
	}
	return ndir, 0
}

func volToUpper(ch int) int {
	if 'a' <= ch && ch <= 'z' {
		ch += 'A' - 'a'
	}
	return ch
}

func joinExeDirAndFName(dir, p string) (name string, err int) {
	if len(p) == 0 {
		return "", EINVAL
	}
	if len(p) > 2 && isSlash(p[0]) && isSlash(p[1]) {
		// \\server\share\path form
		return p, 0
	}
	if len(p) > 1 && p[1] == ':' {
		// has drive letter
		if len(p) == 2 {
			return "", EINVAL
		}
		if isSlash(p[2]) {
			return p, 0
		} else {
			d, err := normalizeDir(dir)
			if err != 0 {
				return "", err
			}
			if volToUpper(int(p[0])) == volToUpper(int(d[0])) {
				return getFullPath(d + "\\" + p[2:])
			} else {
				return getFullPath(p)
			}
		}
	} else {
		// no drive letter
		d, err := normalizeDir(dir)
		if err != 0 {
			return "", err
		}
		if isSlash(p[0]) {
			return getFullPath(d[:2] + p)
		} else {
			return getFullPath(d + "\\" + p)
		}
	}
	// we shouldn't be here
	return "", EINVAL
}

type ProcAttr struct {
	Dir        string
	Env        []string
	Files      []int
	HideWindow bool
	CmdLine    string // used if non-empty, else the windows command line is built by escaping the arguments passed to StartProcess
}

var zeroAttributes ProcAttr

func StartProcess(argv0 string, argv []string, attr *ProcAttr) (pid, handle int, err int) {
	if len(argv0) == 0 {
		return 0, 0, EWINDOWS
	}
	if attr == nil {
		attr = &zeroAttributes
	}
	if len(attr.Files) > 3 {
		return 0, 0, EWINDOWS
	}

	if len(attr.Dir) != 0 {
		// StartProcess assumes that argv0 is relative to attr.Dir,
		// because it implies Chdir(attr.Dir) before executing argv0.
		// Windows CreateProcess assumes the opposite: it looks for
		// argv0 relative to the current directory, and, only once the new
		// process is started, it does Chdir(attr.Dir). We are adjusting
		// for that difference here by making argv0 absolute.
		var err int
		argv0, err = joinExeDirAndFName(attr.Dir, argv0)
		if err != 0 {
			return 0, 0, err
		}
	}
	argv0p := StringToUTF16Ptr(argv0)

	var cmdline string
	// Windows CreateProcess takes the command line as a single string:
	// use attr.CmdLine if set, else build the command line by escaping
	// and joining each argument with spaces
	if attr.CmdLine != "" {
		cmdline = attr.CmdLine
	} else {
		cmdline = makeCmdLine(argv)
	}

	var argvp *uint16
	if len(cmdline) != 0 {
		argvp = StringToUTF16Ptr(cmdline)
	}

	var dirp *uint16
	if len(attr.Dir) != 0 {
		dirp = StringToUTF16Ptr(attr.Dir)
	}

	// Acquire the fork lock so that no other threads
	// create new fds that are not yet close-on-exec
	// before we fork.
	ForkLock.Lock()
	defer ForkLock.Unlock()

	p, _ := GetCurrentProcess()
	fd := make([]int32, len(attr.Files))
	for i := range attr.Files {
		if attr.Files[i] > 0 {
			err := DuplicateHandle(p, int32(attr.Files[i]), p, &fd[i], 0, true, DUPLICATE_SAME_ACCESS)
			if err != 0 {
				return 0, 0, err
			}
			defer CloseHandle(int32(fd[i]))
		}
	}
	si := new(StartupInfo)
	si.Cb = uint32(unsafe.Sizeof(*si))
	si.Flags = STARTF_USESTDHANDLES
	if attr.HideWindow {
		si.Flags |= STARTF_USESHOWWINDOW
		si.ShowWindow = SW_HIDE
	}
	si.StdInput = fd[0]
	si.StdOutput = fd[1]
	si.StdErr = fd[2]

	pi := new(ProcessInformation)

	err = CreateProcess(argv0p, argvp, nil, nil, true, CREATE_UNICODE_ENVIRONMENT, createEnvBlock(attr.Env), dirp, si, pi)
	if err != 0 {
		return 0, 0, err
	}
	defer CloseHandle(pi.Thread)

	return int(pi.ProcessId), int(pi.Process), 0
}

func Exec(argv0 string, argv []string, envv []string) (err int) {
	return EWINDOWS
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/grafana/grafana/pkg/metric/helper"
	"testing"
)

type point struct {
	ts  uint32
	val float64
}

func (p point) String() string {
	return fmt.Sprintf("point{%0.f at %d}", p.val, p.ts)
}

type Checker struct {
	t      *testing.T
	agg    *AggMetric
	points []point
}

func NewChecker(t *testing.T, agg *AggMetric) *Checker {
	return &Checker{t, agg, make([]point, 0)}
}

// always add points in ascending order, never same ts!
func (c *Checker) Add(ts uint32, val float64) {
	c.agg.Add(ts, val)
	c.points = append(c.points, point{ts, val})
}

// from to is the range that gets requested from AggMetric
// first/last is what we use as data range to compare to (both inclusive)
// these may be different because AggMetric returns broader rangers (due to packed format),
func (c *Checker) Verify(from, to, first, last uint32) {
	_, iters := c.agg.Get(from, to)
	// we don't do checking or fancy logic, it is assumed that the caller made sure first and last are ts of actual points
	var pi int // index of first point we want
	var pj int // index of last point we want
	for pi = 0; c.points[pi].ts != first; pi++ {
	}
	for pj = pi; c.points[pj].ts != last; pj++ {
	}
	c.t.Logf("verifying AggMetric.Get(%d,%d) =?= %d <= ts <= %d", from, to, first, last)
	index := pi - 1
	for _, iter := range iters {
		for iter.Next() {
			index++
			tt, vv := iter.Values()
			//c.t.Logf("got (%v,%v).. should be (%v,%v)", tt, vv, c.points[index].ts, c.points[index].val)
			if index > pj {
				c.t.Fatalf("Values()=(%v,%v), want end of stream\n", tt, vv)
			}
			if c.points[index].ts != tt || c.points[index].val != vv {
				c.t.Fatalf("Values()=(%v,%v), want (%v,%v)\n", tt, vv, c.points[index].ts, c.points[index].val)
			}
		}
	}
	if index != pj {
		c.t.Fatalf("not all values returned. missing %v", c.points[index:pj+1])
	}
}

func TestAggMetric(t *testing.T) {
	stats, _ := helper.New(false, "", "standard", "metrics_tank", "")
	initMetrics(stats)

	c := NewChecker(t, NewAggMetric("foo", 100, 5, []aggSetting{}...))

	// basic case, single range
	c.Add(101, 101)
	c.Verify(100, 200, 101, 101)
	c.Add(105, 105)
	c.Verify(100, 199, 101, 105)
	c.Add(115, 115)
	c.Add(125, 125)
	c.Add(135, 135)
	c.Verify(100, 199, 101, 135)

	// add new ranges, aligned and unaligned
	c.Add(200, 200)
	c.Add(315, 315)
	c.Verify(100, 399, 101, 315)

	// get subranges
	c.Verify(120, 299, 101, 200)
	c.Verify(220, 299, 200, 200)
	c.Verify(312, 330, 315, 315)

	// border dancing. good for testing inclusivity and exclusivity
	c.Verify(100, 199, 101, 135)
	c.Verify(100, 200, 101, 135)
	c.Verify(100, 201, 101, 200)
	c.Verify(198, 199, 101, 135)
	c.Verify(199, 200, 101, 135)
	c.Verify(200, 201, 200, 200)
	c.Verify(201, 202, 200, 200)
	c.Verify(299, 300, 200, 200)
	c.Verify(300, 301, 315, 315)

	// skipping
	c.Add(510, 510)
	c.Add(512, 512)
	c.Verify(100, 599, 101, 512)

	// basic wraparound
	c.Add(610, 610)
	c.Add(612, 612)
	c.Add(710, 710)
	c.Add(712, 712)
	// TODO would be nice to test that it panics when requesting old range. something with recover?
	//c.Verify(100, 799, 101, 512)

	// largest range we have so far
	c.Verify(300, 799, 315, 712)
	// a smaller range
	c.Verify(502, 799, 510, 712)

	// the circular buffer had these ranges:
	// 100 200 300 skipped 500
	// then we made it:
	// 600 700 300 skipped 500
	// now we want to do another wrap around with skip (must have cleared old data)
	// let's jump to 1200. the accessible range should then be 800-1200
	// clea 1200 clea clea clea
	// we can't (and shouldn't, due to abstraction) test the clearing itself
	// but we just check we only get this point
	c.Add(1299, 1299)
	// TODO: implement skips and enable this
	//	c.Verify(800, 1299, 1299, 1299)
}

// basic expected RAM usage for 1 iteration (= 1 days)
// 1000 metrics * (3600 * 24 / 10 ) points per metric * 1.3 B/point = 11 MB
// 1000 metrics * 5 agg metrics per metric * (3600 * 24 / 300) points per aggmetric * 1.3B/point = 1.9 MB
// total -> 13 MB
// go test -run=XX -bench=Bench -benchmem -v -memprofile mem.out
// go tool pprof -inuse_space metrics_tank.test mem.out -> shows 25 MB in use

// TODO update once we clean old data, then we should look at numChunks
func BenchmarkAggMetrics1000Metrics1Day(b *testing.B) {
	// we will store 10s metrics in 5 chunks of 2 hours
	// aggragate them in 5min buckets, stored in 1 chunk of 24hours
	chunkSpan := uint32(2 * 3600)
	numChunks := uint32(5)
	//aggSpan := uint32(300)
	//aggChunkSpan := uint32(24 * 3600)
	//numAggChunks := uint32(1)
	chunkMaxStale := uint32(3600)
	metricMaxStale := uint32(21600)

	keys := make([]string, 1000)
	for i := 0; i < 1000; i++ {
		keys[i] = fmt.Sprintf("hello.this.is.a.test.key.%d", i)
	}

	metrics := NewAggMetrics(chunkSpan, numChunks, chunkMaxStale, metricMaxStale, []aggSetting{})
	maxT := 3600 * 24 * uint32(b.N) // b.N in days
	for t := uint32(1); t < maxT; t += 10 {
		for metricI := 0; metricI < 1000; metricI++ {
			m := metrics.GetOrCreate(keys[metricI])
			m.Add(t, float64(t))
		}
	}
}
<commit_msg>make aggregation benchmark actually do aggregations again<commit_after>package main

import (
	"fmt"
	"github.com/grafana/grafana/pkg/metric/helper"
	"testing"
)

type point struct {
	ts  uint32
	val float64
}

func (p point) String() string {
	return fmt.Sprintf("point{%0.f at %d}", p.val, p.ts)
}

type Checker struct {
	t      *testing.T
	agg    *AggMetric
	points []point
}

func NewChecker(t *testing.T, agg *AggMetric) *Checker {
	return &Checker{t, agg, make([]point, 0)}
}

// always add points in ascending order, never same ts!
func (c *Checker) Add(ts uint32, val float64) {
	c.agg.Add(ts, val)
	c.points = append(c.points, point{ts, val})
}

// from to is the range that gets requested from AggMetric
// first/last is what we use as data range to compare to (both inclusive)
// these may be different because AggMetric returns broader rangers (due to packed format),
func (c *Checker) Verify(from, to, first, last uint32) {
	_, iters := c.agg.Get(from, to)
	// we don't do checking or fancy logic, it is assumed that the caller made sure first and last are ts of actual points
	var pi int // index of first point we want
	var pj int // index of last point we want
	for pi = 0; c.points[pi].ts != first; pi++ {
	}
	for pj = pi; c.points[pj].ts != last; pj++ {
	}
	c.t.Logf("verifying AggMetric.Get(%d,%d) =?= %d <= ts <= %d", from, to, first, last)
	index := pi - 1
	for _, iter := range iters {
		for iter.Next() {
			index++
			tt, vv := iter.Values()
			//c.t.Logf("got (%v,%v).. should be (%v,%v)", tt, vv, c.points[index].ts, c.points[index].val)
			if index > pj {
				c.t.Fatalf("Values()=(%v,%v), want end of stream\n", tt, vv)
			}
			if c.points[index].ts != tt || c.points[index].val != vv {
				c.t.Fatalf("Values()=(%v,%v), want (%v,%v)\n", tt, vv, c.points[index].ts, c.points[index].val)
			}
		}
	}
	if index != pj {
		c.t.Fatalf("not all values returned. missing %v", c.points[index:pj+1])
	}
}

func TestAggMetric(t *testing.T) {
	stats, _ := helper.New(false, "", "standard", "metrics_tank", "")
	initMetrics(stats)

	c := NewChecker(t, NewAggMetric("foo", 100, 5, []aggSetting{}...))

	// basic case, single range
	c.Add(101, 101)
	c.Verify(100, 200, 101, 101)
	c.Add(105, 105)
	c.Verify(100, 199, 101, 105)
	c.Add(115, 115)
	c.Add(125, 125)
	c.Add(135, 135)
	c.Verify(100, 199, 101, 135)

	// add new ranges, aligned and unaligned
	c.Add(200, 200)
	c.Add(315, 315)
	c.Verify(100, 399, 101, 315)

	// get subranges
	c.Verify(120, 299, 101, 200)
	c.Verify(220, 299, 200, 200)
	c.Verify(312, 330, 315, 315)

	// border dancing. good for testing inclusivity and exclusivity
	c.Verify(100, 199, 101, 135)
	c.Verify(100, 200, 101, 135)
	c.Verify(100, 201, 101, 200)
	c.Verify(198, 199, 101, 135)
	c.Verify(199, 200, 101, 135)
	c.Verify(200, 201, 200, 200)
	c.Verify(201, 202, 200, 200)
	c.Verify(299, 300, 200, 200)
	c.Verify(300, 301, 315, 315)

	// skipping
	c.Add(510, 510)
	c.Add(512, 512)
	c.Verify(100, 599, 101, 512)

	// basic wraparound
	c.Add(610, 610)
	c.Add(612, 612)
	c.Add(710, 710)
	c.Add(712, 712)
	// TODO would be nice to test that it panics when requesting old range. something with recover?
	//c.Verify(100, 799, 101, 512)

	// largest range we have so far
	c.Verify(300, 799, 315, 712)
	// a smaller range
	c.Verify(502, 799, 510, 712)

	// the circular buffer had these ranges:
	// 100 200 300 skipped 500
	// then we made it:
	// 600 700 300 skipped 500
	// now we want to do another wrap around with skip (must have cleared old data)
	// let's jump to 1200. the accessible range should then be 800-1200
	// clea 1200 clea clea clea
	// we can't (and shouldn't, due to abstraction) test the clearing itself
	// but we just check we only get this point
	c.Add(1299, 1299)
	// TODO: implement skips and enable this
	//	c.Verify(800, 1299, 1299, 1299)
}

// basic expected RAM usage for 1 iteration (= 1 days)
// 1000 metrics * (3600 * 24 / 10 ) points per metric * 1.3 B/point = 11 MB
// 1000 metrics * 5 agg metrics per metric * (3600 * 24 / 300) points per aggmetric * 1.3B/point = 1.9 MB
// total -> 13 MB
// go test -run=XX -bench=Bench -benchmem -v -memprofile mem.out
// go tool pprof -inuse_space metrics_tank.test mem.out -> shows 25 MB in use

// TODO update once we clean old data, then we should look at numChunks
func BenchmarkAggMetrics1000Metrics1Day(b *testing.B) {
	// we will store 10s metrics in 5 chunks of 2 hours
	// aggragate them in 5min buckets, stored in 1 chunk of 24hours
	chunkSpan := uint32(2 * 3600)
	numChunks := uint32(5)
	chunkMaxStale := uint32(3600)
	metricMaxStale := uint32(21600)
	aggSettings := []aggSetting{
		{
			span:      uint32(300),
			chunkSpan: uint32(24 * 3600),
			numChunks: uint32(1),
		},
	}

	keys := make([]string, 1000)
	for i := 0; i < 1000; i++ {
		keys[i] = fmt.Sprintf("hello.this.is.a.test.key.%d", i)
	}

	metrics := NewAggMetrics(chunkSpan, numChunks, chunkMaxStale, metricMaxStale, aggSettings)

	maxT := 3600 * 24 * uint32(b.N) // b.N in days
	for t := uint32(1); t < maxT; t += 10 {
		for metricI := 0; metricI < 1000; metricI++ {
			m := metrics.GetOrCreate(keys[metricI])
			m.Add(t, float64(t))
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcs

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"sync"
	"sync/atomic"
	"time"

	"golang.org/x/net/context"
)

var fDebug = flag.Bool(
	"gcs.debug",
	false,
	"Write FUSE debugging messages to stderr.")

var gLogger *log.Logger
var gLoggerOnce sync.Once

func initLogger() {
	const flags = log.Ldate | log.Ltime | log.Lmicroseconds
	gLogger = log.New(os.Stderr, "gcs: ", flags)
}

// If debugging is enabled, wrap the supplied bucket in a layer that prints
// debug messages.
func newDebugBucket(wrapped Bucket) (b Bucket) {
	b = wrapped

	if *fDebug {
		gLoggerOnce.Do(initLogger)
		b = &debugBucket{
			logger:  gLogger,
			wrapped: b,
		}

		return
	}

	return
}

type debugBucket struct {
	logger  *log.Logger
	wrapped Bucket

	nextRequestID uint64
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func (b *debugBucket) mintRequestID() (id uint64) {
	id = atomic.AddUint64(&b.nextRequestID, 1) - 1
	return
}

func (b *debugBucket) requestLogf(
	id uint64,
	format string,
	v ...interface{}) {
	b.logger.Printf("Req %#16x: %s", id, fmt.Sprintf(format, v...))
}

func (b *debugBucket) startRequest(
	format string,
	v ...interface{}) (id uint64, desc string, start time.Time) {
	start = time.Now()
	id = b.mintRequestID()
	desc = fmt.Sprintf(format, v...)

	b.requestLogf(id, "<- %s", desc)
	return
}

func (b *debugBucket) finishRequest(
	id uint64,
	desc string,
	start time.Time,
	err *error) {
	duration := time.Since(start)

	errDesc := "OK"
	if *err != nil {
		errDesc = (*err).Error()
	}

	b.requestLogf(id, "-> %s (%v): %s", desc, duration, errDesc)
}

////////////////////////////////////////////////////////////////////////
// Reader
////////////////////////////////////////////////////////////////////////

type debugReader struct {
	bucket    *debugBucket
	requestID uint64
	desc      string
	startTime time.Time
	wrapped   io.ReadCloser
}

func (dr *debugReader) Read(p []byte) (n int, err error) {
	n, err = dr.wrapped.Read(p)
	if err != nil {
		dr.bucket.requestLogf(dr.requestID, "-> Read error: %v", err)
	}

	return
}

func (dr *debugReader) Close() (err error) {
	defer dr.bucket.finishRequest(
		dr.requestID,
		dr.desc,
		dr.startTime,
		&err)

	err = dr.wrapped.Close()
	return
}

////////////////////////////////////////////////////////////////////////
// Bucket interface
////////////////////////////////////////////////////////////////////////

func (b *debugBucket) Name() string {
	return b.wrapped.Name()
}

func (b *debugBucket) NewReader(
	ctx context.Context,
	req *ReadObjectRequest) (rc io.ReadCloser, err error) {
	id, desc, start := b.startRequest("Read(%q, %v)", req.Name, req.Range)

	// Call through.
	rc, err = b.wrapped.NewReader(ctx, req)
	if err != nil {
		b.finishRequest(id, desc, start, &err)
		return
	}

	// Return a special reader that prings debug info.
	rc = &debugReader{
		bucket:    b,
		requestID: id,
		desc:      desc,
		startTime: start,
		wrapped:   rc,
	}

	return
}

func (b *debugBucket) CreateObject(
	ctx context.Context,
	req *CreateObjectRequest) (o *Object, err error) {
	id, desc, start := b.startRequest("CreateObject(%q)", req.Name)
	defer b.finishRequest(id, desc, start, &err)

	o, err = b.wrapped.CreateObject(ctx, req)
	return
}

func (b *debugBucket) CopyObject(
	ctx context.Context,
	req *CopyObjectRequest) (o *Object, err error) {
	id, desc, start := b.startRequest(
		"CopyObject(%q, %q)",
		req.SrcName,
		req.DstName)

	defer b.finishRequest(id, desc, start, &err)

	o, err = b.wrapped.CopyObject(ctx, req)
	return
}

func (b *debugBucket) ComposeObjects(
	ctx context.Context,
	req *ComposeObjectsRequest) (o *Object, err error) {
	id, desc, start := b.startRequest(
		"ComposeObjects(%q)",
		req.DstName)

	defer b.finishRequest(id, desc, start, &err)

	o, err = b.wrapped.ComposeObjects(ctx, req)
	return
}

func (b *debugBucket) StatObject(
	ctx context.Context,
	req *StatObjectRequest) (o *Object, err error) {
	id, desc, start := b.startRequest("StatObject(%q)", req.Name)
	defer b.finishRequest(id, desc, start, &err)

	o, err = b.wrapped.StatObject(ctx, req)
	return
}

func (b *debugBucket) ListObjects(
	ctx context.Context,
	req *ListObjectsRequest) (listing *Listing, err error) {
	id, desc, start := b.startRequest("ListObjects()")
	defer b.finishRequest(id, desc, start, &err)

	listing, err = b.wrapped.ListObjects(ctx, req)
	return
}

func (b *debugBucket) UpdateObject(
	ctx context.Context,
	req *UpdateObjectRequest) (o *Object, err error) {
	id, desc, start := b.startRequest("UpdateObject(%q)", req.Name)
	defer b.finishRequest(id, desc, start, &err)

	o, err = b.wrapped.UpdateObject(ctx, req)
	return
}

func (b *debugBucket) DeleteObject(
	ctx context.Context,
	name string) (err error) {
	id, desc, start := b.startRequest("DeleteObject(%q)", name)
	defer b.finishRequest(id, desc, start, &err)

	err = b.wrapped.DeleteObject(ctx, name)
	return
}
<commit_msg>Don't log io.EOF error in --gcs.debug output.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcs

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"sync"
	"sync/atomic"
	"time"

	"golang.org/x/net/context"
)

var fDebug = flag.Bool(
	"gcs.debug",
	false,
	"Write FUSE debugging messages to stderr.")

var gLogger *log.Logger
var gLoggerOnce sync.Once

func initLogger() {
	const flags = log.Ldate | log.Ltime | log.Lmicroseconds
	gLogger = log.New(os.Stderr, "gcs: ", flags)
}

// If debugging is enabled, wrap the supplied bucket in a layer that prints
// debug messages.
func newDebugBucket(wrapped Bucket) (b Bucket) {
	b = wrapped

	if *fDebug {
		gLoggerOnce.Do(initLogger)
		b = &debugBucket{
			logger:  gLogger,
			wrapped: b,
		}

		return
	}

	return
}

type debugBucket struct {
	logger  *log.Logger
	wrapped Bucket

	nextRequestID uint64
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func (b *debugBucket) mintRequestID() (id uint64) {
	id = atomic.AddUint64(&b.nextRequestID, 1) - 1
	return
}

func (b *debugBucket) requestLogf(
	id uint64,
	format string,
	v ...interface{}) {
	b.logger.Printf("Req %#16x: %s", id, fmt.Sprintf(format, v...))
}

func (b *debugBucket) startRequest(
	format string,
	v ...interface{}) (id uint64, desc string, start time.Time) {
	start = time.Now()
	id = b.mintRequestID()
	desc = fmt.Sprintf(format, v...)

	b.requestLogf(id, "<- %s", desc)
	return
}

func (b *debugBucket) finishRequest(
	id uint64,
	desc string,
	start time.Time,
	err *error) {
	duration := time.Since(start)

	errDesc := "OK"
	if *err != nil {
		errDesc = (*err).Error()
	}

	b.requestLogf(id, "-> %s (%v): %s", desc, duration, errDesc)
}

////////////////////////////////////////////////////////////////////////
// Reader
////////////////////////////////////////////////////////////////////////

type debugReader struct {
	bucket    *debugBucket
	requestID uint64
	desc      string
	startTime time.Time
	wrapped   io.ReadCloser
}

func (dr *debugReader) Read(p []byte) (n int, err error) {
	n, err = dr.wrapped.Read(p)

	// Don't log EOF errors, which are par for the course.
	if err != nil && err != io.EOF {
		dr.bucket.requestLogf(dr.requestID, "-> Read error: %v", err)
	}

	return
}

func (dr *debugReader) Close() (err error) {
	defer dr.bucket.finishRequest(
		dr.requestID,
		dr.desc,
		dr.startTime,
		&err)

	err = dr.wrapped.Close()
	return
}

////////////////////////////////////////////////////////////////////////
// Bucket interface
////////////////////////////////////////////////////////////////////////

func (b *debugBucket) Name() string {
	return b.wrapped.Name()
}

func (b *debugBucket) NewReader(
	ctx context.Context,
	req *ReadObjectRequest) (rc io.ReadCloser, err error) {
	id, desc, start := b.startRequest("Read(%q, %v)", req.Name, req.Range)

	// Call through.
	rc, err = b.wrapped.NewReader(ctx, req)
	if err != nil {
		b.finishRequest(id, desc, start, &err)
		return
	}

	// Return a special reader that prings debug info.
	rc = &debugReader{
		bucket:    b,
		requestID: id,
		desc:      desc,
		startTime: start,
		wrapped:   rc,
	}

	return
}

func (b *debugBucket) CreateObject(
	ctx context.Context,
	req *CreateObjectRequest) (o *Object, err error) {
	id, desc, start := b.startRequest("CreateObject(%q)", req.Name)
	defer b.finishRequest(id, desc, start, &err)

	o, err = b.wrapped.CreateObject(ctx, req)
	return
}

func (b *debugBucket) CopyObject(
	ctx context.Context,
	req *CopyObjectRequest) (o *Object, err error) {
	id, desc, start := b.startRequest(
		"CopyObject(%q, %q)",
		req.SrcName,
		req.DstName)

	defer b.finishRequest(id, desc, start, &err)

	o, err = b.wrapped.CopyObject(ctx, req)
	return
}

func (b *debugBucket) ComposeObjects(
	ctx context.Context,
	req *ComposeObjectsRequest) (o *Object, err error) {
	id, desc, start := b.startRequest(
		"ComposeObjects(%q)",
		req.DstName)

	defer b.finishRequest(id, desc, start, &err)

	o, err = b.wrapped.ComposeObjects(ctx, req)
	return
}

func (b *debugBucket) StatObject(
	ctx context.Context,
	req *StatObjectRequest) (o *Object, err error) {
	id, desc, start := b.startRequest("StatObject(%q)", req.Name)
	defer b.finishRequest(id, desc, start, &err)

	o, err = b.wrapped.StatObject(ctx, req)
	return
}

func (b *debugBucket) ListObjects(
	ctx context.Context,
	req *ListObjectsRequest) (listing *Listing, err error) {
	id, desc, start := b.startRequest("ListObjects()")
	defer b.finishRequest(id, desc, start, &err)

	listing, err = b.wrapped.ListObjects(ctx, req)
	return
}

func (b *debugBucket) UpdateObject(
	ctx context.Context,
	req *UpdateObjectRequest) (o *Object, err error) {
	id, desc, start := b.startRequest("UpdateObject(%q)", req.Name)
	defer b.finishRequest(id, desc, start, &err)

	o, err = b.wrapped.UpdateObject(ctx, req)
	return
}

func (b *debugBucket) DeleteObject(
	ctx context.Context,
	name string) (err error) {
	id, desc, start := b.startRequest("DeleteObject(%q)", name)
	defer b.finishRequest(id, desc, start, &err)

	err = b.wrapped.DeleteObject(ctx, name)
	return
}
<|endoftext|>


<commit_before>// Copyright 2012 SocialCode. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package gelf

import (
	"bytes"
	"compress/zlib"
	"encoding/json"
	"io"
	"net"
	"strings"
	"testing"
)

func TestNew(t *testing.T) {
	w, err := New("")
	if err == nil || w != nil {
		t.Errorf("New didn't fail")
		return
	}
}

// tests single-message (non-chunked) messages
func TestWriteSmall(t *testing.T) {
	addr, err := net.ResolveUDPAddr("udp", "127.0.0.1:0")
	if err != nil {
		t.Errorf("ResolveUDPAddr: %s", err)
		return
	}
	conn, err := net.ListenUDP("udp", addr)
	if err != nil {
		t.Errorf("ListenUDP: %s", err)
		return
	}

	w, err := New(conn.LocalAddr().String())
	if err != nil {
		t.Errorf("New: %s", err)
		return
	}

	msgData := []byte("awesomesauce\nbananas")
	w.Write(msgData)

	// the data we get from the wire is zlib compressed
	zBuf := make([]byte, ChunkSize)

	n, err := conn.Read(zBuf)
	if err != nil {
		t.Errorf("Read: %s", err)
		return
	}
	zBuf = zBuf[:n]

	zReader, err := zlib.NewReader(bytes.NewReader(zBuf))
	if err != nil {
		t.Errorf("zlib.NewReader: %s", err)
		return
	}

	var buf bytes.Buffer
	_, err = io.Copy(&buf, zReader)
	if err != nil {
		t.Errorf("io.Copy: %s", err)
		return
	}

	var msg Message
	if err := json.Unmarshal(buf.Bytes(), &msg); err != nil {
		t.Errorf("json.Unmarshal: %s", err)
		return
	}

	if !bytes.Equal(msg.Short, []byte("awesomesauce")) {
		t.Errorf("msg.Short: expected %s, got %s", string(msgData),
			string(msg.Full))
		return
	}

	if !bytes.Equal(msg.Full, msgData) {
		t.Errorf("msg.Full: expected %s, got %s", string(msgData),
			string(msg.Full))
		return
	}

	fileExpected := "/go-gelf/gelf/writer_test.go"
	if !strings.HasSuffix(msg.File, fileExpected) {
		t.Errorf("msg.File: expected %s, got %s", fileExpected,
			msg.File)
	}
}

func TestGetCaller(t *testing.T) {
	file, line := getCallerIgnoringLog(1000)
	if line != 0 || file != "???" {
		t.Errorf("didn't fail 1 %s %d", file, line)
		return
	}

	file, _ = getCallerIgnoringLog(0)
	if !strings.HasSuffix(file, "/gelf/writer_test.go") {
		t.Errorf("not writer_test.go? %s", file)
	}
}
<commit_msg>test: factor out the send/receive stuff<commit_after>// Copyright 2012 SocialCode. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package gelf

import (
	"bytes"
	"compress/zlib"
	"encoding/json"
	"fmt"
	"io"
	"net"
	"strings"
	"testing"
)

func TestNew(t *testing.T) {
	w, err := New("")
	if err == nil || w != nil {
		t.Errorf("New didn't fail")
		return
	}
}

func sendAndRecv(msgData []byte) (*Message, error) {
	addr, err := net.ResolveUDPAddr("udp", "127.0.0.1:0")
	if err != nil {
		return nil, fmt.Errorf("ResolveUDPAddr: %s", err)
	}

	conn, err := net.ListenUDP("udp", addr)
	if err != nil {
		return nil, fmt.Errorf("ListenUDP: %s", err)
	}

	w, err := New(conn.LocalAddr().String())
	if err != nil {
		return nil, fmt.Errorf("New: %s", err)
	}

	w.Write(msgData)

	// the data we get from the wire is zlib compressed
	zBuf := make([]byte, ChunkSize)

	n, err := conn.Read(zBuf)
	if err != nil {
		return nil, fmt.Errorf("Read: %s", err)
	}
	zBuf = zBuf[:n]

	zReader, err := zlib.NewReader(bytes.NewReader(zBuf))
	if err != nil {
		return nil, fmt.Errorf("zlib.NewReader: %s", err)
	}

	var buf bytes.Buffer
	_, err = io.Copy(&buf, zReader)
	if err != nil {
		return nil, fmt.Errorf("io.Copy: %s", err)
	}

	msg := new(Message)
	if err := json.Unmarshal(buf.Bytes(), &msg); err != nil {
		return nil, fmt.Errorf("json.Unmarshal: %s", err)
	}

	return msg, nil
}

// tests single-message (non-chunked) messages
func TestWriteSmall(t *testing.T) {
	msgData := []byte("awesomesauce\nbananas")

	msg, err := sendAndRecv(msgData)
	if err != nil {
		t.Errorf("sendAndRecv: %s", err)
		return
	}

	if !bytes.Equal(msg.Short, []byte("awesomesauce")) {
		t.Errorf("msg.Short: expected %s, got %s", string(msgData),
			string(msg.Full))
		return
	}

	if !bytes.Equal(msg.Full, msgData) {
		t.Errorf("msg.Full: expected %s, got %s", string(msgData),
			string(msg.Full))
		return
	}

	fileExpected := "/go-gelf/gelf/writer_test.go"
	if !strings.HasSuffix(msg.File, fileExpected) {
		t.Errorf("msg.File: expected %s, got %s", fileExpected,
			msg.File)
	}
}

func TestGetCaller(t *testing.T) {
	file, line := getCallerIgnoringLog(1000)
	if line != 0 || file != "???" {
		t.Errorf("didn't fail 1 %s %d", file, line)
		return
	}

	file, _ = getCallerIgnoringLog(0)
	if !strings.HasSuffix(file, "/gelf/writer_test.go") {
		t.Errorf("not writer_test.go? %s", file)
	}
}
<|endoftext|>


<commit_before>package ApkInfGo

import (
	"log"
	"os/exec"
	"strings"
	"regexp"
	"io/ioutil"
	"strconv"
	"os"
	"runtime"
	"fmt"
)
type ApkInfoSt struct {
	Name string
	VersionCode uint32
	VersionName string
	Label string
	Icon string
	SdkVersion uint16
	TargetSdkVersion uint16
	NativeCode string
	FileSize int64
	FilePath string
	Cert ApkCertSt
}

type Conf struct {
	aapt string
	cert *ConfCert
}

func ApkInfo(aaptApp string) *Conf {
	c := &Conf{aapt:aaptApp, cert:nil}
	return c
}

func (c *Conf) CertKeyTool(keytoolApp string) *Conf {
	app := ApkCertificate(keytoolApp)
	c.cert = app
	return c
}

func (c *Conf) File(apk string) *ApkInfoSt {
	d := parse(c, apk)
	if d != nil {
		file, _ := os.Stat(apk)
		d.FileSize = file.Size()
	}
	return d
}

func getLineSeparator() string{
	if runtime.GOOS == "windows" {
		return "\r\n"
	}
	return "\n"
}

func parse(c *Conf, apk string) *ApkInfoSt {
	out, err := exec.Command(c.aapt, "dump", "badging", apk).Output()
	if err != nil {
		log.Printf("err: %q, file: %q", err, apk)
		return nil
	}
	//log.Printf("apk file - %q\n", apk)
	data := strings.Split(string(out), getLineSeparator())
	info := ApkInfoSt{FilePath:apk}
	for _, s := range data{
		arr := strings.Split(s, ":")
		if len(arr) != 2 {
			//log.Printf("error split - %q\n", s)
			continue
		}
		switch arr[0] {
		case "package":
			//log.Printf("package - %q\n", arr[1])
			re := regexp.MustCompile("name='([^']+)?' versionCode='(\\d*)?' versionName='([^']+)?'")
			packageInfo := re.FindStringSubmatch(arr[1])
			info.Name = packageInfo[1]
			info.VersionName = packageInfo[3]
			versionCode, _ := strconv.ParseUint(packageInfo[2], 0, 32)
			info.VersionCode = uint32(versionCode)
			break
		case "sdkVersion":
			//log.Printf("sdkVersion - %q\n", arr[1])
			sdkVersion, _ := strconv.ParseUint(strings.Trim(arr[1], "'"), 0, 16)
			info.SdkVersion = uint16(sdkVersion)
			break
		case "targetSdkVersion":
			//log.Printf("targetSdkVersion - %q\n", arr[1])
			targetSdkVersion, _ := strconv.ParseUint(strings.Trim(arr[1], "'"), 0, 16)
			info.TargetSdkVersion = uint16(targetSdkVersion)
			break
		case "native-code":
			nativeCode := strings.Trim(strings.TrimSpace(arr[1]), "'")
			info.NativeCode = nativeCode
			break
		case "application":
			//log.Printf("application - %q\n", arr[1])
			re2 := regexp.MustCompile("label='([^']+)?' icon='([^']+)?'")
			d := re2.FindStringSubmatch(arr[1])
			info.Label = d[1]
			info.Icon = d[2]
			break
		//default:
		//	log.Printf("%q - %q\n", arr[0], arr[1])
		//	break
		}
	}
	if c.cert != nil {
		info.Cert = *c.cert.File(apk)
	}
	return &info
}

func (c *Conf) Folder(dirname string, recurcive bool) *[]ApkInfoSt {
	files, err := ioutil.ReadDir(dirname)
	if err != nil {
		log.Printf("err: %q", err)
		return nil
	}
	//log.Printf("apk folder - %q\n", dirname)
	infoArr := make([]ApkInfoSt, 0)
	re := regexp.MustCompile(".*\\.apk")
	for _, file := range files {
		if re.MatchString(file.Name()) {
			dir := dirname + string(os.PathSeparator) + file.Name()
			a := parse(c, dir)
			if a != nil {
				a.FileSize = file.Size()
				infoArr = append(infoArr, *a)
			}
		} else if file.IsDir() && recurcive {
			dir := dirname + string(os.PathSeparator) + file.Name()
			//log.Printf("apk subfolder - %q\n", dir)
			arr := (c).Folder(dir, true)
			if arr != nil {
				for _, a := range *arr {
					infoArr = append(infoArr, a)
				}
			}
		}
	}
	return &infoArr
}
<commit_msg>fix incorrect import<commit_after>package ApkInfGo

import (
	"log"
	"os/exec"
	"strings"
	"regexp"
	"io/ioutil"
	"strconv"
	"os"
	"runtime"
)
type ApkInfoSt struct {
	Name string
	VersionCode uint32
	VersionName string
	Label string
	Icon string
	SdkVersion uint16
	TargetSdkVersion uint16
	NativeCode string
	FileSize int64
	FilePath string
	Cert ApkCertSt
}

type Conf struct {
	aapt string
	cert *ConfCert
}

func ApkInfo(aaptApp string) *Conf {
	c := &Conf{aapt:aaptApp, cert:nil}
	return c
}

func (c *Conf) CertKeyTool(keytoolApp string) *Conf {
	app := ApkCertificate(keytoolApp)
	c.cert = app
	return c
}

func (c *Conf) File(apk string) *ApkInfoSt {
	d := parse(c, apk)
	if d != nil {
		file, _ := os.Stat(apk)
		d.FileSize = file.Size()
	}
	return d
}

func getLineSeparator() string{
	if runtime.GOOS == "windows" {
		return "\r\n"
	}
	return "\n"
}

func parse(c *Conf, apk string) *ApkInfoSt {
	out, err := exec.Command(c.aapt, "dump", "badging", apk).Output()
	if err != nil {
		log.Printf("err: %q, file: %q", err, apk)
		return nil
	}
	//log.Printf("apk file - %q\n", apk)
	data := strings.Split(string(out), getLineSeparator())
	info := ApkInfoSt{FilePath:apk}
	for _, s := range data{
		arr := strings.Split(s, ":")
		if len(arr) != 2 {
			//log.Printf("error split - %q\n", s)
			continue
		}
		switch arr[0] {
		case "package":
			//log.Printf("package - %q\n", arr[1])
			re := regexp.MustCompile("name='([^']+)?' versionCode='(\\d*)?' versionName='([^']+)?'")
			packageInfo := re.FindStringSubmatch(arr[1])
			info.Name = packageInfo[1]
			info.VersionName = packageInfo[3]
			versionCode, _ := strconv.ParseUint(packageInfo[2], 0, 32)
			info.VersionCode = uint32(versionCode)
			break
		case "sdkVersion":
			//log.Printf("sdkVersion - %q\n", arr[1])
			sdkVersion, _ := strconv.ParseUint(strings.Trim(arr[1], "'"), 0, 16)
			info.SdkVersion = uint16(sdkVersion)
			break
		case "targetSdkVersion":
			//log.Printf("targetSdkVersion - %q\n", arr[1])
			targetSdkVersion, _ := strconv.ParseUint(strings.Trim(arr[1], "'"), 0, 16)
			info.TargetSdkVersion = uint16(targetSdkVersion)
			break
		case "native-code":
			nativeCode := strings.Trim(strings.TrimSpace(arr[1]), "'")
			info.NativeCode = nativeCode
			break
		case "application":
			//log.Printf("application - %q\n", arr[1])
			re2 := regexp.MustCompile("label='([^']+)?' icon='([^']+)?'")
			d := re2.FindStringSubmatch(arr[1])
			info.Label = d[1]
			info.Icon = d[2]
			break
		//default:
		//	log.Printf("%q - %q\n", arr[0], arr[1])
		//	break
		}
	}
	if c.cert != nil {
		info.Cert = *c.cert.File(apk)
	}
	return &info
}

func (c *Conf) Folder(dirname string, recurcive bool) *[]ApkInfoSt {
	files, err := ioutil.ReadDir(dirname)
	if err != nil {
		log.Printf("err: %q", err)
		return nil
	}
	//log.Printf("apk folder - %q\n", dirname)
	infoArr := make([]ApkInfoSt, 0)
	re := regexp.MustCompile(".*\\.apk")
	for _, file := range files {
		if re.MatchString(file.Name()) {
			dir := dirname + string(os.PathSeparator) + file.Name()
			a := parse(c, dir)
			if a != nil {
				a.FileSize = file.Size()
				infoArr = append(infoArr, *a)
			}
		} else if file.IsDir() && recurcive {
			dir := dirname + string(os.PathSeparator) + file.Name()
			//log.Printf("apk subfolder - %q\n", dir)
			arr := (c).Folder(dir, true)
			if arr != nil {
				for _, a := range *arr {
					infoArr = append(infoArr, a)
				}
			}
		}
	}
	return &infoArr
}
<|endoftext|>


<commit_before>package core

import (
	"encoding/json"
	"fmt"
	"github.com/hailocab/elastigo/api"
)

// The get API allows to get a typed JSON document from the index based on its id.
// GET - retrieves the doc
// HEAD - checks for existence of the doc
// http://www.elasticsearch.org/guide/reference/api/get.html
// TODO: make this implement an interface
func Get(pretty bool, index string, _type string, id string) (api.BaseResponse, error) {
	var url string
	var retval api.BaseResponse
	if len(_type) > 0 {
		url = fmt.Sprintf("/%s/%s/%s?%s", index, _type, id, api.Pretty(pretty))
	} else {
		url = fmt.Sprintf("/%s/%s?%s", index, id, api.Pretty(pretty))
	}
	body, err := api.DoCommand("GET", url, nil)
	if err != nil {
		return retval, err
	}
	if err == nil {
		// marshall into json
		jsonErr := json.Unmarshal(body, &retval)
		if jsonErr != nil {
			return retval, jsonErr
		}
	}
	//fmt.Println(body)
	return retval, err
}

// The API also allows to check for the existance of a document using HEAD

func Exists(pretty bool, index string, _type string, id string) (bool, error) {

	var url string

  var response map[string]interface{}

	if len(_type) > 0 {
		url = fmt.Sprintf("/%s/%s/%s?fields=_id%s", index, _type, id, api.Pretty(pretty))
	} else {
		url = fmt.Sprintf("/%s/%s?fields=_id%s", index, id, api.Pretty(pretty))
	}

  req, err := api.ElasticSearchRequest("HEAD", url)

	if err != nil {
		fmt.Println(err)
	}

  httpStatusCode, _, err := req.Do(&response)

	if err != nil {
		return false, err
	}
  if httpStatusCode == 404 {
    return false, err
  } else {
    return true, err
  }
}
<commit_msg>Simplify error handling on core.get<commit_after>package core

import (
	"encoding/json"
	"fmt"
	"github.com/hailocab/elastigo/api"
)

// The get API allows to get a typed JSON document from the index based on its id.
// GET - retrieves the doc
// HEAD - checks for existence of the doc
// http://www.elasticsearch.org/guide/reference/api/get.html
// TODO: make this implement an interface
func Get(pretty bool, index string, _type string, id string) (api.BaseResponse, error) {
	var url string
	var retval api.BaseResponse
	if len(_type) > 0 {
		url = fmt.Sprintf("/%s/%s/%s?%s", index, _type, id, api.Pretty(pretty))
	} else {
		url = fmt.Sprintf("/%s/%s?%s", index, id, api.Pretty(pretty))
	}
	body, err := api.DoCommand("GET", url, nil)
	if err != nil {
		return retval, err
	}

	// marshall into json
	jsonErr := json.Unmarshal(body, &retval)
	if jsonErr != nil {
		return retval, jsonErr
	}

	//fmt.Println(body)
	return retval, err
}

// The API also allows to check for the existance of a document using HEAD

func Exists(pretty bool, index string, _type string, id string) (bool, error) {

	var url string

  var response map[string]interface{}

	if len(_type) > 0 {
		url = fmt.Sprintf("/%s/%s/%s?fields=_id%s", index, _type, id, api.Pretty(pretty))
	} else {
		url = fmt.Sprintf("/%s/%s?fields=_id%s", index, id, api.Pretty(pretty))
	}

  req, err := api.ElasticSearchRequest("HEAD", url)

	if err != nil {
		fmt.Println(err)
	}

  httpStatusCode, _, err := req.Do(&response)

	if err != nil {
		return false, err
	}
  if httpStatusCode == 404 {
    return false, err
  } else {
    return true, err
  }
}
<|endoftext|>


<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
package generator

import (
	"bytes"
	"crypto/tls"
	"encoding/xml"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"text/template"
	"time"
	"unicode"
)

const maxRecursion uint8 = 20

type GoWsdl struct {
	file, pkg             string
	ignoreTls             bool
	wsdl                  *Wsdl
	resolvedXsdExternals  map[string]bool
	currentRecursionLevel uint8
}

var cacheDir = filepath.Join(os.TempDir(), "gowsdl-cache")

func init() {
	err := os.MkdirAll(cacheDir, 0700)
	if err != nil {
		Log.Crit("Create cache directory", "error", err)
		os.Exit(1)
	}
}

var timeout = time.Duration(30 * time.Second)

func dialTimeout(network, addr string) (net.Conn, error) {
	return net.DialTimeout(network, addr, timeout)
}

func downloadFile(url string, ignoreTls bool) ([]byte, error) {
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{
			InsecureSkipVerify: ignoreTls,
		},
		Dial: dialTimeout,
	}
	client := &http.Client{Transport: tr}

	resp, err := client.Get(url)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()
	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	return data, nil
}

func NewGoWsdl(file, pkg string, ignoreTls bool) (*GoWsdl, error) {
	file = strings.TrimSpace(file)
	if file == "" {
		Log.Crit("WSDL file is required to generate Go proxy")
		os.Exit(2)
	}

	pkg = strings.TrimSpace(pkg)
	if pkg == "" {
		pkg = "myservice"
	}

	return &GoWsdl{
		file:      file,
		pkg:       pkg,
		ignoreTls: ignoreTls,
	}, nil
}

func (g *GoWsdl) Start() (map[string][]byte, error) {
	gocode := make(map[string][]byte)

	err := g.unmarshal()
	if err != nil {
		return nil, err
	}

	var wg sync.WaitGroup

	wg.Add(1)
	go func() {
		defer wg.Done()
		var err error

		gocode["types"], err = g.genTypes()
		if err != nil {
			Log.Error("genTypes", "error", err)
		}
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()
		var err error

		gocode["operations"], err = g.genOperations()
		if err != nil {
			Log.Error("genOperations", "error", err)
		}
	}()

	wg.Wait()

	gocode["header"], err = g.genHeader()
	if err != nil {
		Log.Error("genHeader", "error", err)
	}

	return gocode, nil
}

func (g *GoWsdl) unmarshal() error {
	var data []byte

	parsedUrl, err := url.Parse(g.file)
	if parsedUrl.Scheme == "" {
		Log.Info("Reading", "file", g.file)

		data, err = ioutil.ReadFile(g.file)
		if err != nil {
			return err
		}
	} else {
		Log.Info("Downloading", "file", g.file)

		data, err = downloadFile(g.file, g.ignoreTls)
		if err != nil {
			return err
		}
	}

	g.wsdl = &Wsdl{}
	err = xml.Unmarshal(data, g.wsdl)
	if err != nil {
		return err
	}

	for _, schema := range g.wsdl.Types.Schemas {
		err = g.resolveXsdExternals(schema, parsedUrl)
		if err != nil {
			return err
		}
	}

	return nil
}

func (g *GoWsdl) resolveXsdExternals(schema *XsdSchema, url *url.URL) error {
	for _, incl := range schema.Includes {
		location, err := url.Parse(incl.SchemaLocation)
		if err != nil {
			return err
		}

		_, schemaName := filepath.Split(location.Path)
		if g.resolvedXsdExternals[schemaName] {
			continue
		}

		schemaLocation := location.String()
		if !location.IsAbs() {
			if !url.IsAbs() {
				return errors.New(fmt.Sprintf("Unable to resolve external schema %s through WSDL URL %s", schemaLocation, url))
			}
			schemaLocation = url.Scheme + "://" + url.Host + schemaLocation
		}

		Log.Info("Downloading external schema", "location", schemaLocation)

		data, err := downloadFile(schemaLocation, g.ignoreTls)
		newschema := &XsdSchema{}

		err = xml.Unmarshal(data, newschema)
		if err != nil {
			return err
		}

		if len(newschema.Includes) > 0 &&
			maxRecursion > g.currentRecursionLevel {

			g.currentRecursionLevel++

			//log.Printf("Entering recursion %d\n", g.currentRecursionLevel)
			g.resolveXsdExternals(newschema, url)
		}

		g.wsdl.Types.Schemas = append(g.wsdl.Types.Schemas, newschema)

		if g.resolvedXsdExternals == nil {
			g.resolvedXsdExternals = make(map[string]bool, maxRecursion)
		}
		g.resolvedXsdExternals[schemaName] = true
	}

	return nil
}

func (g *GoWsdl) genTypes() ([]byte, error) {
	funcMap := template.FuncMap{
		"toGoType":             toGoType,
		"stripns":              stripns,
		"replaceReservedWords": replaceReservedWords,
		"makePublic":           makePublic,
		"comment":              comment,
		"targetNamespace":      func() string { return g.wsdl.TargetNamespace },
	}

	//TODO resolve element refs in place.
	//g.resolveElementsRefs()

	data := new(bytes.Buffer)
	tmpl := template.Must(template.New("types").Funcs(funcMap).Parse(typesTmpl))
	err := tmpl.Execute(data, g.wsdl.Types)
	if err != nil {
		return nil, err
	}

	return data.Bytes(), nil
}

// func (g *GoWsdl) resolveElementsRefs() error {
// 	for _, schema := range g.wsdl.Types.Schemas {
// 		for _, globalEl := range schema.Elements {
// 			for _, localEl := range globalEl.ComplexType.Sequence.Elements {

// 			}
// 		}
// 	}
// }

func (g *GoWsdl) genOperations() ([]byte, error) {
	funcMap := template.FuncMap{
		"toGoType":             toGoType,
		"stripns":              stripns,
		"replaceReservedWords": replaceReservedWords,
		"makePublic":           makePublic,
		"findType":             g.findType,
		"findSoapAction":       g.findSoapAction,
		"findServiceAddress":   g.findServiceAddress,
	}

	data := new(bytes.Buffer)
	tmpl := template.Must(template.New("operations").Funcs(funcMap).Parse(opsTmpl))
	err := tmpl.Execute(data, g.wsdl.PortTypes)
	if err != nil {
		return nil, err
	}

	return data.Bytes(), nil
}

func (g *GoWsdl) genHeader() ([]byte, error) {
	funcMap := template.FuncMap{
		"toGoType":             toGoType,
		"stripns":              stripns,
		"replaceReservedWords": replaceReservedWords,
		"makePublic":           makePublic,
		"findType":             g.findType,
		"comment":              comment,
	}

	data := new(bytes.Buffer)
	tmpl := template.Must(template.New("header").Funcs(funcMap).Parse(headerTmpl))
	err := tmpl.Execute(data, g.pkg)
	if err != nil {
		return nil, err
	}

	return data.Bytes(), nil
}

var reservedWords = map[string]string{
	"break":       "break_",
	"default":     "default_",
	"func":        "func_",
	"interface":   "interface_",
	"select":      "select_",
	"case":        "case_",
	"defer":       "defer_",
	"go":          "go_",
	"map":         "map_",
	"struct":      "struct_",
	"chan":        "chan_",
	"else":        "else_",
	"goto":        "goto_",
	"package":     "package_",
	"switch":      "switch_",
	"const":       "const_",
	"fallthrough": "fallthrough_",
	"if":          "if_",
	"range":       "range_",
	"type":        "type_",
	"continue":    "continue_",
	"for":         "for_",
	"import":      "import_",
	"return":      "return_",
	"var":         "var_",
}

// Replaces Go reserved keywords to avoid compilation issues
func replaceReservedWords(identifier string) string {
	value := reservedWords[identifier]
	if value != "" {
		return value
	}
	return normalize(identifier)
}

// Normalizes value to be used as a valid Go identifier, avoiding compilation issues
func normalize(value string) string {
	mapping := func(r rune) rune {
		if unicode.IsLetter(r) || unicode.IsDigit(r) {
			return r
		}
		return -1
	}

	return strings.Map(mapping, value)
}

var xsd2GoTypes = map[string]string{
	"string":        "string",
	"token":         "string",
	"float":         "float32",
	"double":        "float64",
	"decimal":       "float64",
	"integer":       "int32",
	"int":           "int32",
	"short":         "int16",
	"byte":          "int8",
	"long":          "int64",
	"boolean":       "bool",
	"datetime":      "time.Time",
	"date":          "time.Time",
	"time":          "time.Time",
	"base64binary":  "[]byte",
	"hexbinary":     "[]byte",
	"unsignedint":   "uint32",
	"unsignedshort": "uint16",
	"unsignedbyte":  "byte",
	"unsignedlong":  "uint64",
	"anytype":       "interface{}",
}

func toGoType(xsdType string) string {
	// Handles name space, ie. xsd:string, xs:string
	r := strings.Split(xsdType, ":")

	type_ := r[0]

	if len(r) == 2 {
		type_ = r[1]
	}

	value := xsd2GoTypes[strings.ToLower(type_)]

	if value != "" {
		return value
	}

	return "*" + makePublic(type_)
}

// Given a message, finds its type.
//
// I'm not very proud of this function but
// it works for now and performance doesn't
// seem critical at this point
func (g *GoWsdl) findType(message string) string {
	message = stripns(message)

	for _, msg := range g.wsdl.Messages {
		if msg.Name != message {
			continue
		}

		// Assumes document/literal wrapped WS-I
		if len(msg.Parts) == 0 {
			// Message does not have parts. This could be a Port
			// with HTTP binding or SOAP 1.2 binding, which are not currently
			// supported.
			Log.Warn("WSDL does seem to have HTTP or SOAP 1.2 binding which is not currently supported.")
			continue
		}
		part := msg.Parts[0]
		if part.Type != "" {
			return stripns(part.Type)
		}

		elRef := stripns(part.Element)

		for _, schema := range g.wsdl.Types.Schemas {
			for _, el := range schema.Elements {
				if strings.EqualFold(elRef, el.Name) {
					if el.Type != "" {
						return stripns(el.Type)
					}
					return el.Name
				}
			}
		}
	}
	return ""
}

// TODO(c4milo): Add support for namespaces instead of striping them out
// TODO(c4milo): improve runtime complexity if performance turns out to be an issue.
func (g *GoWsdl) findSoapAction(operation, portType string) string {
	for _, binding := range g.wsdl.Binding {
		if stripns(binding.Type) != portType {
			continue
		}

		for _, soapOp := range binding.Operations {
			if soapOp.Name == operation {
				return soapOp.SoapOperation.SoapAction
			}
		}
	}
	return ""
}

func (g *GoWsdl) findServiceAddress(name string) string {
	for _, service := range g.wsdl.Service {
		for _, port := range service.Ports {
			if port.Name == name {
				return port.SoapAddress.Location
			}
		}
	}
	return ""
}

// TODO(c4milo): Add namespace support instead of stripping it
func stripns(xsdType string) string {
	r := strings.Split(xsdType, ":")
	type_ := r[0]

	if len(r) == 2 {
		type_ = r[1]
	}

	return type_
}

func makePublic(field_ string) string {
	field := []rune(field_)
	if len(field) == 0 {
		return field_
	}

	field[0] = unicode.ToUpper(field[0])
	return string(field)
}

func comment(text string) string {
	lines := strings.Split(text, "\n")

	var output string
	if len(lines) == 1 && lines[0] == "" {
		return ""
	}

	// Helps to determine if
	// there is an actual comment
	// without screwing newlines
	// in real comments.
	hasComment := false

	for _, line := range lines {
		line = strings.TrimLeftFunc(line, unicode.IsSpace)
		if line != "" {
			hasComment = true
		}
		output += "\n// " + line
	}

	if hasComment {
		return output
	}
	return ""
}
<commit_msg>Fixes #29<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
package generator

import (
	"bytes"
	"crypto/tls"
	"encoding/xml"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"text/template"
	"time"
	"unicode"
)

const maxRecursion uint8 = 20

type GoWsdl struct {
	file, pkg             string
	ignoreTls             bool
	wsdl                  *Wsdl
	resolvedXsdExternals  map[string]bool
	currentRecursionLevel uint8
}

var cacheDir = filepath.Join(os.TempDir(), "gowsdl-cache")

func init() {
	err := os.MkdirAll(cacheDir, 0700)
	if err != nil {
		Log.Crit("Create cache directory", "error", err)
		os.Exit(1)
	}
}

var timeout = time.Duration(30 * time.Second)

func dialTimeout(network, addr string) (net.Conn, error) {
	return net.DialTimeout(network, addr, timeout)
}

func downloadFile(url string, ignoreTls bool) ([]byte, error) {
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{
			InsecureSkipVerify: ignoreTls,
		},
		Dial: dialTimeout,
	}
	client := &http.Client{Transport: tr}

	resp, err := client.Get(url)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()
	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	return data, nil
}

func NewGoWsdl(file, pkg string, ignoreTls bool) (*GoWsdl, error) {
	file = strings.TrimSpace(file)
	if file == "" {
		Log.Crit("WSDL file is required to generate Go proxy")
		os.Exit(2)
	}

	pkg = strings.TrimSpace(pkg)
	if pkg == "" {
		pkg = "myservice"
	}

	return &GoWsdl{
		file:      file,
		pkg:       pkg,
		ignoreTls: ignoreTls,
	}, nil
}

func (g *GoWsdl) Start() (map[string][]byte, error) {
	gocode := make(map[string][]byte)

	err := g.unmarshal()
	if err != nil {
		return nil, err
	}

	var wg sync.WaitGroup

	wg.Add(1)
	go func() {
		defer wg.Done()
		var err error

		gocode["types"], err = g.genTypes()
		if err != nil {
			Log.Error("genTypes", "error", err)
		}
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()
		var err error

		gocode["operations"], err = g.genOperations()
		if err != nil {
			Log.Error("genOperations", "error", err)
		}
	}()

	wg.Wait()

	gocode["header"], err = g.genHeader()
	if err != nil {
		Log.Error("genHeader", "error", err)
	}

	return gocode, nil
}

func (g *GoWsdl) unmarshal() error {
	var data []byte

	parsedUrl, err := url.Parse(g.file)
	if parsedUrl.Scheme == "" {
		Log.Info("Reading", "file", g.file)

		data, err = ioutil.ReadFile(g.file)
		if err != nil {
			return err
		}
	} else {
		Log.Info("Downloading", "file", g.file)

		data, err = downloadFile(g.file, g.ignoreTls)
		if err != nil {
			return err
		}
	}

	g.wsdl = &Wsdl{}
	err = xml.Unmarshal(data, g.wsdl)
	if err != nil {
		return err
	}

	for _, schema := range g.wsdl.Types.Schemas {
		err = g.resolveXsdExternals(schema, parsedUrl)
		if err != nil {
			return err
		}
	}

	return nil
}

func (g *GoWsdl) resolveXsdExternals(schema *XsdSchema, url *url.URL) error {
	for _, incl := range schema.Includes {
		location, err := url.Parse(incl.SchemaLocation)
		if err != nil {
			return err
		}

		_, schemaName := filepath.Split(location.Path)
		if g.resolvedXsdExternals[schemaName] {
			continue
		}

		schemaLocation := location.String()
		if !location.IsAbs() {
			if !url.IsAbs() {
				return errors.New(fmt.Sprintf("Unable to resolve external schema %s through WSDL URL %s", schemaLocation, url))
			}
			schemaLocation = url.Scheme + "://" + url.Host + schemaLocation
		}

		Log.Info("Downloading external schema", "location", schemaLocation)

		data, err := downloadFile(schemaLocation, g.ignoreTls)
		newschema := &XsdSchema{}

		err = xml.Unmarshal(data, newschema)
		if err != nil {
			return err
		}

		if len(newschema.Includes) > 0 &&
			maxRecursion > g.currentRecursionLevel {

			g.currentRecursionLevel++

			//log.Printf("Entering recursion %d\n", g.currentRecursionLevel)
			g.resolveXsdExternals(newschema, url)
		}

		g.wsdl.Types.Schemas = append(g.wsdl.Types.Schemas, newschema)

		if g.resolvedXsdExternals == nil {
			g.resolvedXsdExternals = make(map[string]bool, maxRecursion)
		}
		g.resolvedXsdExternals[schemaName] = true
	}

	return nil
}

func (g *GoWsdl) genTypes() ([]byte, error) {
	funcMap := template.FuncMap{
		"toGoType":             toGoType,
		"stripns":              stripns,
		"replaceReservedWords": replaceReservedWords,
		"makePublic":           makePublic,
		"comment":              comment,
		"targetNamespace":      func() string { return g.wsdl.TargetNamespace },
	}

	//TODO resolve element refs in place.
	//g.resolveElementsRefs()

	data := new(bytes.Buffer)
	tmpl := template.Must(template.New("types").Funcs(funcMap).Parse(typesTmpl))
	err := tmpl.Execute(data, g.wsdl.Types)
	if err != nil {
		return nil, err
	}

	return data.Bytes(), nil
}

// func (g *GoWsdl) resolveElementsRefs() error {
// 	for _, schema := range g.wsdl.Types.Schemas {
// 		for _, globalEl := range schema.Elements {
// 			for _, localEl := range globalEl.ComplexType.Sequence.Elements {

// 			}
// 		}
// 	}
// }

func (g *GoWsdl) genOperations() ([]byte, error) {
	funcMap := template.FuncMap{
		"toGoType":             toGoType,
		"stripns":              stripns,
		"replaceReservedWords": replaceReservedWords,
		"makePublic":           makePublic,
		"findType":             g.findType,
		"findSoapAction":       g.findSoapAction,
		"findServiceAddress":   g.findServiceAddress,
	}

	data := new(bytes.Buffer)
	tmpl := template.Must(template.New("operations").Funcs(funcMap).Parse(opsTmpl))
	err := tmpl.Execute(data, g.wsdl.PortTypes)
	if err != nil {
		return nil, err
	}

	return data.Bytes(), nil
}

func (g *GoWsdl) genHeader() ([]byte, error) {
	funcMap := template.FuncMap{
		"toGoType":             toGoType,
		"stripns":              stripns,
		"replaceReservedWords": replaceReservedWords,
		"makePublic":           makePublic,
		"findType":             g.findType,
		"comment":              comment,
	}

	data := new(bytes.Buffer)
	tmpl := template.Must(template.New("header").Funcs(funcMap).Parse(headerTmpl))
	err := tmpl.Execute(data, g.pkg)
	if err != nil {
		return nil, err
	}

	return data.Bytes(), nil
}

var reservedWords = map[string]string{
	"break":       "break_",
	"default":     "default_",
	"func":        "func_",
	"interface":   "interface_",
	"select":      "select_",
	"case":        "case_",
	"defer":       "defer_",
	"go":          "go_",
	"map":         "map_",
	"struct":      "struct_",
	"chan":        "chan_",
	"else":        "else_",
	"goto":        "goto_",
	"package":     "package_",
	"switch":      "switch_",
	"const":       "const_",
	"fallthrough": "fallthrough_",
	"if":          "if_",
	"range":       "range_",
	"type":        "type_",
	"continue":    "continue_",
	"for":         "for_",
	"import":      "import_",
	"return":      "return_",
	"var":         "var_",
}

// Replaces Go reserved keywords to avoid compilation issues
func replaceReservedWords(identifier string) string {
	value := reservedWords[identifier]
	if value != "" {
		return value
	}
	return normalize(identifier)
}

// Normalizes value to be used as a valid Go identifier, avoiding compilation issues
func normalize(value string) string {
	mapping := func(r rune) rune {
		if unicode.IsLetter(r) || unicode.IsDigit(r) {
			return r
		}
		return -1
	}

	return strings.Map(mapping, value)
}

var xsd2GoTypes = map[string]string{
	"string":        "string",
	"token":         "string",
	"float":         "float32",
	"double":        "float64",
	"decimal":       "float64",
	"integer":       "int32",
	"int":           "int32",
	"short":         "int16",
	"byte":          "int8",
	"long":          "int64",
	"boolean":       "bool",
	"datetime":      "time.Time",
	"date":          "time.Time",
	"time":          "time.Time",
	"base64binary":  "[]byte",
	"hexbinary":     "[]byte",
	"unsignedint":   "uint32",
	"unsignedshort": "uint16",
	"unsignedbyte":  "byte",
	"unsignedlong":  "uint64",
	"anytype":       "interface{}",
}

func toGoType(xsdType string) string {
	// Handles name space, ie. xsd:string, xs:string
	r := strings.Split(xsdType, ":")

	type_ := r[0]

	if len(r) == 2 {
		type_ = r[1]
	}

	value := xsd2GoTypes[strings.ToLower(type_)]

	if value != "" {
		return value
	}

	return "*" + replaceReservedWords(makePublic(type_))
}

// Given a message, finds its type.
//
// I'm not very proud of this function but
// it works for now and performance doesn't
// seem critical at this point
func (g *GoWsdl) findType(message string) string {
	message = stripns(message)

	for _, msg := range g.wsdl.Messages {
		if msg.Name != message {
			continue
		}

		// Assumes document/literal wrapped WS-I
		if len(msg.Parts) == 0 {
			// Message does not have parts. This could be a Port
			// with HTTP binding or SOAP 1.2 binding, which are not currently
			// supported.
			Log.Warn("WSDL does seem to have HTTP or SOAP 1.2 binding which is not currently supported.")
			continue
		}
		part := msg.Parts[0]
		if part.Type != "" {
			return stripns(part.Type)
		}

		elRef := stripns(part.Element)

		for _, schema := range g.wsdl.Types.Schemas {
			for _, el := range schema.Elements {
				if strings.EqualFold(elRef, el.Name) {
					if el.Type != "" {
						return stripns(el.Type)
					}
					return el.Name
				}
			}
		}
	}
	return ""
}

// TODO(c4milo): Add support for namespaces instead of striping them out
// TODO(c4milo): improve runtime complexity if performance turns out to be an issue.
func (g *GoWsdl) findSoapAction(operation, portType string) string {
	for _, binding := range g.wsdl.Binding {
		if stripns(binding.Type) != portType {
			continue
		}

		for _, soapOp := range binding.Operations {
			if soapOp.Name == operation {
				return soapOp.SoapOperation.SoapAction
			}
		}
	}
	return ""
}

func (g *GoWsdl) findServiceAddress(name string) string {
	for _, service := range g.wsdl.Service {
		for _, port := range service.Ports {
			if port.Name == name {
				return port.SoapAddress.Location
			}
		}
	}
	return ""
}

// TODO(c4milo): Add namespace support instead of stripping it
func stripns(xsdType string) string {
	r := strings.Split(xsdType, ":")
	type_ := r[0]

	if len(r) == 2 {
		type_ = r[1]
	}

	return type_
}

func makePublic(field_ string) string {
	field := []rune(field_)
	if len(field) == 0 {
		return field_
	}

	field[0] = unicode.ToUpper(field[0])
	return string(field)
}

func comment(text string) string {
	lines := strings.Split(text, "\n")

	var output string
	if len(lines) == 1 && lines[0] == "" {
		return ""
	}

	// Helps to determine if
	// there is an actual comment
	// without screwing newlines
	// in real comments.
	hasComment := false

	for _, line := range lines {
		line = strings.TrimLeftFunc(line, unicode.IsSpace)
		if line != "" {
			hasComment = true
		}
		output += "\n// " + line
	}

	if hasComment {
		return output
	}
	return ""
}
<|endoftext|>


<commit_before>// Copyright 2017 The Neugram Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gengo_test

import (
	"bytes"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"testing"

	"neugram.io/ng/gengo"
)

func TestGeneratedPrograms(t *testing.T) {
	files, err := filepath.Glob("../eval/testdata/*.ng")
	if err != nil {
		t.Fatal(err)
	}
	if len(files) == 0 {
		t.Fatal("cannot find testdata")
	}

	for _, file := range files {
		file := file
		test := file[len("../eval/testdata/") : len(file)-3]
		exclude := []string{ // TODO remove this list
			"import3",
			"ellipsis",
			"error6",
			"error7",
			"import3_error",
			"import4",
			"import5",
			"import8",
			"method2",
			"op1",
			"slice1",
		}
		donotrun := false
		for _, ex := range exclude {
			if test == ex {
				donotrun = true
			}
		}
		if donotrun {
			continue
		}
		t.Run(test, func(t *testing.T) {
			t.Parallel()
			res, err := gengo.GenGo(file, "main")
			if err != nil {
				if strings.HasSuffix(test, "_error") {
					return
				}
				t.Fatal(err)
			}

			f, err := ioutil.TempFile("", "gengo-"+test)
			if err != nil {
				t.Fatal(err)
			}
			tmpgo := f.Name() + ".go"
			f.Close()
			os.Remove(f.Name())
			f, err = os.Create(tmpgo)
			if err != nil {
				t.Fatal(err)
			}
			defer os.Remove(f.Name())

			if _, err := f.Write(res); err != nil {
				t.Fatal(err)
			}
			if err := f.Close(); err != nil {
				t.Fatal(err)
			}
			binname := strings.TrimSuffix(f.Name(), ".go")

			cmd := exec.Command("go", "build", "-o", binname, f.Name())
			out, err := cmd.CombinedOutput()
			if err != nil {
				if strings.HasSuffix(test, "_error") {
					return
				}
				t.Fatalf("failed to build: %v\n%s", err, out)
			}
			defer os.Remove(binname)

			cmd = exec.Command(binname)
			out, err = cmd.CombinedOutput()
			if err != nil {
				if strings.HasSuffix(test, "_error") {
					return
				}
				if strings.HasSuffix(test, "_panic") {
					// TODO: check errors make sense
					return
				}
				t.Fatalf("failed to run: %v\n%s", err, out)
			}
			if !bytes.HasSuffix(out, []byte("OK\n")) {
				t.Logf("output:\n%s", out)
				t.Error("test missing OK")
			}
		})
	}
}
<commit_msg>gengo: two more tests already passing<commit_after>// Copyright 2017 The Neugram Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gengo_test

import (
	"bytes"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"testing"

	"neugram.io/ng/gengo"
)

func TestGeneratedPrograms(t *testing.T) {
	files, err := filepath.Glob("../eval/testdata/*.ng")
	if err != nil {
		t.Fatal(err)
	}
	if len(files) == 0 {
		t.Fatal("cannot find testdata")
	}

	for _, file := range files {
		file := file
		test := file[len("../eval/testdata/") : len(file)-3]
		exclude := []string{ // TODO remove this list
			"import3",
			"error6",
			"error7",
			"import3_error",
			"import4",
			"import5",
			"import8",
			"method2",
			"op1",
		}
		donotrun := false
		for _, ex := range exclude {
			if test == ex {
				donotrun = true
			}
		}
		if donotrun {
			continue
		}
		t.Run(test, func(t *testing.T) {
			t.Parallel()
			res, err := gengo.GenGo(file, "main")
			if err != nil {
				if strings.HasSuffix(test, "_error") {
					return
				}
				t.Fatal(err)
			}

			f, err := ioutil.TempFile("", "gengo-"+test)
			if err != nil {
				t.Fatal(err)
			}
			tmpgo := f.Name() + ".go"
			f.Close()
			os.Remove(f.Name())
			f, err = os.Create(tmpgo)
			if err != nil {
				t.Fatal(err)
			}
			defer os.Remove(f.Name())

			if _, err := f.Write(res); err != nil {
				t.Fatal(err)
			}
			if err := f.Close(); err != nil {
				t.Fatal(err)
			}
			binname := strings.TrimSuffix(f.Name(), ".go")

			cmd := exec.Command("go", "build", "-o", binname, f.Name())
			out, err := cmd.CombinedOutput()
			if err != nil {
				if strings.HasSuffix(test, "_error") {
					return
				}
				t.Fatalf("failed to build: %v\n%s", err, out)
			}
			defer os.Remove(binname)

			cmd = exec.Command(binname)
			out, err = cmd.CombinedOutput()
			if err != nil {
				if strings.HasSuffix(test, "_error") {
					return
				}
				if strings.HasSuffix(test, "_panic") {
					// TODO: check errors make sense
					return
				}
				t.Fatalf("failed to run: %v\n%s", err, out)
			}
			if !bytes.HasSuffix(out, []byte("OK\n")) {
				t.Logf("output:\n%s", out)
				t.Error("test missing OK")
			}
		})
	}
}
<|endoftext|>


<commit_before>package csi

import (
	"github.com/container-storage-interface/spec/lib/go/csi"
	"go.pedge.io/dlog"
	"golang.org/x/net/context"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

func (s *OsdCsiServer) GetNodeID(ctx context.Context, req *csi.GetNodeIDRequest) (*csi.GetNodeIDResponse, error) {
	dlog.Debugf("GetNodeID req[%#v]", req)

	// Check arguments
	if req.GetVersion() == nil {
		return nil, status.Error(codes.InvalidArgument, "Version must be provided")
	}

	clus, err := s.cluster.Enumerate()
	if err != nil {
		return nil, status.Errorf(codes.Internal, "Unable to Enumerate cluster: %s", err)
	}

	result := &csi.GetNodeIDResponse{
		NodeId: clus.NodeId,
	}

	dlog.Infof("NodeId is %s", result.NodeId)

	return result, nil
}
<commit_msg>comment for travis build<commit_after>package csi

import (
	"github.com/container-storage-interface/spec/lib/go/csi"
	"go.pedge.io/dlog"
	"golang.org/x/net/context"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

// Get PX Node id for local node.
func (s *OsdCsiServer) GetNodeID(ctx context.Context, req *csi.GetNodeIDRequest) (*csi.GetNodeIDResponse, error) {
	dlog.Debugf("GetNodeID req[%#v]", req)

	// Check arguments
	if req.GetVersion() == nil {
		return nil, status.Error(codes.InvalidArgument, "Version must be provided")
	}

	clus, err := s.cluster.Enumerate()
	if err != nil {
		return nil, status.Errorf(codes.Internal, "Unable to Enumerate cluster: %s", err)
	}

	result := &csi.GetNodeIDResponse{
		NodeId: clus.NodeId,
	}

	dlog.Infof("NodeId is %s", result.NodeId)

	return result, nil
}
<|endoftext|>


<commit_before>package keeper

import (
	"context"

	"github.com/cosmos/cosmos-sdk/store/prefix"
	sdk "github.com/cosmos/cosmos-sdk/types"
	"github.com/cosmos/cosmos-sdk/types/query"
	"github.com/zigbee-alliance/distributed-compliance-ledger/x/pki/types"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

func (k Keeper) RejectedCertificateAll(c context.Context, req *types.QueryAllRejectedCertificatesRequest) (*types.QueryAllRejectedCertificatesResponse, error) {
	if req == nil {
		return nil, status.Error(codes.InvalidArgument, "invalid request")
	}

	var rejectedCertificates []types.RejectedCertificate
	ctx := sdk.UnwrapSDKContext(c)

	store := ctx.KVStore(k.storeKey)
	rejectedCertificateStore := prefix.NewStore(store, types.KeyPrefix(types.RejectedCertificateKeyPrefix))

	pageRes, err := query.Paginate(rejectedCertificateStore, req.Pagination, func(key []byte, value []byte) error {
		var rejectedCertificate types.RejectedCertificate
		if err := k.cdc.Unmarshal(value, &rejectedCertificate); err != nil {
			return err
		}

		rejectedCertificates = append(rejectedCertificates, rejectedCertificate)

		return nil
	})
	if err != nil {
		return nil, status.Error(codes.Internal, err.Error())
	}

	return &types.QueryAllRejectedCertificatesResponse{RejectedCertificate: rejectedCertificates, Pagination: pageRes}, nil
}

func (k Keeper) RejectedCertificate(c context.Context, req *types.QueryGetRejectedCertificatesRequest) (*types.QueryGetRejectedCertificatesResponse, error) {
	if req == nil {
		return nil, status.Error(codes.InvalidArgument, "invalid request")
	}
	ctx := sdk.UnwrapSDKContext(c)

	val, found := k.GetRejectedCertificate(
		ctx,
		req.Subject,
		req.SubjectKeyId,
	)
	if !found {
		return nil, status.Error(codes.InvalidArgument, "not found")
	}

	return &types.QueryGetRejectedCertificatesResponse{RejectedCertificate: val}, nil
}
<commit_msg>We should return NotFound in all requst where we don't find some object<commit_after>package keeper

import (
	"context"

	"github.com/cosmos/cosmos-sdk/store/prefix"
	sdk "github.com/cosmos/cosmos-sdk/types"
	"github.com/cosmos/cosmos-sdk/types/query"
	"github.com/zigbee-alliance/distributed-compliance-ledger/x/pki/types"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

func (k Keeper) RejectedCertificateAll(c context.Context, req *types.QueryAllRejectedCertificatesRequest) (*types.QueryAllRejectedCertificatesResponse, error) {
	if req == nil {
		return nil, status.Error(codes.InvalidArgument, "invalid request")
	}

	var rejectedCertificates []types.RejectedCertificate
	ctx := sdk.UnwrapSDKContext(c)

	store := ctx.KVStore(k.storeKey)
	rejectedCertificateStore := prefix.NewStore(store, types.KeyPrefix(types.RejectedCertificateKeyPrefix))

	pageRes, err := query.Paginate(rejectedCertificateStore, req.Pagination, func(key []byte, value []byte) error {
		var rejectedCertificate types.RejectedCertificate
		if err := k.cdc.Unmarshal(value, &rejectedCertificate); err != nil {
			return err
		}

		rejectedCertificates = append(rejectedCertificates, rejectedCertificate)

		return nil
	})
	if err != nil {
		return nil, status.Error(codes.Internal, err.Error())
	}

	return &types.QueryAllRejectedCertificatesResponse{RejectedCertificate: rejectedCertificates, Pagination: pageRes}, nil
}

func (k Keeper) RejectedCertificate(c context.Context, req *types.QueryGetRejectedCertificatesRequest) (*types.QueryGetRejectedCertificatesResponse, error) {
	if req == nil {
		return nil, status.Error(codes.NotFound, "invalid request")
	}
	ctx := sdk.UnwrapSDKContext(c)

	val, found := k.GetRejectedCertificate(
		ctx,
		req.Subject,
		req.SubjectKeyId,
	)
	if !found {
		return nil, status.Error(codes.NotFound, "not found")
	}

	return &types.QueryGetRejectedCertificatesResponse{RejectedCertificate: val}, nil
}
<|endoftext|>


<commit_before>package graft

import (
	"fmt"
)

//State for a graft node
type State int8

// Allowable states for a Graft node.
const (
	FOLLOWER State = iota
	LEADER
	CANDIDATE
	CLOSED
)

const sonyflakeTimeUnit = 1e7 // nsec, i.e. 10 msec
// These constants are the bit lengths of Sonyflake ID parts.
const (
	BitLenTime      = 39                               // bit length of time
	BitLenSequence  = 8                                // bit length of sequence number
	BitLenMachineID = 63 - BitLenTime - BitLenSequence // bit length of machine id
)

// Convenience for printing, etc.
func (s State) String() string {
	switch s {
	case FOLLOWER:
		return "Follower"
	case LEADER:
		return "Leader"
	case CANDIDATE:
		return "Candidate"
	case CLOSED:
		return "Closed"
	default:
		return fmt.Sprintf("Unknown[%d]", s)
	}
}
<commit_msg>add StateChange<commit_after>package graft

import (
	"fmt"
)

// StateChange captures "from" and "to" States
type StateChange struct {
	// From is the previous state.
	From State

	// To is the new state.
	To State
}

//State for a graft node
type State int8

// Allowable states for a Graft node.
const (
	FOLLOWER State = iota
	LEADER
	CANDIDATE
	CLOSED
)

const sonyflakeTimeUnit = 1e7 // nsec, i.e. 10 msec
// These constants are the bit lengths of Sonyflake ID parts.
const (
	BitLenTime      = 39                               // bit length of time
	BitLenSequence  = 8                                // bit length of sequence number
	BitLenMachineID = 63 - BitLenTime - BitLenSequence // bit length of machine id
)

// Convenience for printing, etc.
func (s State) String() string {
	switch s {
	case FOLLOWER:
		return "Follower"
	case LEADER:
		return "Leader"
	case CANDIDATE:
		return "Candidate"
	case CLOSED:
		return "Closed"
	default:
		return fmt.Sprintf("Unknown[%d]", s)
	}
}
<|endoftext|>