<commit_before>// +build !luminous,!mimic

package admin

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"testing"
	"time"

	"github.com/ceph/go-ceph/cephfs"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

var snapDir = ".snapshots"

func fsConnect(t *testing.T) *cephfs.MountInfo {
	mount, err := cephfs.CreateMount()
	require.NoError(t, err)
	require.NotNil(t, mount)

	err = mount.ReadDefaultConfigFile()
	require.NoError(t, err)
	err = mount.SetConfigOption("client_snapdir", snapDir)
	require.NoError(t, err)

	timeout := time.After(time.Second * 5)
	ch := make(chan error)
	go func(mount *cephfs.MountInfo) {
		ch <- mount.Mount()
	}(mount)
	select {
	case err = <-ch:
	case <-timeout:
		err = fmt.Errorf("timed out waiting for connect")
	}
	require.NoError(t, err)
	return mount
}

func writeFile(t *testing.T, mount *cephfs.MountInfo, path string, content []byte) {
	f1, err := mount.Open(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0700)
	require.NoError(t, err)
	defer f1.Close()
	f1.WriteAt(content, 0)
}

func readFile(t *testing.T, mount *cephfs.MountInfo, path string) []byte {
	f1, err := mount.Open(path, os.O_RDONLY, 0)
	require.NoError(t, err)
	defer f1.Close()
	b, err := ioutil.ReadAll(f1)
	require.NoError(t, err)
	return b
}

func getSnapPath(t *testing.T, mount *cephfs.MountInfo, subvol, snapname string) string {
	// I wish there was a nicer way to do this
	snapPath := path.Join(subvol, snapDir, snapname)
	_, err := mount.Statx(snapPath, cephfs.StatxBasicStats, 0)
	if err == nil {
		return snapPath
	}
	snapPath = path.Join(path.Dir(subvol), snapDir, snapname, path.Base(subvol))
	_, err = mount.Statx(snapPath, cephfs.StatxBasicStats, 0)
	if err == nil {
		return snapPath
	}
	t.Fatalf("did not find a snap path for %s", snapname)
	return ""
}

// TestWorkflow aims to do more than just exercise the API calls, but rather to
// also check that they do what they say on the tin.  This means importing the
// cephfs lib in addition to the admin lib and reading and writing to the
// subvolume, snapshot, and clone as appropriate.
func TestWorkflow(t *testing.T) {
	fsa := getFSAdmin(t)
	volume := "cephfs"
	group := "workflow1"

	// verify the volume we want to use
	l, err := fsa.ListVolumes()
	require.NoError(t, err)
	require.Contains(t, l, volume)

	err = fsa.CreateSubVolumeGroup(volume, group, nil)
	require.NoError(t, err)
	defer func() {
		err := fsa.RemoveSubVolumeGroup(volume, group)
		assert.NoError(t, err)
	}()

	subname := "files1"
	svopts := &SubVolumeOptions{
		Mode: 0777,
		Size: 2 * gibiByte,
	}
	err = fsa.CreateSubVolume(volume, group, subname, svopts)
	require.NoError(t, err)
	defer func() {
		err := fsa.RemoveSubVolume(volume, group, subname)
		assert.NoError(t, err)
	}()

	// getpath
	subPath, err := fsa.SubVolumePath(volume, group, subname)
	require.NoError(t, err)
	require.NotEqual(t, "", subPath)

	// connect to volume, cd to path (?)
	mount := fsConnect(t)
	defer func(mount *cephfs.MountInfo) {
		assert.NoError(t, mount.Unmount())
		assert.NoError(t, mount.Release())
	}(mount)

	err = mount.ChangeDir(subPath)
	require.NoError(t, err)

	// write some dirs & files
	err = mount.MakeDir("content1", 0770)
	require.NoError(t, err)

	writeFile(t, mount, "content1/robots.txt",
		[]byte("robbie\nr2\nbender\nclaptrap\n"))
	writeFile(t, mount, "content1/songs.txt",
		[]byte("none of them knew they were robots\n"))
	assert.NoError(t, mount.MakeDir("content1/emptyDir1", 0770))

	err = mount.MakeDir("content2", 0770)
	require.NoError(t, err)

	writeFile(t, mount, "content2/androids.txt",
		[]byte("data\nmarvin\n"))
	assert.NoError(t, mount.MakeDir("content2/docs", 0770))
	writeFile(t, mount, "content2/docs/lore.txt",
		[]byte("Compendium\nLegend\nLore\nDeadweight\nSpirit at Aphelion\n"))

	assert.NoError(t, mount.SyncFs())

	// take a snapshot

	snapname1 := "hotSpans1"
	err = fsa.CreateSubVolumeSnapshot(volume, group, subname, snapname1)
	require.NoError(t, err)
	defer func() {
		err := fsa.RemoveSubVolumeSnapshot(volume, group, subname, snapname1)
		assert.NoError(t, err)
	}()

	sinfo, err := fsa.SubVolumeSnapshotInfo(volume, group, subname, snapname1)
	require.NoError(t, err)
	require.NotNil(t, sinfo)

	time.Sleep(500 * time.Millisecond) // is there a race?

	// examine the snapshot
	snapPath := getSnapPath(t, mount, subPath, snapname1)
	require.NotEqual(t, "", snapPath)

	tempPath := path.Join(snapPath, "content1/robots.txt")
	txt := readFile(t, mount, tempPath)
	assert.Contains(t, string(txt), "robbie")

	// original subvol can be manipulated
	err = mount.Rename("content2/docs/lore.txt", "content1/lore.txt")
	assert.NoError(t, err)
	writeFile(t, mount, "content1/songs.txt",
		[]byte("none of them knew they were robots\nars moriendi\n"))

	// snapshot may not be modified
	err = mount.Rename(
		path.Join(snapPath, "content2/docs/lore.txt"),
		path.Join(snapPath, "content1/lore.txt"))
	assert.Error(t, err)
	txt = readFile(t, mount, path.Join(snapPath, "content2/docs/lore.txt"))
	assert.Contains(t, string(txt), "Spirit")

	// make a clone

	clonename := "files2"
	err = fsa.CloneSubVolumeSnapshot(
		volume, group, subname, snapname1, clonename,
		&CloneOptions{TargetGroup: group})
	var x NotProtectedError
	if errors.As(err, &x) {
		err = fsa.ProtectSubVolumeSnapshot(volume, group, subname, snapname1)
		assert.NoError(t, err)
		defer func() {
			err := fsa.UnprotectSubVolumeSnapshot(volume, group, subname, snapname1)
			assert.NoError(t, err)
		}()

		err = fsa.CloneSubVolumeSnapshot(
			volume, group, subname, snapname1, clonename,
			&CloneOptions{TargetGroup: group})
	}
	require.NoError(t, err)
	defer func() {
		err := fsa.ForceRemoveSubVolume(volume, group, clonename)
		assert.NoError(t, err)
	}()

	// wait for cloning to complete
	for done := false; !done; {
		status, err := fsa.CloneStatus(volume, group, clonename)
		assert.NoError(t, err)
		assert.NotNil(t, status)
		switch status.State {
		case ClonePending, CloneInProgress:
			time.Sleep(5 * time.Millisecond)
		case CloneComplete:
			done = true
		case CloneFailed:
			t.Fatal("clone failed")
		default:
			t.Fatalf("invalid status.State: %q", status.State)
		}
	}

	// examine the clone
	clonePath, err := fsa.SubVolumePath(volume, group, clonename)
	require.NoError(t, err)
	require.NotEqual(t, "", clonePath)

	txt = readFile(t, mount, path.Join(clonePath, "content1/robots.txt"))
	assert.Contains(t, string(txt), "robbie")

	// clones are r/w
	err = mount.Rename(
		path.Join(clonePath, "content2/docs/lore.txt"),
		path.Join(clonePath, "content1/lore.txt"))
	assert.NoError(t, err)
	txt = readFile(t, mount, path.Join(clonePath, "content1/lore.txt"))
	assert.Contains(t, string(txt), "Spirit")

	// it reflects what was in the snapshot
	txt = readFile(t, mount, path.Join(clonePath, "content1/songs.txt"))
	assert.Contains(t, string(txt), "robots")
	assert.NotContains(t, string(txt), "moriendi")

	// ... with it's own independent data
	writeFile(t, mount, path.Join(clonePath, "content1/songs.txt"),
		[]byte("none of them knew they were robots\nsweet charity\n"))

	// (orig)
	txt = readFile(t, mount, "content1/songs.txt")
	assert.Contains(t, string(txt), "robots")
	assert.Contains(t, string(txt), "moriendi")
	assert.NotContains(t, string(txt), "charity")

	// (clone)
	txt = readFile(t, mount, path.Join(clonePath, "content1/songs.txt"))
	assert.Contains(t, string(txt), "robots")
	assert.NotContains(t, string(txt), "moriendi")
	assert.Contains(t, string(txt), "charity")
}
<commit_msg>cephfs admin: make fsConnect utility function more flexible<commit_after>// +build !luminous,!mimic

package admin

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"testing"
	"time"

	"github.com/ceph/go-ceph/cephfs"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

var snapDir = ".snapshots"

func fsConnect(t *testing.T, configFile string) *cephfs.MountInfo {
	mount, err := cephfs.CreateMount()
	require.NoError(t, err)
	require.NotNil(t, mount)

	if configFile == "" {
		err = mount.ReadDefaultConfigFile()
		require.NoError(t, err)
	} else {
		err = mount.ReadConfigFile(configFile)
		require.NoError(t, err)
	}
	err = mount.SetConfigOption("client_snapdir", snapDir)
	require.NoError(t, err)

	timeout := time.After(time.Second * 5)
	ch := make(chan error)
	go func(mount *cephfs.MountInfo) {
		ch <- mount.Mount()
	}(mount)
	select {
	case err = <-ch:
	case <-timeout:
		err = fmt.Errorf("timed out waiting for connect")
	}
	require.NoError(t, err)
	return mount
}

func writeFile(t *testing.T, mount *cephfs.MountInfo, path string, content []byte) {
	f1, err := mount.Open(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0700)
	require.NoError(t, err)
	defer f1.Close()
	f1.WriteAt(content, 0)
}

func readFile(t *testing.T, mount *cephfs.MountInfo, path string) []byte {
	f1, err := mount.Open(path, os.O_RDONLY, 0)
	require.NoError(t, err)
	defer f1.Close()
	b, err := ioutil.ReadAll(f1)
	require.NoError(t, err)
	return b
}

func getSnapPath(t *testing.T, mount *cephfs.MountInfo, subvol, snapname string) string {
	// I wish there was a nicer way to do this
	snapPath := path.Join(subvol, snapDir, snapname)
	_, err := mount.Statx(snapPath, cephfs.StatxBasicStats, 0)
	if err == nil {
		return snapPath
	}
	snapPath = path.Join(path.Dir(subvol), snapDir, snapname, path.Base(subvol))
	_, err = mount.Statx(snapPath, cephfs.StatxBasicStats, 0)
	if err == nil {
		return snapPath
	}
	t.Fatalf("did not find a snap path for %s", snapname)
	return ""
}

// TestWorkflow aims to do more than just exercise the API calls, but rather to
// also check that they do what they say on the tin.  This means importing the
// cephfs lib in addition to the admin lib and reading and writing to the
// subvolume, snapshot, and clone as appropriate.
func TestWorkflow(t *testing.T) {
	fsa := getFSAdmin(t)
	volume := "cephfs"
	group := "workflow1"

	// verify the volume we want to use
	l, err := fsa.ListVolumes()
	require.NoError(t, err)
	require.Contains(t, l, volume)

	err = fsa.CreateSubVolumeGroup(volume, group, nil)
	require.NoError(t, err)
	defer func() {
		err := fsa.RemoveSubVolumeGroup(volume, group)
		assert.NoError(t, err)
	}()

	subname := "files1"
	svopts := &SubVolumeOptions{
		Mode: 0777,
		Size: 2 * gibiByte,
	}
	err = fsa.CreateSubVolume(volume, group, subname, svopts)
	require.NoError(t, err)
	defer func() {
		err := fsa.RemoveSubVolume(volume, group, subname)
		assert.NoError(t, err)
	}()

	// getpath
	subPath, err := fsa.SubVolumePath(volume, group, subname)
	require.NoError(t, err)
	require.NotEqual(t, "", subPath)

	// connect to volume, cd to path (?)
	mount := fsConnect(t, "")
	defer func(mount *cephfs.MountInfo) {
		assert.NoError(t, mount.Unmount())
		assert.NoError(t, mount.Release())
	}(mount)

	err = mount.ChangeDir(subPath)
	require.NoError(t, err)

	// write some dirs & files
	err = mount.MakeDir("content1", 0770)
	require.NoError(t, err)

	writeFile(t, mount, "content1/robots.txt",
		[]byte("robbie\nr2\nbender\nclaptrap\n"))
	writeFile(t, mount, "content1/songs.txt",
		[]byte("none of them knew they were robots\n"))
	assert.NoError(t, mount.MakeDir("content1/emptyDir1", 0770))

	err = mount.MakeDir("content2", 0770)
	require.NoError(t, err)

	writeFile(t, mount, "content2/androids.txt",
		[]byte("data\nmarvin\n"))
	assert.NoError(t, mount.MakeDir("content2/docs", 0770))
	writeFile(t, mount, "content2/docs/lore.txt",
		[]byte("Compendium\nLegend\nLore\nDeadweight\nSpirit at Aphelion\n"))

	assert.NoError(t, mount.SyncFs())

	// take a snapshot

	snapname1 := "hotSpans1"
	err = fsa.CreateSubVolumeSnapshot(volume, group, subname, snapname1)
	require.NoError(t, err)
	defer func() {
		err := fsa.RemoveSubVolumeSnapshot(volume, group, subname, snapname1)
		assert.NoError(t, err)
	}()

	sinfo, err := fsa.SubVolumeSnapshotInfo(volume, group, subname, snapname1)
	require.NoError(t, err)
	require.NotNil(t, sinfo)

	time.Sleep(500 * time.Millisecond) // is there a race?

	// examine the snapshot
	snapPath := getSnapPath(t, mount, subPath, snapname1)
	require.NotEqual(t, "", snapPath)

	tempPath := path.Join(snapPath, "content1/robots.txt")
	txt := readFile(t, mount, tempPath)
	assert.Contains(t, string(txt), "robbie")

	// original subvol can be manipulated
	err = mount.Rename("content2/docs/lore.txt", "content1/lore.txt")
	assert.NoError(t, err)
	writeFile(t, mount, "content1/songs.txt",
		[]byte("none of them knew they were robots\nars moriendi\n"))

	// snapshot may not be modified
	err = mount.Rename(
		path.Join(snapPath, "content2/docs/lore.txt"),
		path.Join(snapPath, "content1/lore.txt"))
	assert.Error(t, err)
	txt = readFile(t, mount, path.Join(snapPath, "content2/docs/lore.txt"))
	assert.Contains(t, string(txt), "Spirit")

	// make a clone

	clonename := "files2"
	err = fsa.CloneSubVolumeSnapshot(
		volume, group, subname, snapname1, clonename,
		&CloneOptions{TargetGroup: group})
	var x NotProtectedError
	if errors.As(err, &x) {
		err = fsa.ProtectSubVolumeSnapshot(volume, group, subname, snapname1)
		assert.NoError(t, err)
		defer func() {
			err := fsa.UnprotectSubVolumeSnapshot(volume, group, subname, snapname1)
			assert.NoError(t, err)
		}()

		err = fsa.CloneSubVolumeSnapshot(
			volume, group, subname, snapname1, clonename,
			&CloneOptions{TargetGroup: group})
	}
	require.NoError(t, err)
	defer func() {
		err := fsa.ForceRemoveSubVolume(volume, group, clonename)
		assert.NoError(t, err)
	}()

	// wait for cloning to complete
	for done := false; !done; {
		status, err := fsa.CloneStatus(volume, group, clonename)
		assert.NoError(t, err)
		assert.NotNil(t, status)
		switch status.State {
		case ClonePending, CloneInProgress:
			time.Sleep(5 * time.Millisecond)
		case CloneComplete:
			done = true
		case CloneFailed:
			t.Fatal("clone failed")
		default:
			t.Fatalf("invalid status.State: %q", status.State)
		}
	}

	// examine the clone
	clonePath, err := fsa.SubVolumePath(volume, group, clonename)
	require.NoError(t, err)
	require.NotEqual(t, "", clonePath)

	txt = readFile(t, mount, path.Join(clonePath, "content1/robots.txt"))
	assert.Contains(t, string(txt), "robbie")

	// clones are r/w
	err = mount.Rename(
		path.Join(clonePath, "content2/docs/lore.txt"),
		path.Join(clonePath, "content1/lore.txt"))
	assert.NoError(t, err)
	txt = readFile(t, mount, path.Join(clonePath, "content1/lore.txt"))
	assert.Contains(t, string(txt), "Spirit")

	// it reflects what was in the snapshot
	txt = readFile(t, mount, path.Join(clonePath, "content1/songs.txt"))
	assert.Contains(t, string(txt), "robots")
	assert.NotContains(t, string(txt), "moriendi")

	// ... with it's own independent data
	writeFile(t, mount, path.Join(clonePath, "content1/songs.txt"),
		[]byte("none of them knew they were robots\nsweet charity\n"))

	// (orig)
	txt = readFile(t, mount, "content1/songs.txt")
	assert.Contains(t, string(txt), "robots")
	assert.Contains(t, string(txt), "moriendi")
	assert.NotContains(t, string(txt), "charity")

	// (clone)
	txt = readFile(t, mount, path.Join(clonePath, "content1/songs.txt"))
	assert.Contains(t, string(txt), "robots")
	assert.NotContains(t, string(txt), "moriendi")
	assert.Contains(t, string(txt), "charity")
}
<|endoftext|>


<commit_before>// Copyright 2015 ChaiShushan <chaishushan{AT}gmail.com>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ingore

// 打包 gopl-zh 为 zip 文件.
//
// 文件名格式: gopl-zh-20151001-2ae607.zip
package main

import (
	"archive/zip"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"
)

func main() {
	// Git版本号
	gitVersion := getGitCommitVersion()

	// zip文件名
	zipBaseName := fmt.Sprintf("gopl-zh-%s-%s", time.Now().Format("20060102"), gitVersion[:6])

	os.Remove(zipBaseName + ".zip")
	file, err := os.Create(zipBaseName + ".zip")
	if err != nil {
		log.Fatal("os.Create: ", err)
	}
	defer file.Close()

	zipFile := zip.NewWriter(file)
	defer zipFile.Close()

	// create /gopl-zh-20151001-2ae607/
	f, err := zipFile.Create(zipBaseName + "/")
	if err != nil {
		log.Fatal(err)
	}
	if _, err = f.Write([]byte("")); err != nil {
		log.Fatal(err)
	}

	dir := "_book"
	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			log.Fatal("filepath.Walk: ", err)
		}
		if info.IsDir() {
			return nil
		}
		relpath, err := filepath.Rel(dir, path)
		if err != nil {
			log.Fatal("filepath.Rel: ", err)
		}

		filename := filepath.ToSlash(relpath)
		if isIngoreFile(filename) {
			return nil
		}

		data, err := ioutil.ReadFile(path)
		if err != nil {
			log.Fatal("ioutil.ReadFile: ", err)
		}

		f, err := zipFile.Create(zipBaseName + "/" + filename)
		if err != nil {
			log.Fatal(err)
		}
		if _, err = f.Write(data); err != nil {
			log.Fatal(err)
		}

		fmt.Printf("%s\n", filename)
		return nil
	})

	fmt.Printf("Done\n")
}

// 获取Git最新的版本号
//
//	git log -1
//	commit 0460c1b3bb8fbb7e2fc88961e69aa37f4041d6c1
//	Merge: b2d582a e826457
//	Author: chai2010 <chaishushan{AT}gmail.com>
//	Date:   Mon Feb 1 08:04:44 2016 +0800
//
//		Merge pull request #249 from sunclx/patch-3
//
//		fix typo
func getGitCommitVersion() (version string) {
	cmdOut, err := exec.Command(`git`, `log`, `-1`).CombinedOutput()
	if err != nil {
		return "master"
	}
	for _, line := range strings.Split(string(cmdOut), "\n") {
		line := strings.TrimSpace(line)
		if strings.HasPrefix(line, "commit") {
			version = strings.TrimSpace(line[len("commit"):])
			return
		}
	}
	return "master"
}

func cpFile(dst, src string) {
	err := os.MkdirAll(filepath.Dir(dst), 0666)
	if err != nil && !os.IsExist(err) {
		log.Fatal("cpFile: ", err)
	}
	fsrc, err := os.Open(src)
	if err != nil {
		log.Fatal("cpFile: ", err)
	}
	defer fsrc.Close()

	fdst, err := os.Create(dst)
	if err != nil {
		log.Fatal("cpFile: ", err)
	}
	defer fdst.Close()
	if _, err = io.Copy(fdst, fsrc); err != nil {
		log.Fatal("cpFile: ", err)
	}
}

func isIngoreFile(path string) bool {
	if strings.HasPrefix(path, ".git") {
		return true
	}
	if strings.HasSuffix(path, ".gitignore") {
		return true
	}

	if strings.HasPrefix(path, "rpc.v2014") {
		return true
	}
	if strings.HasPrefix(path, "testdata") {
		return true
	}

	if strings.HasSuffix(path, "uis.zip") {
		return true
	}
	if strings.HasSuffix(path, ".go") {
		return true
	}

	return false
}
<commit_msg>打包时包含源文件<commit_after>// Copyright 2015 ChaiShushan <chaishushan{AT}gmail.com>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ingore

// 打包 gopl-zh 为 zip 文件.
//
// 文件名格式: gopl-zh-20151001-2ae607.zip
package main

import (
	"archive/zip"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"
)

func main() {
	// Git版本号
	gitVersion := getGitCommitVersion()

	// zip文件名
	zipBaseName := fmt.Sprintf("gopl-zh-%s-%s", time.Now().Format("20060102"), gitVersion[:6])

	// 导出Git
	exportGitToZip("./_book/" + zipBaseName + ".source.zip")

	os.Remove(zipBaseName + ".zip")
	file, err := os.Create(zipBaseName + ".zip")
	if err != nil {
		log.Fatal("os.Create: ", err)
	}
	defer file.Close()

	zipFile := zip.NewWriter(file)
	defer zipFile.Close()

	// create /gopl-zh-20151001-2ae607/
	f, err := zipFile.Create(zipBaseName + "/")
	if err != nil {
		log.Fatal(err)
	}
	if _, err = f.Write([]byte("")); err != nil {
		log.Fatal(err)
	}

	dir := "_book"
	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			log.Fatal("filepath.Walk: ", err)
		}
		if info.IsDir() {
			return nil
		}
		relpath, err := filepath.Rel(dir, path)
		if err != nil {
			log.Fatal("filepath.Rel: ", err)
		}

		filename := filepath.ToSlash(relpath)
		if isIngoreFile(filename) {
			return nil
		}

		data, err := ioutil.ReadFile(path)
		if err != nil {
			log.Fatal("ioutil.ReadFile: ", err)
		}

		f, err := zipFile.Create(zipBaseName + "/" + filename)
		if err != nil {
			log.Fatal(err)
		}
		if _, err = f.Write(data); err != nil {
			log.Fatal(err)
		}

		fmt.Printf("%s\n", filename)
		return nil
	})

	fmt.Printf("Done\n")
}

// 获取Git最新的版本号
//
//	git log -1
//	commit 0460c1b3bb8fbb7e2fc88961e69aa37f4041d6c1
//	Merge: b2d582a e826457
//	Author: chai2010 <chaishushan{AT}gmail.com>
//	Date:   Mon Feb 1 08:04:44 2016 +0800
//
//		Merge pull request #249 from sunclx/patch-3
//
//		fix typo
func getGitCommitVersion() (version string) {
	cmdOut, err := exec.Command(`git`, `log`, `-1`).CombinedOutput()
	if err != nil {
		return "master"
	}
	for _, line := range strings.Split(string(cmdOut), "\n") {
		line := strings.TrimSpace(line)
		if strings.HasPrefix(line, "commit") {
			version = strings.TrimSpace(line[len("commit"):])
			return
		}
	}
	return "master"
}

// 导出Git到Zip文件
func exportGitToZip(filename string) {
	if !strings.HasSuffix(filename, ".zip") {
		filename += ".zip"
	}
	if _, err := exec.Command(`git`, `archive`, `--format`, `zip`, `--output`, filename, `master`).CombinedOutput(); err != nil {
		log.Fatal("cpFile: ", err)
	}
}

func cpFile(dst, src string) {
	err := os.MkdirAll(filepath.Dir(dst), 0666)
	if err != nil && !os.IsExist(err) {
		log.Fatal("cpFile: ", err)
	}
	fsrc, err := os.Open(src)
	if err != nil {
		log.Fatal("cpFile: ", err)
	}
	defer fsrc.Close()

	fdst, err := os.Create(dst)
	if err != nil {
		log.Fatal("cpFile: ", err)
	}
	defer fdst.Close()
	if _, err = io.Copy(fdst, fsrc); err != nil {
		log.Fatal("cpFile: ", err)
	}
}

func isIngoreFile(path string) bool {
	if strings.HasPrefix(path, ".git") {
		return true
	}
	if strings.HasSuffix(path, ".gitignore") {
		return true
	}

	if strings.HasPrefix(path, "rpc.v2014") {
		return true
	}
	if strings.HasPrefix(path, "testdata") {
		return true
	}

	if strings.HasSuffix(path, "uis.zip") {
		return true
	}
	if strings.HasSuffix(path, ".go") {
		return true
	}

	return false
}
<|endoftext|>


<commit_before>/*
	This file is part of go-ethereum

	go-ethereum is free software: you can redistribute it and/or modify
	it under the terms of the GNU General Public License as published by
	the Free Software Foundation, either version 3 of the License, or
	(at your option) any later version.

	go-ethereum is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.

	You should have received a copy of the GNU General Public License
	along with go-ethereum.  If not, see <http://www.gnu.org/licenses/>.
*/
package rpc

import (
	"encoding/json"
	"github.com/ethereum/go-ethereum/logger"
	"io"
	"net/http"
	"github.com/ethereum/go-ethereum/ethutil"
	"github.com/ethereum/go-ethereum/state"
)

var rpclogger = logger.NewLogger("RPC")

type JsonWrapper struct{}

func (self JsonWrapper) Send(writer io.Writer, v interface{}) (n int, err error) {
	var payload []byte
	payload, err = json.Marshal(v)
	if err != nil {
		rpclogger.Fatalln("Error marshalling JSON", err)
		return 0, err
	}
	rpclogger.Infof("Sending payload: %s", payload)

	return writer.Write(payload)
}

func (self JsonWrapper) ParseRequestBody(req *http.Request) (RpcRequest, error) {
	var reqParsed RpcRequest

	// Convert JSON to native types
	d := json.NewDecoder(req.Body)
	// d.UseNumber()
	defer req.Body.Close()
	err := d.Decode(&reqParsed)

	if err != nil {
		rpclogger.Errorln("Error decoding JSON: ", err)
		return reqParsed, err
	}
	rpclogger.DebugDetailf("Parsed request: %s", reqParsed)

	return reqParsed, nil
}

func toHex(b []byte) string {
	return "0x" + ethutil.Bytes2Hex(b)
}
func fromHex(s string) []byte {
	if len(s) > 1 {
		if s[0:2] == "0x" {
			s = s[2:]
		}
		return ethutil.Hex2Bytes(s)
	}
	return nil
}

type RpcServer interface {
	Start()
	Stop()
}

type Log struct {
	Address string   `json:"address"`
	Topics  []string `json:"topics"`
	Data    string   `json:"data"`
}

func toLogs(logs state.Logs) (ls []Log) {
	ls = make([]Log, len(logs))

	for i, log := range logs {
		var l Log
		l.Topics = make([]string, len(log.Topics()))
		l.Address = toHex(log.Address())
		l.Data = toHex(log.Data())
		for j, topic := range log.Topics() {
			l.Topics[j] = toHex(topic)
		}
		ls[i] = l
	}

	return
}
<commit_msg>Decrease log level for Sending RPC payload<commit_after>/*
	This file is part of go-ethereum

	go-ethereum is free software: you can redistribute it and/or modify
	it under the terms of the GNU General Public License as published by
	the Free Software Foundation, either version 3 of the License, or
	(at your option) any later version.

	go-ethereum is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.

	You should have received a copy of the GNU General Public License
	along with go-ethereum.  If not, see <http://www.gnu.org/licenses/>.
*/
package rpc

import (
	"encoding/json"
	"io"
	"net/http"

	"github.com/ethereum/go-ethereum/ethutil"
	"github.com/ethereum/go-ethereum/logger"
	"github.com/ethereum/go-ethereum/state"
)

var rpclogger = logger.NewLogger("RPC")

type JsonWrapper struct{}

func (self JsonWrapper) Send(writer io.Writer, v interface{}) (n int, err error) {
	var payload []byte
	payload, err = json.Marshal(v)
	if err != nil {
		rpclogger.Fatalln("Error marshalling JSON", err)
		return 0, err
	}
	rpclogger.DebugDetailf("Sending payload: %s", payload)

	return writer.Write(payload)
}

func (self JsonWrapper) ParseRequestBody(req *http.Request) (RpcRequest, error) {
	var reqParsed RpcRequest

	// Convert JSON to native types
	d := json.NewDecoder(req.Body)
	// d.UseNumber()
	defer req.Body.Close()
	err := d.Decode(&reqParsed)

	if err != nil {
		rpclogger.Errorln("Error decoding JSON: ", err)
		return reqParsed, err
	}
	rpclogger.DebugDetailf("Parsed request: %s", reqParsed)

	return reqParsed, nil
}

func toHex(b []byte) string {
	return "0x" + ethutil.Bytes2Hex(b)
}
func fromHex(s string) []byte {
	if len(s) > 1 {
		if s[0:2] == "0x" {
			s = s[2:]
		}
		return ethutil.Hex2Bytes(s)
	}
	return nil
}

type RpcServer interface {
	Start()
	Stop()
}

type Log struct {
	Address string   `json:"address"`
	Topics  []string `json:"topics"`
	Data    string   `json:"data"`
}

func toLogs(logs state.Logs) (ls []Log) {
	ls = make([]Log, len(logs))

	for i, log := range logs {
		var l Log
		l.Topics = make([]string, len(log.Topics()))
		l.Address = toHex(log.Address())
		l.Data = toHex(log.Data())
		for j, topic := range log.Topics() {
			l.Topics[j] = toHex(topic)
		}
		ls[i] = l
	}

	return
}
<|endoftext|>


<commit_before>package cmd

import (
	"context"
	"fmt"
	"html/template"
	"log"
	"os"
	"os/exec"

	"github.com/gobuffalo/buffalo/buffalo/cmd/generate"
	"github.com/markbates/refresh/refresh"
	"github.com/spf13/cobra"
)

// devCmd represents the dev command
var devCmd = &cobra.Command{
	Use:   "dev",
	Short: "Runs your Buffalo app in 'development' mode",
	Long: `Runs your Buffalo app in 'development' mode.
This includes rebuilding your application when files change.
This behavior can be changed in your .buffalo.dev.yml file.`,
	Run: func(c *cobra.Command, args []string) {
		defer func() {
			msg := "There was a problem starting the dev server, Please review the troubleshooting docs: %s\n"
			cause := "Unknown"
			if r := recover(); r != nil {
				if err, ok := r.(error); ok {
					cause = err.Error()
				}
			}
			fmt.Printf(msg, cause)
		}()
		os.Setenv("GO_ENV", "development")
		ctx := context.Background()
		ctx, cancelFunc := context.WithCancel(ctx)
		go func() {
			err := startDevServer(ctx)
			if err != nil {
				cancelFunc()
				log.Fatal(err)
			}
		}()
		go func() {
			err := startWebpack(ctx)
			if err != nil {
				cancelFunc()
				log.Fatal(err)
			}
		}()
		// wait for the ctx to finish
		<-ctx.Done()
	},
}

func startWebpack(ctx context.Context) error {
	cfgFile := "./webpack.config.js"
	_, err := os.Stat(cfgFile)
	if err != nil {
		// there's no webpack, so don't do anything
		return nil
	}
	cmd := exec.Command(generate.WebpackPath, "--watch")
	cmd.Stdin = os.Stdin
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	return cmd.Run()
}

func startDevServer(ctx context.Context) error {
	cfgFile := "./.buffalo.dev.yml"
	_, err := os.Stat(cfgFile)
	if err != nil {
		f, err := os.Create(cfgFile)
		if err != nil {
			return err
		}
		t, err := template.New("").Parse(nRefresh)
		if err != nil {
			return err
		}
		err = t.Execute(f, map[string]interface{}{
			"name": "buffalo",
		})
		if err != nil {
			return err
		}
	}
	c := &refresh.Configuration{}
	err = c.Load(cfgFile)
	if err != nil {
		return err
	}
	r := refresh.New(c)
	return r.Start()
}

func init() {
	RootCmd.AddCommand(devCmd)
}
<commit_msg>fixed the generation of the refresh file if it didn't exist<commit_after>package cmd

import (
	"context"
	"fmt"
	"log"
	"os"
	"os/exec"

	"github.com/gobuffalo/buffalo/buffalo/cmd/generate"
	"github.com/gobuffalo/velvet"
	"github.com/markbates/refresh/refresh"
	"github.com/spf13/cobra"
)

// devCmd represents the dev command
var devCmd = &cobra.Command{
	Use:   "dev",
	Short: "Runs your Buffalo app in 'development' mode",
	Long: `Runs your Buffalo app in 'development' mode.
This includes rebuilding your application when files change.
This behavior can be changed in your .buffalo.dev.yml file.`,
	Run: func(c *cobra.Command, args []string) {
		defer func() {
			msg := "There was a problem starting the dev server, Please review the troubleshooting docs: %s\n"
			cause := "Unknown"
			if r := recover(); r != nil {
				if err, ok := r.(error); ok {
					cause = err.Error()
				}
			}
			fmt.Printf(msg, cause)
		}()
		os.Setenv("GO_ENV", "development")
		ctx := context.Background()
		ctx, cancelFunc := context.WithCancel(ctx)
		go func() {
			err := startDevServer(ctx)
			if err != nil {
				cancelFunc()
				log.Fatal(err)
			}
		}()
		go func() {
			err := startWebpack(ctx)
			if err != nil {
				cancelFunc()
				log.Fatal(err)
			}
		}()
		// wait for the ctx to finish
		<-ctx.Done()
	},
}

func startWebpack(ctx context.Context) error {
	cfgFile := "./webpack.config.js"
	_, err := os.Stat(cfgFile)
	if err != nil {
		// there's no webpack, so don't do anything
		return nil
	}
	cmd := exec.Command(generate.WebpackPath, "--watch")
	cmd.Stdin = os.Stdin
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	return cmd.Run()
}

func startDevServer(ctx context.Context) error {
	cfgFile := "./.buffalo.dev.yml"
	_, err := os.Stat(cfgFile)
	if err != nil {
		f, err := os.Create(cfgFile)
		if err != nil {
			return err
		}
		defer f.Close()
		t, err := velvet.Render(nRefresh, velvet.NewContextWith(map[string]interface{}{
			"name": "buffalo",
		}))
		if err != nil {
			return err
		}
		_, err = f.WriteString(t)
		if err != nil {
			return err
		}
	}
	c := &refresh.Configuration{}
	err = c.Load(cfgFile)
	if err != nil {
		return err
	}
	r := refresh.NewWithContext(c, ctx)
	return r.Start()
}

func init() {
	RootCmd.AddCommand(devCmd)
}
<|endoftext|>


<commit_before>package controller

import (
	"net/http"
	"net/http/httptest"
	"os"
	"testing"

	"hermes/ratings/handler"

	"github.com/gavv/httpexpect"
	"github.com/labstack/echo"
)

var (
	routes = map[string]echo.HandlerFunc{"PostStats": PostStats}
	port   = getPort("HERMES_STATS_PORT", 7000)
)

func TestCount(t *testing.T) {
	handler := handler.Handler(3000, routes)
	server := httptest.NewServer(handler)

	defer server.Close()

	server.URL = "http://localhost:" + port

	e := httpexpect.WithConfig(httpexpect.Config{
		BaseURL:  server.URL,
		Reporter: httpexpect.NewAssertReporter(t),
		Printers: []httpexpect.Printer{
			httpexpect.NewDebugPrinter(t, true),
		},
	})

	query := `
	{
		"query": "query Example($field: Field!) { count(field: $field) }",
		"variables": {
		  "field": {
			  "name": "messages"
		  }
		}
	  }
	`

	meta := map[string]interface{}{
		"meta": map[string]interface{}{
			"code":    http.StatusOK,
			"message": "OK"}}

	data := map[string]interface{}{}

	r := e.POST("/ratings").
		WithHeader("Content-Type", "application/json; charset=UTF-8").
		WithHeader("Accept", "application/json").
		WithText(query).
		Expect()

	r.Status(http.StatusOK)
	r.Header("Content-Type").Equal("application/json; charset=UTF-8")

	json := r.JSON().Object()

	json.ContainsMap(meta)
	json.ContainsMap(data).ContainsKey("count")
}

func TestCountWithOperator(t *testing.T) {
	handler := handler.Handler(3000, routes)
	server := httptest.NewServer(handler)

	defer server.Close()

	server.URL = "http://localhost:" + port

	e := httpexpect.WithConfig(httpexpect.Config{
		BaseURL:  server.URL,
		Reporter: httpexpect.NewAssertReporter(t),
		Printers: []httpexpect.Printer{
			httpexpect.NewDebugPrinter(t, true),
		},
	})

	query := `
	{
		"query": "query Example($field: Field!) { count(field: $field) }",
		"variables": {
		  "field": {
			  "name": "messages.status",
			  "eq": 0
		  }
		}
	  }
	`

	meta := map[string]interface{}{
		"meta": map[string]interface{}{
			"code":    http.StatusOK,
			"message": "OK"}}

	data := map[string]interface{}{}

	r := e.POST("/ratings").
		WithHeader("Content-Type", "application/json; charset=UTF-8").
		WithHeader("Accept", "application/json").
		WithText(query).
		Expect()

	r.Status(http.StatusOK)
	r.Header("Content-Type").Equal("application/json; charset=UTF-8")

	json := r.JSON().Object()

	json.ContainsMap(meta)
	json.ContainsMap(data).ContainsKey("count")
}

func TestCountAnd(t *testing.T) {
	handler := handler.Handler(3000, routes)
	server := httptest.NewServer(handler)

	defer server.Close()

	server.URL = "http://localhost:" + port

	e := httpexpect.WithConfig(httpexpect.Config{
		BaseURL:  server.URL,
		Reporter: httpexpect.NewAssertReporter(t),
		Printers: []httpexpect.Printer{
			httpexpect.NewDebugPrinter(t, true),
		},
	})

	query := `
	{
		"query": "query Example($field: Field!, $and: [Field!]) { count(field: $field, and: $and) }",
		"variables": {
		  "field": {
			  "name": "messages.status",
			  "eq": 0
		  },
		  "and": {
			  "name": "messages.rating_id",
			  "lte": 5
		  }
		}
	  }
	`

	meta := map[string]interface{}{
		"meta": map[string]interface{}{
			"code":    http.StatusOK,
			"message": "OK"}}

	data := map[string]interface{}{}

	r := e.POST("/ratings").
		WithHeader("Content-Type", "application/json; charset=UTF-8").
		WithHeader("Accept", "application/json").
		WithText(query).
		Expect()

	r.Status(http.StatusOK)
	r.Header("Content-Type").Equal("application/json; charset=UTF-8")

	json := r.JSON().Object()

	json.ContainsMap(meta)
	json.ContainsMap(data).ContainsKey("count")
}

func TestCountOr(t *testing.T) {
	handler := handler.Handler(3000, routes)
	server := httptest.NewServer(handler)

	defer server.Close()

	server.URL = "http://localhost:" + port

	e := httpexpect.WithConfig(httpexpect.Config{
		BaseURL:  server.URL,
		Reporter: httpexpect.NewAssertReporter(t),
		Printers: []httpexpect.Printer{
			httpexpect.NewDebugPrinter(t, true),
		},
	})

	query := `
	{
		"query": "query Example($field: Field!, $or: [Field!]) { count(field: $field, or: $or) }",
		"variables": {
		  "field": {
			  "name": "messages.status",
			  "eq": 0
		  },
		  "or": {
			  "name": "messages.rating_id",
			  "lte": 5
		  }
		}
	  }
	`

	meta := map[string]interface{}{
		"meta": map[string]interface{}{
			"code":    http.StatusOK,
			"message": "OK"}}

	data := map[string]interface{}{}

	r := e.POST("/ratings").
		WithHeader("Content-Type", "application/json; charset=UTF-8").
		WithHeader("Accept", "application/json").
		WithText(query).
		Expect()

	r.Status(http.StatusOK)
	r.Header("Content-Type").Equal("application/json; charset=UTF-8")

	json := r.JSON().Object()

	json.ContainsMap(meta)
	json.ContainsMap(data).ContainsKey("count")
}

func getPort(env string, defaultPort int) string {
	port := os.Getenv(env)

	if len(port) == 0 {
		return string(defaultPort)
	}

	return port
}
<commit_msg>Added average tests<commit_after>package controller

import (
	"net/http"
	"net/http/httptest"
	"os"
	"testing"

	"hermes/ratings/handler"

	"github.com/gavv/httpexpect"
	"github.com/labstack/echo"
)

var (
	routes = map[string]echo.HandlerFunc{"PostStats": PostStats}
	port   = getPort("HERMES_STATS_PORT", 7000)
)

func TestCount(t *testing.T) {
	handler := handler.Handler(3000, routes)
	server := httptest.NewServer(handler)

	defer server.Close()

	server.URL = "http://localhost:" + port

	e := httpexpect.WithConfig(httpexpect.Config{
		BaseURL:  server.URL,
		Reporter: httpexpect.NewAssertReporter(t),
		Printers: []httpexpect.Printer{
			httpexpect.NewDebugPrinter(t, true),
		},
	})

	query := `
	{
		"query": "query Example($field: Field!) { count(field: $field) }",
		"variables": {
		  "field": {
			  "name": "messages"
		  }
		}
	  }
	`

	meta := map[string]interface{}{
		"meta": map[string]interface{}{
			"code":    http.StatusOK,
			"message": "OK"}}

	r := e.POST("/ratings").
		WithHeader("Content-Type", "application/json; charset=UTF-8").
		WithHeader("Accept", "application/json").
		WithText(query).
		Expect()

	r.Status(http.StatusOK)
	r.Header("Content-Type").Equal("application/json; charset=UTF-8")

	json := r.JSON().Object()

	json.ContainsMap(meta)
	json.ContainsKey("data").ContainsKey("count")
	json.NotContainsKey("errors")
}

func TestCountWithOperator(t *testing.T) {
	handler := handler.Handler(3000, routes)
	server := httptest.NewServer(handler)

	defer server.Close()

	server.URL = "http://localhost:" + port

	e := httpexpect.WithConfig(httpexpect.Config{
		BaseURL:  server.URL,
		Reporter: httpexpect.NewAssertReporter(t),
		Printers: []httpexpect.Printer{
			httpexpect.NewDebugPrinter(t, true),
		},
	})

	query := `
	{
		"query": "query Example($field: Field!) { count(field: $field) }",
		"variables": {
		  "field": {
			  "name": "messages.status",
			  "eq": 0
		  }
		}
	  }
	`

	meta := map[string]interface{}{
		"meta": map[string]interface{}{
			"code":    http.StatusOK,
			"message": "OK"}}

	r := e.POST("/ratings").
		WithHeader("Content-Type", "application/json; charset=UTF-8").
		WithHeader("Accept", "application/json").
		WithText(query).
		Expect()

	r.Status(http.StatusOK)
	r.Header("Content-Type").Equal("application/json; charset=UTF-8")

	json := r.JSON().Object()

	json.ContainsMap(meta)
	json.ContainsKey("data").ContainsKey("count")
	json.NotContainsKey("errors")
}

func TestCountAnd(t *testing.T) {
	handler := handler.Handler(3000, routes)
	server := httptest.NewServer(handler)

	defer server.Close()

	server.URL = "http://localhost:" + port

	e := httpexpect.WithConfig(httpexpect.Config{
		BaseURL:  server.URL,
		Reporter: httpexpect.NewAssertReporter(t),
		Printers: []httpexpect.Printer{
			httpexpect.NewDebugPrinter(t, true),
		},
	})

	query := `
	{
		"query": "query Example($field: Field!, $and: [Field!]) { count(field: $field, and: $and) }",
		"variables": {
		  "field": {
			  "name": "messages.status",
			  "eq": 0
		  },
		  "and": {
			  "name": "messages.rating_id",
			  "lte": 5
		  }
		}
	  }
	`

	meta := map[string]interface{}{
		"meta": map[string]interface{}{
			"code":    http.StatusOK,
			"message": "OK"}}

	r := e.POST("/ratings").
		WithHeader("Content-Type", "application/json; charset=UTF-8").
		WithHeader("Accept", "application/json").
		WithText(query).
		Expect()

	r.Status(http.StatusOK)
	r.Header("Content-Type").Equal("application/json; charset=UTF-8")

	json := r.JSON().Object()

	json.ContainsMap(meta)
	json.ContainsKey("data").ContainsKey("count")
	json.NotContainsKey("errors")
}

func TestCountOr(t *testing.T) {
	handler := handler.Handler(3000, routes)
	server := httptest.NewServer(handler)

	defer server.Close()

	server.URL = "http://localhost:" + port

	e := httpexpect.WithConfig(httpexpect.Config{
		BaseURL:  server.URL,
		Reporter: httpexpect.NewAssertReporter(t),
		Printers: []httpexpect.Printer{
			httpexpect.NewDebugPrinter(t, true),
		},
	})

	query := `
	{
		"query": "query Example($field: Field!, $or: [Field!]) { count(field: $field, or: $or) }",
		"variables": {
		  "field": {
			  "name": "messages.status",
			  "eq": 0
		  },
		  "or": {
			  "name": "messages.rating_id",
			  "lte": 5
		  }
		}
	  }
	`

	meta := map[string]interface{}{
		"meta": map[string]interface{}{
			"code":    http.StatusOK,
			"message": "OK"}}

	r := e.POST("/ratings").
		WithHeader("Content-Type", "application/json; charset=UTF-8").
		WithHeader("Accept", "application/json").
		WithText(query).
		Expect()

	r.Status(http.StatusOK)
	r.Header("Content-Type").Equal("application/json; charset=UTF-8")

	json := r.JSON().Object()

	json.ContainsMap(meta)
	json.ContainsKey("data").ContainsKey("count")
	json.NotContainsKey("errors")
}

func TestCountInvalidTableBadRequest(t *testing.T) {
	handler := handler.Handler(3000, routes)
	server := httptest.NewServer(handler)

	defer server.Close()

	server.URL = "http://localhost:" + port

	e := httpexpect.WithConfig(httpexpect.Config{
		BaseURL:  server.URL,
		Reporter: httpexpect.NewAssertReporter(t),
		Printers: []httpexpect.Printer{
			httpexpect.NewDebugPrinter(t, true),
		},
	})

	query := `
	{
		"query": "query Example($field: Field!) { count(field: $field) }",
		"variables": {
		  "field": {
			  "name": "example"
		  }
		}
	  }
	`

	meta := map[string]interface{}{
		"meta": map[string]interface{}{
			"code":    http.StatusBadRequest,
			"message": "Bad Request"}}

	r := e.POST("/ratings").
		WithHeader("Content-Type", "application/json; charset=UTF-8").
		WithHeader("Accept", "application/json").
		WithText(query).
		Expect()

	r.Status(http.StatusOK)
	r.Header("Content-Type").Equal("application/json; charset=UTF-8")

	json := r.JSON().Object()

	json.ContainsMap(meta)
	json.NotContainsKey("data")
	json.ContainsKey("errors")
}

func TestCountInvalidFieldBadRequest(t *testing.T) {
	handler := handler.Handler(3000, routes)
	server := httptest.NewServer(handler)

	defer server.Close()

	server.URL = "http://localhost:" + port

	e := httpexpect.WithConfig(httpexpect.Config{
		BaseURL:  server.URL,
		Reporter: httpexpect.NewAssertReporter(t),
		Printers: []httpexpect.Printer{
			httpexpect.NewDebugPrinter(t, true),
		},
	})

	query := `
	{
		"query": "query Example($field: Field!) { count(field: $field) }",
		"variables": {
		  "field": {
			  "name": "ratings.example"
		  }
		}
	  }
	`

	meta := map[string]interface{}{
		"meta": map[string]interface{}{
			"code":    http.StatusBadRequest,
			"message": "Bad Request"}}

	r := e.POST("/ratings").
		WithHeader("Content-Type", "application/json; charset=UTF-8").
		WithHeader("Accept", "application/json").
		WithText(query).
		Expect()

	r.Status(http.StatusOK)
	r.Header("Content-Type").Equal("application/json; charset=UTF-8")

	json := r.JSON().Object()

	json.ContainsMap(meta)
	json.NotContainsKey("data")
	json.ContainsKey("errors")
}

func TestAverage(t *testing.T) {
	handler := handler.Handler(3000, routes)
	server := httptest.NewServer(handler)

	defer server.Close()

	server.URL = "http://localhost:" + port

	e := httpexpect.WithConfig(httpexpect.Config{
		BaseURL:  server.URL,
		Reporter: httpexpect.NewAssertReporter(t),
		Printers: []httpexpect.Printer{
			httpexpect.NewDebugPrinter(t, true),
		},
	})

	query := `
	{
		"query": "query Example($field: Field!) { average(field: $field) }",
		"variables": {
		  "field": {
			  "name": "ratings.rating"
		  }
		}
	  }
	`

	meta := map[string]interface{}{
		"meta": map[string]interface{}{
			"code":    http.StatusOK,
			"message": "OK"}}

	r := e.POST("/ratings").
		WithHeader("Content-Type", "application/json; charset=UTF-8").
		WithHeader("Accept", "application/json").
		WithText(query).
		Expect()

	r.Status(http.StatusOK)
	r.Header("Content-Type").Equal("application/json; charset=UTF-8")

	json := r.JSON().Object()

	json.ContainsMap(meta)
	json.ContainsKey("data").ContainsKey("average")
	json.NotContainsKey("errors")
}

func TestAverageAnd(t *testing.T) {
	handler := handler.Handler(3000, routes)
	server := httptest.NewServer(handler)

	defer server.Close()

	server.URL = "http://localhost:" + port

	e := httpexpect.WithConfig(httpexpect.Config{
		BaseURL:  server.URL,
		Reporter: httpexpect.NewAssertReporter(t),
		Printers: []httpexpect.Printer{
			httpexpect.NewDebugPrinter(t, true),
		},
	})

	query := `
	{
		"query": "query Example($field: Field!, $and: [Field!]) { average(field: $field, and: $and) }",
		"variables": {
		  "field": {
			  "name": "ratings.rating"
		  },
		  "and": {
			  "name": "ratings.has_message",
			  "eq": true
		  }
		}
	  }
	`

	meta := map[string]interface{}{
		"meta": map[string]interface{}{
			"code":    http.StatusOK,
			"message": "OK"}}

	r := e.POST("/ratings").
		WithHeader("Content-Type", "application/json; charset=UTF-8").
		WithHeader("Accept", "application/json").
		WithText(query).
		Expect()

	r.Status(http.StatusOK)
	r.Header("Content-Type").Equal("application/json; charset=UTF-8")

	json := r.JSON().Object()

	json.ContainsMap(meta)
	json.ContainsKey("data").ContainsKey("average")
	json.NotContainsKey("errors")
}

func TestAverageOr(t *testing.T) {
	handler := handler.Handler(3000, routes)
	server := httptest.NewServer(handler)

	defer server.Close()

	server.URL = "http://localhost:" + port

	e := httpexpect.WithConfig(httpexpect.Config{
		BaseURL:  server.URL,
		Reporter: httpexpect.NewAssertReporter(t),
		Printers: []httpexpect.Printer{
			httpexpect.NewDebugPrinter(t, true),
		},
	})

	query := `
	{
		"query": "query Example($field: Field!, $or: [Field!]) { average(field: $field, or: $or) }",
		"variables": {
		  "field": {
			  "name": "ratings.rating"
		  },
		  "or": {
			"name": "ratings.has_message",
			"eq": true
		  }
		}
	  }
	`

	meta := map[string]interface{}{
		"meta": map[string]interface{}{
			"code":    http.StatusOK,
			"message": "OK"}}

	r := e.POST("/ratings").
		WithHeader("Content-Type", "application/json; charset=UTF-8").
		WithHeader("Accept", "application/json").
		WithText(query).
		Expect()

	r.Status(http.StatusOK)
	r.Header("Content-Type").Equal("application/json; charset=UTF-8")

	json := r.JSON().Object()

	json.ContainsMap(meta)
	json.ContainsKey("data").ContainsKey("average")
	json.NotContainsKey("errors")
}

func TestAverageInvalidTableBadRequest(t *testing.T) {
	handler := handler.Handler(3000, routes)
	server := httptest.NewServer(handler)

	defer server.Close()

	server.URL = "http://localhost:" + port

	e := httpexpect.WithConfig(httpexpect.Config{
		BaseURL:  server.URL,
		Reporter: httpexpect.NewAssertReporter(t),
		Printers: []httpexpect.Printer{
			httpexpect.NewDebugPrinter(t, true),
		},
	})

	query := `
	{
		"query": "query Example($field: Field!) { average(field: $field) }",
		"variables": {
		  "field": {
			  "name": "example"
		  }
		}
	  }
	`

	meta := map[string]interface{}{
		"meta": map[string]interface{}{
			"code":    http.StatusBadRequest,
			"message": "Bad Request"}}

	r := e.POST("/ratings").
		WithHeader("Content-Type", "application/json; charset=UTF-8").
		WithHeader("Accept", "application/json").
		WithText(query).
		Expect()

	r.Status(http.StatusOK)
	r.Header("Content-Type").Equal("application/json; charset=UTF-8")

	json := r.JSON().Object()

	json.ContainsMap(meta)
	json.NotContainsKey("data")
	json.ContainsKey("errors")
}

func TestAverageInvalidFieldBadRequest(t *testing.T) {
	handler := handler.Handler(3000, routes)
	server := httptest.NewServer(handler)

	defer server.Close()

	server.URL = "http://localhost:" + port

	e := httpexpect.WithConfig(httpexpect.Config{
		BaseURL:  server.URL,
		Reporter: httpexpect.NewAssertReporter(t),
		Printers: []httpexpect.Printer{
			httpexpect.NewDebugPrinter(t, true),
		},
	})

	query := `
	{
		"query": "query Example($field: Field!) { average(field: $field) }",
		"variables": {
		  "field": {
			  "name": "ratings.example"
		  }
		}
	  }
	`

	meta := map[string]interface{}{
		"meta": map[string]interface{}{
			"code":    http.StatusBadRequest,
			"message": "Bad Request"}}

	r := e.POST("/ratings").
		WithHeader("Content-Type", "application/json; charset=UTF-8").
		WithHeader("Accept", "application/json").
		WithText(query).
		Expect()

	r.Status(http.StatusOK)
	r.Header("Content-Type").Equal("application/json; charset=UTF-8")

	json := r.JSON().Object()

	json.ContainsMap(meta)
	json.NotContainsKey("data")
	json.ContainsKey("errors")
}

func TestAverageNoFieldBadRequest(t *testing.T) {
	handler := handler.Handler(3000, routes)
	server := httptest.NewServer(handler)

	defer server.Close()

	server.URL = "http://localhost:" + port

	e := httpexpect.WithConfig(httpexpect.Config{
		BaseURL:  server.URL,
		Reporter: httpexpect.NewAssertReporter(t),
		Printers: []httpexpect.Printer{
			httpexpect.NewDebugPrinter(t, true),
		},
	})

	query := `
	{
		"query": "query Example($field: Field!) { average(field: $field) }",
		"variables": {
		  "field": {
			  "name": "ratings"
		  }
		}
	  }
	`

	meta := map[string]interface{}{
		"meta": map[string]interface{}{
			"code":    http.StatusBadRequest,
			"message": "Bad Request"}}

	r := e.POST("/ratings").
		WithHeader("Content-Type", "application/json; charset=UTF-8").
		WithHeader("Accept", "application/json").
		WithText(query).
		Expect()

	r.Status(http.StatusOK)
	r.Header("Content-Type").Equal("application/json; charset=UTF-8")

	json := r.JSON().Object()

	json.ContainsMap(meta)
	json.NotContainsKey("data")
	json.ContainsKey("errors")
}

func getPort(env string, defaultPort int) string {
	port := os.Getenv(env)

	if len(port) == 0 {
		return string(defaultPort)
	}

	return port
}
<|endoftext|>


<commit_before>package docstore

// package docstore

// TODO(tsileo): use   -tags 'prod' for building binary

import (
	"bytes"
	"fmt"
	"path/filepath"
	"time"

	"github.com/blevesearch/segment"
	log "github.com/inconshreveable/log15"
	"github.com/reiver/go-porterstemmer"
	"github.com/yuin/gopher-lua"

	luautil "a4.io/blobstash/pkg/apps/luautil"
)

type QueryMatcher interface {
	Match(map[string]interface{}) (bool, error)
	Close() error
}

type MatchAllEngine struct{}

func (mae *MatchAllEngine) Match(_ map[string]interface{}) (bool, error) {
	return true, nil
}

func (mae *MatchAllEngine) Close() error { return nil }

type LuaQueryEngine struct {
	storedQueries   map[string]*storedQuery // Stored query store
	storedQueryName string                  // Requested stored query name if any

	code  string
	query interface{} // Raw query
	q     lua.LValue

	matchFunc func(map[string]interface{}) (bool, error)
	L         *lua.LState // Lua state that will live the whole query

	logger log.Logger
}

func (lqe *LuaQueryEngine) Close() error {
	lqe.L.Close()
	return nil
}

func setGlobals(L *lua.LState) {
	// FIXME(tsileo): a `use_index(index_field, value)` and have the optimizer use it
	L.SetGlobal("porterstemmer", L.NewFunction(ltokenize))
	L.SetGlobal("porterstemmer_stem", L.NewFunction(stem))
}

func (docstore *DocStore) newLuaQueryEngine(query *query) (*LuaQueryEngine, error) {
	engine := &LuaQueryEngine{
		storedQueries: docstore.storedQueries,
		query:         query.storedQueryArgs,

		// XXX(tsileo): code expects a Lua script that returns a function(doc) for matching docs
		code: queryToScript(query), // FIXME(tsileo): query.script seems not to work

		storedQueryName: query.storedQuery,
		L:               lua.NewState(),
		q:               lua.LNil,
		logger:          docstore.logger.New("submodule", "lua_query_engine"),
	}
	fmt.Printf("code=\n\n%s\n\n", engine.code)
	setGlobals(engine.L)
	engine.logger.Debug("init", "query", engine.query)
	// Parse the Lua query, which should be defined as a `function(doc) -> bool`, we parse it only once, then we got
	// a "Lua func" Go object which we can call repeatedly for each document.
	// XXX(tsileo): keep the function (along with the Lua context `L` for a few minutes) in a cache, so if a client is paginating
	// through results, it will reuse the func/Lua context. (Cache[hash(script)] = FuncWithContextReadyToCall)
	var ret *lua.LFunction
	if engine.query != nil {
		if engine.storedQueryName != "" {
			// XXX(tsileo): concerns: the script should be checked at startup because right now,
			// a user have to actually try a query before we can see if it's valud Lua.
			engine.logger.Debug("loading stored query", "name", engine.storedQueryName)
			squery, ok := engine.storedQueries[engine.storedQueryName]
			if !ok {
				return nil, fmt.Errorf("Unknown stored query name")
			}
			luautil.AddToPath(engine.L, filepath.Dir(squery.Main))
			engine.L.SetGlobal("query", luautil.InterfaceToLValue(engine.L, engine.query))
			if err := engine.L.DoFile(squery.Main); err != nil {
				panic(err)
			}
			ret = engine.L.Get(-1).(*lua.LFunction)
		}
	} else {
		// XXX(tsileo): queryToString converted the basic function to a script retunring a function
		if err := engine.L.DoString(engine.code); err != nil {
			return nil, err
		}
		ret = engine.L.Get(-1).(*lua.LFunction)
		fmt.Printf("extracted fun %v\n", ret)
	}
	if ret != nil {
		matchDoc := func(doc map[string]interface{}) (bool, error) {
			if err := engine.L.CallByParam(lua.P{
				Fn:      ret,
				NRet:    1,
				Protect: true,
			}, luautil.InterfaceToLValue(engine.L, doc)); err != nil {
				return false, err
			}
			if engine.L.Get(-1) == lua.LTrue {
				return true, nil
			}
			return false, nil
		}

		engine.matchFunc = matchDoc
	}
	return engine, nil
}

func (lqe *LuaQueryEngine) Match(doc map[string]interface{}) (bool, error) {
	start := time.Now()
	var out bool
	L := lqe.L

	if lqe.matchFunc != nil {
		return lqe.matchFunc(doc)
	}

	// FIXME(tsileo): REMOVE ME, DEAD CODE (ensure it is)
	L.SetGlobal("doc", luautil.InterfaceToLValue(L, doc))
	// TODO(tsileo): a debug mode for debug print
	// TODO(tsileo): harvesine function for geoquery
	// TODO(tsileo): handle near:<place>, is:archived, contains:<link|todo|...>, tag:mytag

	// XXX(tsileo): be able to fetch a blob and parse it??
	if err := lqe.L.DoString(lqe.code); err != nil {
		return out, err
	}
	ret := lqe.L.Get(-1)
	if ret == lua.LTrue {
		out = true
	}
	lqe.logger.Debug("match code ran", "duration", time.Since(start))
	lqe.L.Pop(1)
	return out, nil
}

func stem(L *lua.LState) int {
	in := L.ToString(1)
	L.Push(lua.LString(porterstemmer.StemString(in)))
	return 1
}

func ltokenize(L *lua.LState) int {
	in := L.ToString(1)
	out, err := tokenize([]byte(in))
	if err != nil {
		panic(err)
	}
	L.Push(luautil.InterfaceToLValue(L, out))
	return 1
}

func tokenize(data []byte) (map[string]interface{}, error) {
	out := map[string]interface{}{}
	segmenter := segment.NewWordSegmenter(bytes.NewReader(data))
	for segmenter.Segment() {
		if segmenter.Type() == segment.Letter {
			out[porterstemmer.StemString(segmenter.Text())] = true
		}
	}
	if err := segmenter.Err(); err != nil {
		return nil, err
	}
	return out, nil
}
<commit_msg>docstore: cleanup Lua query engine<commit_after>package docstore

// package docstore

// TODO(tsileo): use   -tags 'prod' for building binary

import (
	"bytes"
	"fmt"
	"path/filepath"
	"time"

	"github.com/blevesearch/segment"
	log "github.com/inconshreveable/log15"
	"github.com/reiver/go-porterstemmer"
	"github.com/yuin/gopher-lua"

	luautil "a4.io/blobstash/pkg/apps/luautil"
)

type QueryMatcher interface {
	Match(map[string]interface{}) (bool, error)
	Close() error
}

type MatchAllEngine struct{}

func (mae *MatchAllEngine) Match(_ map[string]interface{}) (bool, error) {
	return true, nil
}

func (mae *MatchAllEngine) Close() error { return nil }

type LuaQueryEngine struct {
	storedQueries   map[string]*storedQuery // Stored query store
	storedQueryName string                  // Requested stored query name if any

	code  string
	query interface{} // Raw query
	q     lua.LValue

	matchFunc func(map[string]interface{}) (bool, error)
	L         *lua.LState // Lua state that will live the whole query

	logger log.Logger
}

func (lqe *LuaQueryEngine) Close() error {
	lqe.L.Close()
	return nil
}

func setGlobals(L *lua.LState) {
	// FIXME(tsileo): a `use_index(index_field, value)` and have the optimizer use it
	// TODO(tsileo): harvesine function for geoquery
	// TODO(tsileo): current time helper
	L.SetGlobal("porterstemmer", L.NewFunction(ltokenize))
	L.SetGlobal("porterstemmer_stem", L.NewFunction(stem))
}

func (docstore *DocStore) newLuaQueryEngine(query *query) (*LuaQueryEngine, error) {
	engine := &LuaQueryEngine{
		storedQueries:   docstore.storedQueries,
		query:           query.storedQueryArgs,
		code:            queryToScript(query),
		storedQueryName: query.storedQuery,
		L:               lua.NewState(),
		q:               lua.LNil,
		logger:          docstore.logger.New("submodule", "lua_query_engine"),
	}
	fmt.Printf("code=\n\n%s\n\n", engine.code)
	setGlobals(engine.L)
	engine.logger.Debug("init", "query", engine.query)
	// Parse the Lua query, which should be defined as a `function(doc) -> bool`, we parse it only once, then we got
	// a "Lua func" Go object which we can call repeatedly for each document.
	// XXX(tsileo): keep the function (along with the Lua context `L` for a few minutes) in a cache, so if a client is paginating
	// through results, it will reuse the func/Lua context. (Cache[hash(script)] = FuncWithContextReadyToCall)
	var ret *lua.LFunction
	if engine.query != nil {
		if engine.storedQueryName != "" {
			// XXX(tsileo): concerns: the script should be checked at startup because right now,
			// a user have to actually try a query before we can see if it's valud Lua.
			engine.logger.Debug("loading stored query", "name", engine.storedQueryName)
			squery, ok := engine.storedQueries[engine.storedQueryName]
			if !ok {
				return nil, fmt.Errorf("Unknown stored query name")
			}
			luautil.AddToPath(engine.L, filepath.Dir(squery.Main))
			engine.L.SetGlobal("query", luautil.InterfaceToLValue(engine.L, engine.query))
			if err := engine.L.DoFile(squery.Main); err != nil {
				panic(err)
			}
			ret = engine.L.Get(-1).(*lua.LFunction)
		}
	} else {
		// XXX(tsileo): queryToString converted the basic function to a script retunring a function
		if err := engine.L.DoString(engine.code); err != nil {
			return nil, err
		}
		ret = engine.L.Get(-1).(*lua.LFunction)
		fmt.Printf("extracted fun %v\n", ret)
	}
	if ret != nil {
		matchDoc := func(doc map[string]interface{}) (bool, error) {
			if err := engine.L.CallByParam(lua.P{
				Fn:      ret,
				NRet:    1,
				Protect: true,
			}, luautil.InterfaceToLValue(engine.L, doc)); err != nil {
				return false, err
			}
			ret := engine.L.Get(-1)
			engine.L.Pop(1)
			if ret == lua.LTrue {
				return true, nil
			}
			return false, nil
		}

		engine.matchFunc = matchDoc
	}
	return engine, nil
}

func (lqe *LuaQueryEngine) Match(doc map[string]interface{}) (bool, error) {
	start := time.Now()
	var out bool
	var err error

	if lqe.matchFunc == nil {
		return false, fmt.Errorf("missing matchFunc")
	}

	if out, err = lqe.matchFunc(doc); err != nil {
		return false, err
	}

	lqe.logger.Debug("match code ran", "duration", time.Since(start))
	return out, nil
}

func stem(L *lua.LState) int {
	in := L.ToString(1)
	L.Push(lua.LString(porterstemmer.StemString(in)))
	return 1
}

func ltokenize(L *lua.LState) int {
	in := L.ToString(1)
	out, err := tokenize([]byte(in))
	if err != nil {
		panic(err)
	}
	L.Push(luautil.InterfaceToLValue(L, out))
	return 1
}

func tokenize(data []byte) (map[string]interface{}, error) {
	out := map[string]interface{}{}
	segmenter := segment.NewWordSegmenter(bytes.NewReader(data))
	for segmenter.Segment() {
		if segmenter.Type() == segment.Letter {
			out[porterstemmer.StemString(segmenter.Text())] = true
		}
	}
	if err := segmenter.Err(); err != nil {
		return nil, err
	}
	return out, nil
}
<|endoftext|>


<commit_before>package gcp

import (
	"io/ioutil"
	"net/http"
	"path"
)

// GetComputeMetadata fetches the metadata from instance or project.
// e.g. curl -L http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip -H 'Metadata-Flavor:Google'
func GetComputeMetadata(key string) ([]byte, error) {
	const endpoint = "http://metadata.google.internal/computeMetadata/v1/"
	ep := path.Join(endpoint, key)

	req, err := http.NewRequest("GET", ep, nil)
	if err != nil {
		return nil, err
	}
	req.Header = map[string][]string{"Metadata-Flavor": {"Google"}}

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}

	bts, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	resp.Body.Close()
	return bts, nil
}
<commit_msg>pkg/gcp: do not use path.Join with scheme<commit_after>package gcp

import (
	"io/ioutil"
	"net/http"
	"strings"
)

// GetComputeMetadata fetches the metadata from instance or project.
// e.g. curl -L http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip -H 'Metadata-Flavor:Google'
func GetComputeMetadata(key string) ([]byte, error) {
	const endpoint = "http://metadata.google.internal/computeMetadata/v1/"
	if strings.HasPrefix(key, "/") {
		key = key[1:]
	}
	ep := endpoint + key

	req, err := http.NewRequest("GET", ep, nil)
	if err != nil {
		return nil, err
	}
	req.Header = map[string][]string{"Metadata-Flavor": {"Google"}}

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}

	bts, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	resp.Body.Close()
	return bts, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package osutil

import (
	"log"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"sync"

	"camlistore.org/pkg/buildinfo"
)

// HomeDir returns the path to the user's home directory.
// It returns the empty string if the value isn't known.
func HomeDir() string {
	failInTests()
	if runtime.GOOS == "windows" {
		return os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
	}
	return os.Getenv("HOME")
}

// Username returns the current user's username, as
// reported by the relevant environment variable.
func Username() string {
	if runtime.GOOS == "windows" {
		return os.Getenv("USERNAME")
	}
	return os.Getenv("USER")
}

var cacheDirOnce sync.Once

func CacheDir() string {
	cacheDirOnce.Do(makeCacheDir)
	return cacheDir()
}

func cacheDir() string {
	if d := os.Getenv("CAMLI_CACHE_DIR"); d != "" {
		return d
	}
	failInTests()
	switch runtime.GOOS {
	case "darwin":
		return filepath.Join(HomeDir(), "Library", "Caches", "Camlistore")
	case "windows":
		// Per http://technet.microsoft.com/en-us/library/cc749104(v=ws.10).aspx
		// these should both exist. But that page overwhelms me. Just try them
		// both. This seems to work.
		for _, ev := range []string{"TEMP", "TMP"} {
			if v := os.Getenv(ev); v != "" {
				return filepath.Join(v, "Camlistore")
			}
		}
		panic("No Windows TEMP or TMP environment variables found; please file a bug report.")
	}
	if xdg := os.Getenv("XDG_CACHE_HOME"); xdg != "" {
		return filepath.Join(xdg, "camlistore")
	}
	return filepath.Join(HomeDir(), ".cache", "camlistore")
}

func makeCacheDir() {
	err := os.MkdirAll(cacheDir(), 0700)
	if err != nil {
		log.Fatalf("Could not create cacheDir %v: %v", cacheDir(), err)
	}
}

func CamliVarDir() string {
	if d := os.Getenv("CAMLI_VAR_DIR"); d != "" {
		return d
	}
	failInTests()
	switch runtime.GOOS {
	case "windows":
		return filepath.Join(os.Getenv("APPDATA"), "Camlistore")
	case "darwin":
		return filepath.Join(HomeDir(), "Library", "Camlistore")
	}
	return filepath.Join(HomeDir(), "var", "camlistore")
}

func CamliBlobRoot() string {
	return filepath.Join(CamliVarDir(), "blobs")
}

func CamliConfigDir() string {
	if p := os.Getenv("CAMLI_CONFIG_DIR"); p != "" {
		return p
	}
	failInTests()
	if runtime.GOOS == "windows" {
		return filepath.Join(os.Getenv("APPDATA"), "Camlistore")
	}
	if xdg := os.Getenv("XDG_CONFIG_HOME"); xdg != "" {
		return filepath.Join(xdg, "camlistore")
	}
	return filepath.Join(HomeDir(), ".config", "camlistore")
}

func UserServerConfigPath() string {
	return filepath.Join(CamliConfigDir(), "server-config.json")
}

func UserClientConfigPath() string {
	return filepath.Join(CamliConfigDir(), "client-config.json")
}

// IdentitySecretRing returns the path to the default GPG
// secret keyring. It is overriden by the CAMLI_SECRET_RING
// environment variable.
func IdentitySecretRing() string {
	if e := os.Getenv("CAMLI_SECRET_RING"); e != "" {
		return e
	}
	return filepath.Join(CamliConfigDir(), "identity-secring.gpg")
}

// DefaultTLSCert returns the path to the default TLS certificate
// file that is used (creating if necessary) when TLS is specified
// without the cert file.
func DefaultTLSCert() string {
	return filepath.Join(CamliConfigDir(), "selfgen_pem.crt")
}

// DefaultTLSKey returns the path to the default TLS key
// file that is used (creating if necessary) when TLS is specified
// without the key file.
func DefaultTLSKey() string {
	return filepath.Join(CamliConfigDir(), "selfgen_pem.key")
}

// Find the correct absolute path corresponding to a relative path,
// searching the following sequence of directories:
// 1. Working Directory
// 2. CAMLI_CONFIG_DIR (deprecated, will complain if this is on env)
// 3. (windows only) APPDATA/camli
// 4. All directories in CAMLI_INCLUDE_PATH (standard PATH form for OS)
func FindCamliInclude(configFile string) (absPath string, err error) {
	// Try to open as absolute / relative to CWD
	_, err = os.Stat(configFile)
	if err == nil {
		return configFile, nil
	}
	if filepath.IsAbs(configFile) {
		// End of the line for absolute path
		return "", err
	}

	// Try the config dir
	configDir := CamliConfigDir()
	if _, err = os.Stat(filepath.Join(configDir, configFile)); err == nil {
		return filepath.Join(configDir, configFile), nil
	}

	// Finally, search CAMLI_INCLUDE_PATH
	p := os.Getenv("CAMLI_INCLUDE_PATH")
	for _, d := range strings.Split(p, string(filepath.ListSeparator)) {
		if _, err = os.Stat(filepath.Join(d, configFile)); err == nil {
			return filepath.Join(d, configFile), nil
		}
	}

	return "", os.ErrNotExist
}

// GoPackagePath returns the path to the provided Go package's
// source directory.
// pkg may be a path prefix without any *.go files.
// The error is os.ErrNotExist if GOPATH is unset or the directory
// doesn't exist in any GOPATH component.
func GoPackagePath(pkg string) (path string, err error) {
	gp := os.Getenv("GOPATH")
	if gp == "" {
		return path, os.ErrNotExist
	}
	for _, p := range filepath.SplitList(gp) {
		dir := filepath.Join(p, "src", filepath.FromSlash(pkg))
		fi, err := os.Stat(dir)
		if os.IsNotExist(err) {
			continue
		}
		if err != nil {
			return "", err
		}
		if !fi.IsDir() {
			continue
		}
		return dir, nil
	}
	return path, os.ErrNotExist
}

func failInTests() {
	strict, _ := strconv.ParseBool(os.Getenv("CAMLI_STRICT_PATHS")) // temporary; will soon be always true
	if strict && buildinfo.TestingLinked() {
		panic("Unexpected non-hermetic use of host configuration during testing. (alternatively: the 'testing' package got accidentally linked in)")
	}
}
<commit_msg>osutil: fail if tests access host paths<commit_after>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package osutil

import (
	"log"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"

	"camlistore.org/pkg/buildinfo"
)

// HomeDir returns the path to the user's home directory.
// It returns the empty string if the value isn't known.
func HomeDir() string {
	failInTests()
	if runtime.GOOS == "windows" {
		return os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
	}
	return os.Getenv("HOME")
}

// Username returns the current user's username, as
// reported by the relevant environment variable.
func Username() string {
	if runtime.GOOS == "windows" {
		return os.Getenv("USERNAME")
	}
	return os.Getenv("USER")
}

var cacheDirOnce sync.Once

func CacheDir() string {
	cacheDirOnce.Do(makeCacheDir)
	return cacheDir()
}

func cacheDir() string {
	if d := os.Getenv("CAMLI_CACHE_DIR"); d != "" {
		return d
	}
	failInTests()
	switch runtime.GOOS {
	case "darwin":
		return filepath.Join(HomeDir(), "Library", "Caches", "Camlistore")
	case "windows":
		// Per http://technet.microsoft.com/en-us/library/cc749104(v=ws.10).aspx
		// these should both exist. But that page overwhelms me. Just try them
		// both. This seems to work.
		for _, ev := range []string{"TEMP", "TMP"} {
			if v := os.Getenv(ev); v != "" {
				return filepath.Join(v, "Camlistore")
			}
		}
		panic("No Windows TEMP or TMP environment variables found; please file a bug report.")
	}
	if xdg := os.Getenv("XDG_CACHE_HOME"); xdg != "" {
		return filepath.Join(xdg, "camlistore")
	}
	return filepath.Join(HomeDir(), ".cache", "camlistore")
}

func makeCacheDir() {
	err := os.MkdirAll(cacheDir(), 0700)
	if err != nil {
		log.Fatalf("Could not create cacheDir %v: %v", cacheDir(), err)
	}
}

func CamliVarDir() string {
	if d := os.Getenv("CAMLI_VAR_DIR"); d != "" {
		return d
	}
	failInTests()
	switch runtime.GOOS {
	case "windows":
		return filepath.Join(os.Getenv("APPDATA"), "Camlistore")
	case "darwin":
		return filepath.Join(HomeDir(), "Library", "Camlistore")
	}
	return filepath.Join(HomeDir(), "var", "camlistore")
}

func CamliBlobRoot() string {
	return filepath.Join(CamliVarDir(), "blobs")
}

func CamliConfigDir() string {
	if p := os.Getenv("CAMLI_CONFIG_DIR"); p != "" {
		return p
	}
	failInTests()
	if runtime.GOOS == "windows" {
		return filepath.Join(os.Getenv("APPDATA"), "Camlistore")
	}
	if xdg := os.Getenv("XDG_CONFIG_HOME"); xdg != "" {
		return filepath.Join(xdg, "camlistore")
	}
	return filepath.Join(HomeDir(), ".config", "camlistore")
}

func UserServerConfigPath() string {
	return filepath.Join(CamliConfigDir(), "server-config.json")
}

func UserClientConfigPath() string {
	return filepath.Join(CamliConfigDir(), "client-config.json")
}

// IdentitySecretRing returns the path to the default GPG
// secret keyring. It is overriden by the CAMLI_SECRET_RING
// environment variable.
func IdentitySecretRing() string {
	if e := os.Getenv("CAMLI_SECRET_RING"); e != "" {
		return e
	}
	return filepath.Join(CamliConfigDir(), "identity-secring.gpg")
}

// DefaultTLSCert returns the path to the default TLS certificate
// file that is used (creating if necessary) when TLS is specified
// without the cert file.
func DefaultTLSCert() string {
	return filepath.Join(CamliConfigDir(), "selfgen_pem.crt")
}

// DefaultTLSKey returns the path to the default TLS key
// file that is used (creating if necessary) when TLS is specified
// without the key file.
func DefaultTLSKey() string {
	return filepath.Join(CamliConfigDir(), "selfgen_pem.key")
}

// Find the correct absolute path corresponding to a relative path,
// searching the following sequence of directories:
// 1. Working Directory
// 2. CAMLI_CONFIG_DIR (deprecated, will complain if this is on env)
// 3. (windows only) APPDATA/camli
// 4. All directories in CAMLI_INCLUDE_PATH (standard PATH form for OS)
func FindCamliInclude(configFile string) (absPath string, err error) {
	// Try to open as absolute / relative to CWD
	_, err = os.Stat(configFile)
	if err == nil {
		return configFile, nil
	}
	if filepath.IsAbs(configFile) {
		// End of the line for absolute path
		return "", err
	}

	// Try the config dir
	configDir := CamliConfigDir()
	if _, err = os.Stat(filepath.Join(configDir, configFile)); err == nil {
		return filepath.Join(configDir, configFile), nil
	}

	// Finally, search CAMLI_INCLUDE_PATH
	p := os.Getenv("CAMLI_INCLUDE_PATH")
	for _, d := range strings.Split(p, string(filepath.ListSeparator)) {
		if _, err = os.Stat(filepath.Join(d, configFile)); err == nil {
			return filepath.Join(d, configFile), nil
		}
	}

	return "", os.ErrNotExist
}

// GoPackagePath returns the path to the provided Go package's
// source directory.
// pkg may be a path prefix without any *.go files.
// The error is os.ErrNotExist if GOPATH is unset or the directory
// doesn't exist in any GOPATH component.
func GoPackagePath(pkg string) (path string, err error) {
	gp := os.Getenv("GOPATH")
	if gp == "" {
		return path, os.ErrNotExist
	}
	for _, p := range filepath.SplitList(gp) {
		dir := filepath.Join(p, "src", filepath.FromSlash(pkg))
		fi, err := os.Stat(dir)
		if os.IsNotExist(err) {
			continue
		}
		if err != nil {
			return "", err
		}
		if !fi.IsDir() {
			continue
		}
		return dir, nil
	}
	return path, os.ErrNotExist
}

func failInTests() {
	if buildinfo.TestingLinked() {
		panic("Unexpected non-hermetic use of host configuration during testing. (alternatively: the 'testing' package got accidentally linked in)")
	}
}
<|endoftext|>


<commit_before>package layouts

import (
	"errors"
	"reflect"
	"text/template"
)

// TODO remove once Go 1.2 is released.

// These are comparison functions available to text/template
// since Go 1.2.
// Copied from http://tip.golang.org/src/pkg/text/template/funcs.go

// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

var go12Funcs = template.FuncMap{
	"eq": eq, // ==
	"ge": ge, // >=
	"gt": gt, // >
	"le": le, // <=
	"lt": lt, // <
	"ne": ne, // !=
}

// Comparison.

// TODO: Perhaps allow comparison between signed and unsigned integers.

var (
	errBadComparisonType = errors.New("invalid type for comparison")
	errBadComparison     = errors.New("incompatible types for comparison")
	errNoComparison      = errors.New("missing argument for comparison")
)

type kind int

const (
	invalidKind kind = iota
	boolKind
	complexKind
	intKind
	floatKind
	integerKind
	stringKind
	uintKind
)

func basicKind(v reflect.Value) (kind, error) {
	switch v.Kind() {
	case reflect.Bool:
		return boolKind, nil
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		return intKind, nil
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
		return uintKind, nil
	case reflect.Float32, reflect.Float64:
		return floatKind, nil
	case reflect.Complex64, reflect.Complex128:
		return complexKind, nil
	case reflect.String:
		return stringKind, nil
	}
	return invalidKind, errBadComparisonType
}

// eq evaluates the comparison a == b || a == c || ...
func eq(arg1 interface{}, arg2 ...interface{}) (bool, error) {
	v1 := reflect.ValueOf(arg1)
	k1, err := basicKind(v1)
	if err != nil {
		return false, err
	}
	if len(arg2) == 0 {
		return false, errNoComparison
	}
	for _, arg := range arg2 {
		v2 := reflect.ValueOf(arg)
		k2, err := basicKind(v2)
		if err != nil {
			return false, err
		}
		if k1 != k2 {
			return false, errBadComparison
		}
		truth := false
		switch k1 {
		case boolKind:
			truth = v1.Bool() == v2.Bool()
		case complexKind:
			truth = v1.Complex() == v2.Complex()
		case floatKind:
			truth = v1.Float() == v2.Float()
		case intKind:
			truth = v1.Int() == v2.Int()
		case stringKind:
			truth = v1.String() == v2.String()
		case uintKind:
			truth = v1.Uint() == v2.Uint()
		default:
			panic("invalid kind")
		}
		if truth {
			return true, nil
		}
	}
	return false, nil
}

// ne evaluates the comparison a != b.
func ne(arg1, arg2 interface{}) (bool, error) {
	// != is the inverse of ==.
	equal, err := eq(arg1, arg2)
	return !equal, err
}

// lt evaluates the comparison a < b.
func lt(arg1, arg2 interface{}) (bool, error) {
	v1 := reflect.ValueOf(arg1)
	k1, err := basicKind(v1)
	if err != nil {
		return false, err
	}
	v2 := reflect.ValueOf(arg2)
	k2, err := basicKind(v2)
	if err != nil {
		return false, err
	}
	if k1 != k2 {
		return false, errBadComparison
	}
	truth := false
	switch k1 {
	case boolKind, complexKind:
		return false, errBadComparisonType
	case floatKind:
		truth = v1.Float() < v2.Float()
	case intKind:
		truth = v1.Int() < v2.Int()
	case stringKind:
		truth = v1.String() < v2.String()
	case uintKind:
		truth = v1.Uint() < v2.Uint()
	default:
		panic("invalid kind")
	}
	return truth, nil
}

// le evaluates the comparison <= b.
func le(arg1, arg2 interface{}) (bool, error) {
	// <= is < or ==.
	lessThan, err := lt(arg1, arg2)
	if lessThan || err != nil {
		return lessThan, err
	}
	return eq(arg1, arg2)
}

// gt evaluates the comparison a > b.
func gt(arg1, arg2 interface{}) (bool, error) {
	// > is the inverse of <=.
	lessOrEqual, err := le(arg1, arg2)
	if err != nil {
		return false, err
	}
	return !lessOrEqual, nil
}

// ge evaluates the comparison a >= b.
func ge(arg1, arg2 interface{}) (bool, error) {
	// >= is the inverse of <.
	lessThan, err := lt(arg1, arg2)
	if err != nil {
		return false, err
	}
	return !lessThan, nil
}
<commit_msg>Fix typo.<commit_after>package layouts

import (
	"errors"
	"reflect"
	"text/template"
)

// TODO remove once Go 1.2 is released.

// These are comparison functions available in text/template
// since Go 1.2.
// Copied from http://tip.golang.org/src/pkg/text/template/funcs.go

// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

var go12Funcs = template.FuncMap{
	"eq": eq, // ==
	"ge": ge, // >=
	"gt": gt, // >
	"le": le, // <=
	"lt": lt, // <
	"ne": ne, // !=
}

// Comparison.

// TODO: Perhaps allow comparison between signed and unsigned integers.

var (
	errBadComparisonType = errors.New("invalid type for comparison")
	errBadComparison     = errors.New("incompatible types for comparison")
	errNoComparison      = errors.New("missing argument for comparison")
)

type kind int

const (
	invalidKind kind = iota
	boolKind
	complexKind
	intKind
	floatKind
	integerKind
	stringKind
	uintKind
)

func basicKind(v reflect.Value) (kind, error) {
	switch v.Kind() {
	case reflect.Bool:
		return boolKind, nil
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		return intKind, nil
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
		return uintKind, nil
	case reflect.Float32, reflect.Float64:
		return floatKind, nil
	case reflect.Complex64, reflect.Complex128:
		return complexKind, nil
	case reflect.String:
		return stringKind, nil
	}
	return invalidKind, errBadComparisonType
}

// eq evaluates the comparison a == b || a == c || ...
func eq(arg1 interface{}, arg2 ...interface{}) (bool, error) {
	v1 := reflect.ValueOf(arg1)
	k1, err := basicKind(v1)
	if err != nil {
		return false, err
	}
	if len(arg2) == 0 {
		return false, errNoComparison
	}
	for _, arg := range arg2 {
		v2 := reflect.ValueOf(arg)
		k2, err := basicKind(v2)
		if err != nil {
			return false, err
		}
		if k1 != k2 {
			return false, errBadComparison
		}
		truth := false
		switch k1 {
		case boolKind:
			truth = v1.Bool() == v2.Bool()
		case complexKind:
			truth = v1.Complex() == v2.Complex()
		case floatKind:
			truth = v1.Float() == v2.Float()
		case intKind:
			truth = v1.Int() == v2.Int()
		case stringKind:
			truth = v1.String() == v2.String()
		case uintKind:
			truth = v1.Uint() == v2.Uint()
		default:
			panic("invalid kind")
		}
		if truth {
			return true, nil
		}
	}
	return false, nil
}

// ne evaluates the comparison a != b.
func ne(arg1, arg2 interface{}) (bool, error) {
	// != is the inverse of ==.
	equal, err := eq(arg1, arg2)
	return !equal, err
}

// lt evaluates the comparison a < b.
func lt(arg1, arg2 interface{}) (bool, error) {
	v1 := reflect.ValueOf(arg1)
	k1, err := basicKind(v1)
	if err != nil {
		return false, err
	}
	v2 := reflect.ValueOf(arg2)
	k2, err := basicKind(v2)
	if err != nil {
		return false, err
	}
	if k1 != k2 {
		return false, errBadComparison
	}
	truth := false
	switch k1 {
	case boolKind, complexKind:
		return false, errBadComparisonType
	case floatKind:
		truth = v1.Float() < v2.Float()
	case intKind:
		truth = v1.Int() < v2.Int()
	case stringKind:
		truth = v1.String() < v2.String()
	case uintKind:
		truth = v1.Uint() < v2.Uint()
	default:
		panic("invalid kind")
	}
	return truth, nil
}

// le evaluates the comparison <= b.
func le(arg1, arg2 interface{}) (bool, error) {
	// <= is < or ==.
	lessThan, err := lt(arg1, arg2)
	if lessThan || err != nil {
		return lessThan, err
	}
	return eq(arg1, arg2)
}

// gt evaluates the comparison a > b.
func gt(arg1, arg2 interface{}) (bool, error) {
	// > is the inverse of <=.
	lessOrEqual, err := le(arg1, arg2)
	if err != nil {
		return false, err
	}
	return !lessOrEqual, nil
}

// ge evaluates the comparison a >= b.
func ge(arg1, arg2 interface{}) (bool, error) {
	// >= is the inverse of <.
	lessThan, err := lt(arg1, arg2)
	if err != nil {
		return false, err
	}
	return !lessThan, nil
}
<|endoftext|>


<commit_before>package main

import (
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"time"
	"log"
)

func init() {
	expiration, err := time.ParseDuration(config.LoginAttempts.LogExpiration)
	if err != nil {
		log.Fatal("LoginAttemptsExpireAfter is not set: ", err)
	}
	LoginAttemptsExpireAfter = expiration
}

var LoginAttemptsExpireAfter time.Duration

type LoginAttempt struct {
	ID   bson.ObjectId `bson:"_id,omitempty"`
	IP   string        `bson:"ip"`
	User string        `bson:"user"`
	Time time.Time     `bson:"time"`
}

var LoginAttemptsIndex mgo.Index = mgo.Index{
	Key:         []string{"username", "email"},
	Unique:      true,
	ExpireAfter: LoginAttemptsExpireAfter,
}
<commit_msg>[refactoring] LoginAttempt index<commit_after>package main

import (
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"time"
	"log"
)

func init() {
	expiration, err := time.ParseDuration(config.LoginAttempts.LogExpiration)
	if err != nil {
		log.Fatal("LoginAttemptsExpireAfter is not set: ", err)
	}
	LoginAttemptsExpireAfter = expiration
}

var LoginAttemptsExpireAfter time.Duration

type LoginAttempt struct {
	ID   bson.ObjectId `bson:"_id,omitempty"`
	IP   string        `bson:"ip"`
	User string        `bson:"user"`
	Time time.Time     `bson:"time"`
}

var LoginAttemptsIndex mgo.Index = mgo.Index{
	Key:         []string{"ip", "user"},
	Unique:      true,
	ExpireAfter: LoginAttemptsExpireAfter,
}
<|endoftext|>