<commit_before>// Copyright 2017 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package downloader

import (
	"archive/tar"
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http/httptest"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"testing"

	"go.chromium.org/luci/common/data/caching/cache"
	"go.chromium.org/luci/common/data/stringset"
	"go.chromium.org/luci/common/isolated"
	"go.chromium.org/luci/common/isolatedclient"
	"go.chromium.org/luci/common/isolatedclient/isolatedfake"
	"go.chromium.org/luci/common/testing/testfs"

	. "github.com/smartystreets/goconvey/convey"
)

func TestNormalizePathSeparator(t *testing.T) {
	t.Parallel()

	Convey("Check path normalization", t, func() {
		Convey("posix path", func() {
			So(normalizePathSeparator("a/b"), ShouldEqual, filepath.Join("a", "b"))
		})

		Convey("windows path", func() {
			So(normalizePathSeparator(`a\b`), ShouldEqual, filepath.Join("a", "b"))
		})
	})
}

func TestDownloaderFetchIsolated(t *testing.T) {
	t.Parallel()
	ctx := context.Background()

	data1 := []byte("hello world!")
	data2 := []byte("wat")
	tardata := genTar(t)

	namespace := isolatedclient.DefaultNamespace
	h := isolated.GetHash(namespace)
	server := isolatedfake.New()
	data1hash := server.Inject(namespace, data1)
	data2hash := server.Inject(namespace, data2)
	tardatahash := server.Inject(namespace, tardata)
	tardataname := fmt.Sprintf("%s.tar", tardatahash)

	onePath := filepath.Join("foo", "one.txt")
	twoPath := filepath.Join("foo", "two.txt")
	posixPath := "posix/path"
	winPath := `win\path`
	isolated1 := isolated.New(h)
	isolated1.Files = map[string]isolated.File{
		onePath:     isolated.BasicFile(data1hash, 0664, int64(len(data1))),
		twoPath:     isolated.BasicFile(data2hash, 0764, int64(len(data2))),
		posixPath:   isolated.BasicFile(data1hash, 0664, int64(len(data1))),
		winPath:     isolated.BasicFile(data2hash, 0664, int64(len(data2))),
		tardataname: isolated.TarFile(tardatahash, int64(len(tardata))),
	}
	isolated1bytes, _ := json.Marshal(&isolated1)
	isolated1hash := server.Inject(namespace, isolated1bytes)

	lolPath := filepath.Join("bar", "lol.txt")
	oloPath := filepath.Join("foo", "boz", "olo.txt")
	isolated2 := isolated.New(h)
	isolated2.Files = map[string]isolated.File{
		lolPath: isolated.BasicFile(data1hash, 0664, int64(len(data1))),
		oloPath: isolated.BasicFile(data2hash, 0664, int64(len(data2))),
	}
	isolatedFiles := stringset.NewFromSlice([]string{
		tardataname,
		onePath,
		twoPath,
		normalizePathSeparator(posixPath),
		normalizePathSeparator(winPath),
		lolPath,
		oloPath,
		// In tardata
		"file1",
		"file2",
		filepath.Join("tar", "posix", "path"),
		filepath.Join("tar", "win", "path"),
	}...)
	blahPath := "blah.txt"

	// Symlinks not supported on Windows.
	if runtime.GOOS != "windows" {
		isolated2.Files[blahPath] = isolated.SymLink(oloPath)
		isolatedFiles.Add(blahPath)
	}
	isolated2.Includes = isolated.HexDigests{isolated1hash}
	isolated2bytes, _ := json.Marshal(&isolated2)
	isolated2hash := server.Inject(namespace, isolated2bytes)

	ts := httptest.NewServer(server)
	defer ts.Close()
	client := isolatedclient.NewClient(ts.URL, isolatedclient.WithNamespace(namespace))

	Convey(`A downloader should be able to download the isolated.`, t, func() {
		tmpDir, err := ioutil.TempDir("", "isolated")
		So(err, ShouldBeNil)
		defer func() {
			So(os.RemoveAll(tmpDir), ShouldBeNil)
		}()

		mu := sync.Mutex{}
		var files []string
		d := New(ctx, client, isolated2hash, tmpDir, &Options{
			FileCallback: func(name string, _ *isolated.File) {
				mu.Lock()
				files = append(files, name)
				mu.Unlock()
			},
		})
		So(d.Wait(), ShouldBeNil)
		So(stringset.NewFromSlice(files...), ShouldResemble, isolatedFiles)

		b, err := ioutil.ReadFile(filepath.Join(tmpDir, onePath))
		So(err, ShouldBeNil)
		So(b, ShouldResemble, data1)

		fi, err := os.Stat(filepath.Join(tmpDir, onePath))
		So(err, ShouldBeNil)
		// to ignore effect of umask, only check executable bit.
		So(fi.Mode()&0111, ShouldEqual, 0)

		b, err = ioutil.ReadFile(filepath.Join(tmpDir, twoPath))
		So(err, ShouldBeNil)
		So(b, ShouldResemble, data2)

		fi, err = os.Stat(filepath.Join(tmpDir, twoPath))
		So(err, ShouldBeNil)
		// to ignore effect of umask, only check executable bit.
		So(fi.Mode()&0011, ShouldEqual, 0)

		b, err = ioutil.ReadFile(filepath.Join(tmpDir, lolPath))
		So(err, ShouldBeNil)
		So(b, ShouldResemble, data1)

		b, err = ioutil.ReadFile(filepath.Join(tmpDir, oloPath))
		So(err, ShouldBeNil)
		So(b, ShouldResemble, data2)

		// Check files in tar archive
		_, err = os.Stat(filepath.Join(tmpDir, "file1"))
		So(err, ShouldBeNil)

		_, err = os.Stat(filepath.Join(tmpDir, "file2"))
		So(err, ShouldBeNil)

		_, err = os.Stat(filepath.Join(tmpDir, "tar", "posix", "path"))
		So(err, ShouldBeNil)

		_, err = os.Stat(filepath.Join(tmpDir, "tar", "win", "path"))
		So(err, ShouldBeNil)

		_, err = ioutil.ReadFile(filepath.Join(tmpDir, tardataname))
		So(os.IsNotExist(err), ShouldBeTrue)

		if runtime.GOOS != "windows" {
			l, err := os.Readlink(filepath.Join(tmpDir, blahPath))
			So(err, ShouldBeNil)
			So(l, ShouldResemble, oloPath)
		}
	})
}

// genTar returns a valid tar file.
func genTar(t *testing.T) []byte {
	b := bytes.Buffer{}
	tw := tar.NewWriter(&b)
	d := []byte("hello file1")
	if err := tw.WriteHeader(&tar.Header{Name: "file1", Mode: 0644, Typeflag: tar.TypeReg, Size: int64(len(d))}); err != nil {
		t.Fatal(err)
	}
	if _, err := tw.Write(d); err != nil {
		t.Fatal(err)
	}
	d = []byte(strings.Repeat("hello file2", 100))
	if err := tw.WriteHeader(&tar.Header{Name: "file2", Mode: 0644, Typeflag: tar.TypeReg, Size: int64(len(d))}); err != nil {
		t.Fatal(err)
	}
	if _, err := tw.Write(d); err != nil {
		t.Fatal(err)
	}
	d = []byte("posixpath")
	if err := tw.WriteHeader(&tar.Header{Name: "tar/posix/path", Mode: 0644, Typeflag: tar.TypeReg, Size: int64(len(d))}); err != nil {
		t.Fatal(err)
	}
	if _, err := tw.Write(d); err != nil {
		t.Fatal(err)
	}
	d = []byte("winpath")
	if err := tw.WriteHeader(&tar.Header{Name: `tar\win\path`, Mode: 0644, Typeflag: tar.TypeReg, Size: int64(len(d))}); err != nil {
		t.Fatal(err)
	}
	if _, err := tw.Write(d); err != nil {
		t.Fatal(err)
	}

	if err := tw.Close(); err != nil {
		t.Fatal(err)
	}
	return b.Bytes()
}

func TestDownloaderWithCache(t *testing.T) {
	t.Parallel()
	Convey(`TestDownloaderWithCache`, t, testfs.MustWithTempDir(t, "", func(tmpDir string) {
		ctx := context.Background()

		miss := []byte("cache miss")
		hit := []byte("cache hit")
		namespace := isolatedclient.DefaultNamespace
		h := isolated.GetHash(namespace)
		server := isolatedfake.New()
		misshash := server.Inject(namespace, miss)
		hithash := isolated.HashBytes(isolated.GetHash(namespace), hit)

		missPath := filepath.Join("foo", "miss.txt")
		hitPath := filepath.Join("foo", "hit.txt")
		isolated1 := isolated.New(h)

		isolated1.Files = map[string]isolated.File{
			missPath: isolated.BasicFile(misshash, 0664, int64(len(miss))),
			hitPath:  isolated.BasicFile(hithash, 0664, int64(len(hit))),
		}
		isolated1bytes, _ := json.Marshal(&isolated1)
		isolated1hash := server.Inject(namespace, isolated1bytes)

		isolatedFiles := stringset.NewFromSlice([]string{
			missPath,
			hitPath,
		}...)

		ts := httptest.NewServer(server)
		defer ts.Close()
		client := isolatedclient.NewClient(ts.URL, isolatedclient.WithNamespace(namespace))

		mu := sync.Mutex{}
		var files []string
		policy := cache.Policies{
			MaxSize:  1024,
			MaxItems: 1024,
		}
		var cacheObj cache.Cache
		Convey("memcache", func() {
			cacheObj = cache.NewMemory(policy, namespace)
		})

		Convey("diskcache", func() {
			cacheDir := filepath.Join(tmpDir, "cache")
			So(os.MkdirAll(cacheDir, os.ModePerm), ShouldBeNil)
			var err error
			cacheObj, err = cache.NewDisk(policy, cacheDir, namespace)
			So(err, ShouldBeNil)
		})

		So(cacheObj.Add(hithash, bytes.NewReader(hit)), ShouldBeNil)

		d := New(ctx, client, isolated1hash, tmpDir, &Options{
			FileCallback: func(name string, _ *isolated.File) {
				mu.Lock()
				files = append(files, name)
				mu.Unlock()
			},
			Cache: cacheObj,
		})
		So(d.Wait(), ShouldBeNil)
		So(stringset.NewFromSlice(files...), ShouldResemble, isolatedFiles)

		So(cacheObj.Touch(misshash), ShouldBeTrue)
	}))
}

func TestFetchAndMap(t *testing.T) {
	t.Parallel()
	Convey(`TestFetchAndMap`, t, func() {
		ctx := context.Background()
		tmpDir, err := ioutil.TempDir("", "isolated")
		So(err, ShouldBeNil)
		defer func() {
			So(os.RemoveAll(tmpDir), ShouldBeNil)
		}()

		data1 := []byte("hello world!")

		namespace := isolatedclient.DefaultNamespace
		h := isolated.GetHash(namespace)
		server := isolatedfake.New()
		data1hash := server.Inject(namespace, data1)

		onePath := filepath.Join("foo", "one.txt")
		onePathFile := isolated.BasicFile(data1hash, 0664, int64(len(data1)))
		isolated1 := isolated.New(h)
		isolated1.Files = map[string]isolated.File{
			onePath: onePathFile,
		}
		isolated1bytes, _ := json.Marshal(&isolated1)
		isolated1hash := server.Inject(namespace, isolated1bytes)

		ts := httptest.NewServer(server)
		defer ts.Close()
		client := isolatedclient.NewClient(ts.URL, isolatedclient.WithNamespace(namespace))

		policy := cache.Policies{
			MaxSize:  1024,
			MaxItems: 1024,
		}
		memcache := cache.NewMemory(policy, namespace)

		isomap, stats, err := FetchAndMap(ctx, isolated1hash, client, memcache, tmpDir)
		So(err, ShouldBeNil)
		So(isomap, ShouldResemble, &isolated.Isolated{
			Algo:    "sha-1",
			Version: "1.4",
		})

		So(stats.Duration, ShouldBeGreaterThan, 0)
		So(stats.ItemsCold, ShouldResemble, []byte{120, 156, 226, 1, 4, 0, 0, 255, 255, 0, 13, 0, 13})
		So(stats.ItemsHot, ShouldResemble, []byte(nil))

		buf, err := ioutil.ReadFile(filepath.Join(tmpDir, onePath))
		So(err, ShouldBeNil)
		So(buf, ShouldResemble, data1)
	})
}
<commit_msg>downloader: add tar in TestFetchAndMap<commit_after>// Copyright 2017 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package downloader

import (
	"archive/tar"
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http/httptest"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"testing"

	"go.chromium.org/luci/common/data/caching/cache"
	"go.chromium.org/luci/common/data/stringset"
	"go.chromium.org/luci/common/isolated"
	"go.chromium.org/luci/common/isolatedclient"
	"go.chromium.org/luci/common/isolatedclient/isolatedfake"
	"go.chromium.org/luci/common/testing/testfs"

	. "github.com/smartystreets/goconvey/convey"
)

func TestNormalizePathSeparator(t *testing.T) {
	t.Parallel()

	Convey("Check path normalization", t, func() {
		Convey("posix path", func() {
			So(normalizePathSeparator("a/b"), ShouldEqual, filepath.Join("a", "b"))
		})

		Convey("windows path", func() {
			So(normalizePathSeparator(`a\b`), ShouldEqual, filepath.Join("a", "b"))
		})
	})
}

func TestDownloaderFetchIsolated(t *testing.T) {
	t.Parallel()
	ctx := context.Background()

	data1 := []byte("hello world!")
	data2 := []byte("wat")
	tardata := genTar(t)

	namespace := isolatedclient.DefaultNamespace
	h := isolated.GetHash(namespace)
	server := isolatedfake.New()
	data1hash := server.Inject(namespace, data1)
	data2hash := server.Inject(namespace, data2)
	tardatahash := server.Inject(namespace, tardata)
	tardataname := fmt.Sprintf("%s.tar", tardatahash)

	onePath := filepath.Join("foo", "one.txt")
	twoPath := filepath.Join("foo", "two.txt")
	posixPath := "posix/path"
	winPath := `win\path`
	isolated1 := isolated.New(h)
	isolated1.Files = map[string]isolated.File{
		onePath:     isolated.BasicFile(data1hash, 0664, int64(len(data1))),
		twoPath:     isolated.BasicFile(data2hash, 0764, int64(len(data2))),
		posixPath:   isolated.BasicFile(data1hash, 0664, int64(len(data1))),
		winPath:     isolated.BasicFile(data2hash, 0664, int64(len(data2))),
		tardataname: isolated.TarFile(tardatahash, int64(len(tardata))),
	}
	isolated1bytes, _ := json.Marshal(&isolated1)
	isolated1hash := server.Inject(namespace, isolated1bytes)

	lolPath := filepath.Join("bar", "lol.txt")
	oloPath := filepath.Join("foo", "boz", "olo.txt")
	isolated2 := isolated.New(h)
	isolated2.Files = map[string]isolated.File{
		lolPath: isolated.BasicFile(data1hash, 0664, int64(len(data1))),
		oloPath: isolated.BasicFile(data2hash, 0664, int64(len(data2))),
	}
	isolatedFiles := stringset.NewFromSlice([]string{
		tardataname,
		onePath,
		twoPath,
		normalizePathSeparator(posixPath),
		normalizePathSeparator(winPath),
		lolPath,
		oloPath,
		// In tardata
		"file1",
		"file2",
		filepath.Join("tar", "posix", "path"),
		filepath.Join("tar", "win", "path"),
	}...)
	blahPath := "blah.txt"

	// Symlinks not supported on Windows.
	if runtime.GOOS != "windows" {
		isolated2.Files[blahPath] = isolated.SymLink(oloPath)
		isolatedFiles.Add(blahPath)
	}
	isolated2.Includes = isolated.HexDigests{isolated1hash}
	isolated2bytes, _ := json.Marshal(&isolated2)
	isolated2hash := server.Inject(namespace, isolated2bytes)

	ts := httptest.NewServer(server)
	defer ts.Close()
	client := isolatedclient.NewClient(ts.URL, isolatedclient.WithNamespace(namespace))

	Convey(`A downloader should be able to download the isolated.`, t, func() {
		tmpDir, err := ioutil.TempDir("", "isolated")
		So(err, ShouldBeNil)
		defer func() {
			So(os.RemoveAll(tmpDir), ShouldBeNil)
		}()

		mu := sync.Mutex{}
		var files []string
		d := New(ctx, client, isolated2hash, tmpDir, &Options{
			FileCallback: func(name string, _ *isolated.File) {
				mu.Lock()
				files = append(files, name)
				mu.Unlock()
			},
		})
		So(d.Wait(), ShouldBeNil)
		So(stringset.NewFromSlice(files...), ShouldResemble, isolatedFiles)

		b, err := ioutil.ReadFile(filepath.Join(tmpDir, onePath))
		So(err, ShouldBeNil)
		So(b, ShouldResemble, data1)

		fi, err := os.Stat(filepath.Join(tmpDir, onePath))
		So(err, ShouldBeNil)
		// to ignore effect of umask, only check executable bit.
		So(fi.Mode()&0111, ShouldEqual, 0)

		b, err = ioutil.ReadFile(filepath.Join(tmpDir, twoPath))
		So(err, ShouldBeNil)
		So(b, ShouldResemble, data2)

		fi, err = os.Stat(filepath.Join(tmpDir, twoPath))
		So(err, ShouldBeNil)
		// to ignore effect of umask, only check executable bit.
		So(fi.Mode()&0011, ShouldEqual, 0)

		b, err = ioutil.ReadFile(filepath.Join(tmpDir, lolPath))
		So(err, ShouldBeNil)
		So(b, ShouldResemble, data1)

		b, err = ioutil.ReadFile(filepath.Join(tmpDir, oloPath))
		So(err, ShouldBeNil)
		So(b, ShouldResemble, data2)

		// Check files in tar archive
		_, err = os.Stat(filepath.Join(tmpDir, "file1"))
		So(err, ShouldBeNil)

		_, err = os.Stat(filepath.Join(tmpDir, "file2"))
		So(err, ShouldBeNil)

		_, err = os.Stat(filepath.Join(tmpDir, "tar", "posix", "path"))
		So(err, ShouldBeNil)

		_, err = os.Stat(filepath.Join(tmpDir, "tar", "win", "path"))
		So(err, ShouldBeNil)

		_, err = ioutil.ReadFile(filepath.Join(tmpDir, tardataname))
		So(os.IsNotExist(err), ShouldBeTrue)

		if runtime.GOOS != "windows" {
			l, err := os.Readlink(filepath.Join(tmpDir, blahPath))
			So(err, ShouldBeNil)
			So(l, ShouldResemble, oloPath)
		}
	})
}

// genTar returns a valid tar file.
func genTar(t *testing.T) []byte {
	b := bytes.Buffer{}
	tw := tar.NewWriter(&b)
	d := []byte("hello file1")
	if err := tw.WriteHeader(&tar.Header{Name: "file1", Mode: 0644, Typeflag: tar.TypeReg, Size: int64(len(d))}); err != nil {
		t.Fatal(err)
	}
	if _, err := tw.Write(d); err != nil {
		t.Fatal(err)
	}
	d = []byte(strings.Repeat("hello file2", 100))
	if err := tw.WriteHeader(&tar.Header{Name: "file2", Mode: 0644, Typeflag: tar.TypeReg, Size: int64(len(d))}); err != nil {
		t.Fatal(err)
	}
	if _, err := tw.Write(d); err != nil {
		t.Fatal(err)
	}
	d = []byte("posixpath")
	if err := tw.WriteHeader(&tar.Header{Name: "tar/posix/path", Mode: 0644, Typeflag: tar.TypeReg, Size: int64(len(d))}); err != nil {
		t.Fatal(err)
	}
	if _, err := tw.Write(d); err != nil {
		t.Fatal(err)
	}
	d = []byte("winpath")
	if err := tw.WriteHeader(&tar.Header{Name: `tar\win\path`, Mode: 0644, Typeflag: tar.TypeReg, Size: int64(len(d))}); err != nil {
		t.Fatal(err)
	}
	if _, err := tw.Write(d); err != nil {
		t.Fatal(err)
	}

	if err := tw.Close(); err != nil {
		t.Fatal(err)
	}
	return b.Bytes()
}

func TestDownloaderWithCache(t *testing.T) {
	t.Parallel()
	Convey(`TestDownloaderWithCache`, t, testfs.MustWithTempDir(t, "", func(tmpDir string) {
		ctx := context.Background()

		miss := []byte("cache miss")
		hit := []byte("cache hit")
		namespace := isolatedclient.DefaultNamespace
		h := isolated.GetHash(namespace)
		server := isolatedfake.New()
		misshash := server.Inject(namespace, miss)
		hithash := isolated.HashBytes(isolated.GetHash(namespace), hit)

		missPath := filepath.Join("foo", "miss.txt")
		hitPath := filepath.Join("foo", "hit.txt")
		isolated1 := isolated.New(h)

		isolated1.Files = map[string]isolated.File{
			missPath: isolated.BasicFile(misshash, 0664, int64(len(miss))),
			hitPath:  isolated.BasicFile(hithash, 0664, int64(len(hit))),
		}
		isolated1bytes, _ := json.Marshal(&isolated1)
		isolated1hash := server.Inject(namespace, isolated1bytes)

		isolatedFiles := stringset.NewFromSlice([]string{
			missPath,
			hitPath,
		}...)

		ts := httptest.NewServer(server)
		defer ts.Close()
		client := isolatedclient.NewClient(ts.URL, isolatedclient.WithNamespace(namespace))

		mu := sync.Mutex{}
		var files []string
		policy := cache.Policies{
			MaxSize:  1024,
			MaxItems: 1024,
		}
		var cacheObj cache.Cache
		Convey("memcache", func() {
			cacheObj = cache.NewMemory(policy, namespace)
		})

		Convey("diskcache", func() {
			cacheDir := filepath.Join(tmpDir, "cache")
			So(os.MkdirAll(cacheDir, os.ModePerm), ShouldBeNil)
			var err error
			cacheObj, err = cache.NewDisk(policy, cacheDir, namespace)
			So(err, ShouldBeNil)
		})

		So(cacheObj.Add(hithash, bytes.NewReader(hit)), ShouldBeNil)

		d := New(ctx, client, isolated1hash, tmpDir, &Options{
			FileCallback: func(name string, _ *isolated.File) {
				mu.Lock()
				files = append(files, name)
				mu.Unlock()
			},
			Cache: cacheObj,
		})
		So(d.Wait(), ShouldBeNil)
		So(stringset.NewFromSlice(files...), ShouldResemble, isolatedFiles)

		So(cacheObj.Touch(misshash), ShouldBeTrue)
	}))
}

func TestFetchAndMap(t *testing.T) {
	t.Parallel()
	Convey(`TestFetchAndMap`, t, func() {
		ctx := context.Background()
		tmpDir, err := ioutil.TempDir("", "isolated")
		So(err, ShouldBeNil)
		defer func() {
			So(os.RemoveAll(tmpDir), ShouldBeNil)
		}()

		data1 := []byte("hello world!")

		namespace := isolatedclient.DefaultNamespace
		h := isolated.GetHash(namespace)
		server := isolatedfake.New()
		data1hash := server.Inject(namespace, data1)

		onePath := filepath.Join("foo", "one.txt")
		onePathFile := isolated.BasicFile(data1hash, 0664, int64(len(data1)))
		isolated1 := isolated.New(h)

		tardata := genTar(t)
		tarhash := server.Inject(namespace, tardata)

		isolated1.Files = map[string]isolated.File{
			onePath:   onePathFile,
			"tar.tar": isolated.TarFile(tarhash, int64(len(tardata))),
		}
		isolated1bytes, _ := json.Marshal(&isolated1)
		isolated1hash := server.Inject(namespace, isolated1bytes)

		ts := httptest.NewServer(server)
		defer ts.Close()
		client := isolatedclient.NewClient(ts.URL, isolatedclient.WithNamespace(namespace))

		policy := cache.Policies{
			MaxSize:  1024 * 1024,
			MaxItems: 1024,
		}
		memcache := cache.NewMemory(policy, namespace)

		isomap, stats, err := FetchAndMap(ctx, isolated1hash, client, memcache, tmpDir)
		So(err, ShouldBeNil)
		So(isomap, ShouldResemble, &isolated.Isolated{
			Algo:    "sha-1",
			Version: "1.4",
		})

		So(stats.Duration, ShouldBeGreaterThan, 0)
		So(stats.ItemsCold, ShouldResemble, []byte{120, 156, 226, 1, 4, 0, 0, 255, 255, 0, 13, 0, 13})
		So(stats.ItemsHot, ShouldResemble, []byte(nil))

		buf, err := ioutil.ReadFile(filepath.Join(tmpDir, onePath))
		So(err, ShouldBeNil)
		So(buf, ShouldResemble, data1)
	})
}
<|endoftext|>


<commit_before>package route

import (
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	. "github.com/SimonRichardson/wishful/useful"
	. "github.com/SimonRichardson/wishful/wishful"
)

var (
	EitherPromise EitherT = NewEitherT(Promise{})
)

func from(x Either) EitherT {
	return EitherPromise.From(Promise{}.Of(x))
}

func parseJson(raw []byte, val AnyVal) Either {
	if err := json.Unmarshal(raw, &val); err != nil {
		return NewLeft(err)
	}
	return NewRight(val)
}

func parseQuery(raw string) Either {
	u, err := url.Parse(raw)
	if err != nil {
		return NewLeft(err)
	}
	return NewRight(u.Query())
}

func JsonParse(val AnyVal) func(raw []byte) EitherT {
	return func(raw []byte) EitherT {
		return from(handleException(func(x AnyVal) Either {
			return parseJson(x.([]byte), val)
		})(raw))
	}
}

func QueryParse(raw string) EitherT {
	return from(handleException(func(x AnyVal) Either {
		return parseQuery(x.(string))
	})(raw))
}

func ReadBody(req *http.Request) EitherT {
	c := req.Header.Get("content-length")
	length, err := strconv.ParseInt(c, 10, 64)
	if err != nil {
		return from(NewLeft(err))
	}

	reader := io.LimitReader(req.Body, length)
	b, e := ioutil.ReadAll(reader)

	if e != nil {
		return from(NewLeft(e))
	}
	if len(b) > int(length) {
		err := errors.New("http: Body too large")
		return from(NewLeft(err))
	}

	return from(NewRight(b))
}

func Json(val AnyVal, req *http.Request) EitherT {
	return ReadBody(req).Chain(func(x AnyVal) Monad {
		return JsonParse(val)(x.([]byte))
	}).(EitherT)
}

func Query(req *http.Request) EitherT {
	return ReadBody(req).Chain(func(x AnyVal) Monad {
		return QueryParse(x.(string))
	}).(EitherT)
}

func Raw(req *http.Request) EitherT {
	return ReadBody(req)
}
<commit_msg>Using strconv.Atoi, make it easier to convert<commit_after>package route

import (
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	. "github.com/SimonRichardson/wishful/useful"
	. "github.com/SimonRichardson/wishful/wishful"
)

var (
	EitherPromise EitherT = NewEitherT(Promise{})
)

func from(x Either) EitherT {
	return EitherPromise.From(Promise{}.Of(x))
}

func parseJson(raw []byte, val AnyVal) Either {
	if err := json.Unmarshal(raw, &val); err != nil {
		return NewLeft(err)
	}
	return NewRight(val)
}

func parseQuery(raw string) Either {
	u, err := url.ParseQuery(raw)
	if err != nil {
		return NewLeft(err)
	}
	return NewRight(u)
}

func JsonParse(val AnyVal) func(raw []byte) EitherT {
	return func(raw []byte) EitherT {
		return from(handleException(func(x AnyVal) Either {
			return parseJson(x.([]byte), val)
		})(raw))
	}
}

func QueryParse(raw string) EitherT {
	return from(handleException(func(x AnyVal) Either {
		return parseQuery(x.(string))
	})(raw))
}

func ReadBody(req *http.Request) EitherT {
	c := req.Header.Get("content-length")
	length, err := strconv.Atoi(c)
	if err != nil {
		return from(NewLeft(err))
	}

	reader := io.LimitReader(req.Body, int64(length))
	b, e := ioutil.ReadAll(reader)

	if e != nil {
		return from(NewLeft(e))
	}
	if len(b) > length {
		err := errors.New("http: Body too large")
		return from(NewLeft(err))
	}

	return from(NewRight(b))
}

func Json(val AnyVal, req *http.Request) EitherT {
	return ReadBody(req).Chain(func(x AnyVal) Monad {
		return JsonParse(val)(x.([]byte))
	}).(EitherT)
}

func Query(req *http.Request) EitherT {
	return ReadBody(req).Chain(func(x AnyVal) Monad {
		return QueryParse(x.(string))
	}).(EitherT)
}

func Raw(req *http.Request) EitherT {
	return ReadBody(req)
}
<|endoftext|>


<commit_before>// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Spencer Kimball (spencer.kimball@gmail.com)

package rpc

import (
	"fmt"
	"io"
	"net"
	"net/http"
	"net/rpc"
	"reflect"
	"strings"
	"sync"

	"github.com/cockroachdb/cockroach/rpc/codec"
	"github.com/cockroachdb/cockroach/security"
	"github.com/cockroachdb/cockroach/util"
	"github.com/cockroachdb/cockroach/util/log"
	"github.com/gogo/protobuf/proto"
)

type method struct {
	handler func(proto.Message, func(proto.Message, error))
	reqType reflect.Type
}

type serverResponse struct {
	req   *rpc.Request
	reply proto.Message
	err   error
}

type syncAdapter func(proto.Message) (proto.Message, error)

func (s syncAdapter) exec(args proto.Message, callback func(proto.Message, error)) {
	go func() {
		callback(s(args))
	}()
}

// Server is a Cockroach-specific RPC server. By default it handles a simple
// heartbeat protocol to measure link health. It also supports close callbacks.
//
// TODO(spencer): heartbeat protocol should also measure link latency.
type Server struct {
	listener net.Listener // Server listener

	activeConns map[net.Conn]struct{}
	handler     http.Handler

	context *Context

	mu             sync.RWMutex          // Mutex protects the fields below
	addr           net.Addr              // Server address; may change if picking unused port
	closed         bool                  // Set upon invocation of Close()
	closeCallbacks []func(conn net.Conn) // Slice of callbacks to invoke on conn close
	methods        map[string]method
}

// NewServer creates a new instance of Server.
func NewServer(addr net.Addr, context *Context) *Server {
	s := &Server{
		context: context,
		addr:    addr,
		methods: map[string]method{},
	}
	heartbeat := &HeartbeatService{
		clock:              context.localClock,
		remoteClockMonitor: context.RemoteClocks,
	}
	if err := heartbeat.Register(s); err != nil {
		log.Fatalf("unable to register heartbeat service with RPC server: %s", err)
	}
	return s
}

// Register a new method handler. `name` is a qualified name of the
// form "Service.Name". `handler` is a function that takes an
// argument of the same type as `reqPrototype`. Both the argument and
// return value of 'handler' should be a pointer to a protocol message
// type. The handler function will be executed in a new goroutine.
func (s *Server) Register(name string, handler func(proto.Message) (proto.Message, error),
	reqPrototype proto.Message) error {
	return s.RegisterAsync(name, syncAdapter(handler).exec, reqPrototype)
}

// RegisterAsync registers an asynchronous method handler. Instead of
// returning a (proto.Message, error) tuple, an asynchronous handler
// receives a callback which it must execute when it is complete. Note
// that async handlers are started in the RPC server's goroutine and
// must not block (i.e. they must start a goroutine or write to a
// channel promptly). However, the fact that they are started in the
// RPC server's goroutine guarantees that the order of requests as
// they were read from the connection is preserved.
func (s *Server) RegisterAsync(name string, handler func(proto.Message, func(proto.Message, error)),
	reqPrototype proto.Message) error {
	s.mu.Lock()
	defer s.mu.Unlock()

	if _, ok := s.methods[name]; ok {
		return util.Errorf("method %s already registered", name)
	}
	reqType := reflect.TypeOf(reqPrototype)
	if reqType.Kind() != reflect.Ptr {
		// net/rpc supports non-pointer requests, but we always use pointers
		// and things are a little simpler this way.
		return util.Errorf("request type not a pointer")
	}
	s.methods[name] = method{
		handler: handler,
		reqType: reqType,
	}
	return nil
}

// AddCloseCallback adds a callback to the closeCallbacks slice to
// be invoked when a connection is closed.
func (s *Server) AddCloseCallback(cb func(conn net.Conn)) {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.closeCallbacks = append(s.closeCallbacks, cb)
}

// Can connect to RPC service using HTTP CONNECT to rpcPath.
var connected = "200 Connected to Go RPC"

// ServeHTTP implements an http.Handler that answers RPC requests.
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path != rpc.DefaultRPCPath {
		if s.handler != nil {
			s.handler.ServeHTTP(w, r)
			return
		}
		http.NotFound(w, r)
		return
	}

	// Note: this code was adapted from net/rpc.Server.ServeHTTP.
	if r.Method != "CONNECT" {
		http.Error(w, "405 must CONNECT", http.StatusMethodNotAllowed)
		return
	}

	// Construct an authentication hook for this security mode and TLS state.
	authHook, err := security.AuthenticationHook(s.context.Insecure, r.TLS)
	if err != nil {
		http.Error(w, err.Error(), http.StatusUnauthorized)
		return
	}

	conn, _, err := w.(http.Hijacker).Hijack()
	if err != nil {
		log.Infof("rpc hijacking %s: %s", r.RemoteAddr, err)
		return
	}
	security.LogTLSState("RPC", r.TLS)
	io.WriteString(conn, "HTTP/1.0 "+connected+"\n\n")

	codec := codec.NewServerCodec(conn, authHook)
	responses := make(chan serverResponse)
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		s.sendResponses(codec, responses)
		wg.Done()
	}()
	s.readRequests(codec, responses)
	wg.Wait()

	codec.Close()

	s.mu.Lock()
	if s.closeCallbacks != nil {
		for _, cb := range s.closeCallbacks {
			cb(conn)
		}
	}
	s.mu.Unlock()
	conn.Close()
}

// Listen listens on the configured address but does not start
// accepting connections until Serve is called.
func (s *Server) Listen() error {
	s.mu.Lock()
	defer s.mu.Unlock()
	tlsConfig, err := s.context.GetServerTLSConfig()
	if err != nil {
		return err
	}
	ln, err := tlsListen(s.addr.Network(), s.addr.String(), tlsConfig)
	if err != nil {
		return err
	}
	s.listener = ln

	addr, err := updatedAddr(s.addr, ln.Addr())
	if err != nil {
		s.Close()
		return err
	}
	s.addr = addr

	return nil
}

// Serve accepts and services connections on the already started
// listener.
func (s *Server) Serve(handler http.Handler) {
	s.handler = handler
	s.activeConns = make(map[net.Conn]struct{})

	server := &http.Server{
		Handler: s,
		ConnState: func(conn net.Conn, state http.ConnState) {
			s.mu.Lock()
			defer s.mu.Unlock()

			switch state {
			case http.StateNew:
				if s.closed {
					conn.Close()
					return
				}
				s.activeConns[conn] = struct{}{}
			case http.StateClosed:
				delete(s.activeConns, conn)
			}
		},
	}

	s.context.Stopper.RunWorker(func() {
		if err := server.Serve(s.listener); err != nil {
			if !strings.HasSuffix(err.Error(), "use of closed network connection") {
				log.Fatal(err)
			}
		}
	})

	s.context.Stopper.RunWorker(func() {
		<-s.context.Stopper.ShouldStop()
		s.Close()
	})
}

// Start runs the RPC server. After this method returns, the socket
// will have been bound. Use Server.Addr() to ascertain server address.
func (s *Server) Start() error {
	if err := s.Listen(); err != nil {
		return err
	}
	s.Serve(s)
	return nil
}

// updatedAddr returns our "official" address based on the address we asked for
// (oldAddr) and the address we successfully bound to (newAddr). It's kind of
// hacky, but necessary to make TLS work.
func updatedAddr(oldAddr, newAddr net.Addr) (net.Addr, error) {
	switch oldAddr.Network() {
	case "tcp", "tcp4", "tcp6":
		// After binding, it's possible that our host and/or port will be
		// different from what we requested. If the hostname is different, we
		// want to keep the original one since it's more likely to match our
		// TLS certificate. But if the port is different, it should be because
		// we asked for ":0" and got an arbitrary unused port; that needs to be
		// reflected in our addr.
		host, oldPort, err := net.SplitHostPort(util.EnsureHost(oldAddr.String()))
		if err != nil {
			return nil, fmt.Errorf("unable to parse original addr '%s': %v",
				oldAddr.String(), err)
		}
		_, newPort, err := net.SplitHostPort(newAddr.String())
		if err != nil {
			return nil, fmt.Errorf("unable to parse new addr '%s': %v",
				newAddr.String(), err)
		}

		if newPort != oldPort && oldPort != "0" {
			log.Warningf("asked for port %s, got %s", oldPort, newPort)
		}

		return util.MakeUnresolvedAddr("tcp", net.JoinHostPort(host, newPort)), nil

	case "unix":
		if oldAddr.String() != newAddr.String() {
			return nil, fmt.Errorf("asked for unix addr %s, got %s", oldAddr, newAddr)
		}
		return newAddr, nil

	default:
		return nil, fmt.Errorf("unexpected network type: %s", oldAddr.Network())
	}
}

// Addr returns the server's network address.
func (s *Server) Addr() net.Addr {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.addr
}

// Close closes the listener.
func (s *Server) Close() {
	// If the server didn't start properly, it might not have a listener.
	if s.listener != nil {
		s.listener.Close()
	}

	s.mu.Lock()
	defer s.mu.Unlock()
	s.closed = true

	for conn := range s.activeConns {
		conn.Close()
	}
}

// readRequests synchronously reads a stream of requests from a
// connection. Each request is handled in a new background goroutine;
// when the handler finishes the response is written to the responses
// channel. When the connection is closed (and any pending requests
// have finished), we close the responses channel.
func (s *Server) readRequests(codec rpc.ServerCodec, responses chan<- serverResponse) {
	var wg sync.WaitGroup
	defer func() {
		wg.Wait()
		close(responses)
	}()

	for {
		req, meth, args, err := s.readRequest(codec)
		if err == io.EOF || err == io.ErrUnexpectedEOF {
			return
		} else if err != nil {
			log.Warningf("rpc: server cannot decode request: %s", err)
			return
		}

		if meth.handler == nil {
			responses <- serverResponse{
				req: req,
				err: util.Errorf("rpc: couldn't find method: %s", req.ServiceMethod),
			}
			continue
		}

		wg.Add(1)
		meth.handler(args, func(reply proto.Message, err error) {
			responses <- serverResponse{
				req:   req,
				reply: reply,
				err:   err,
			}
			wg.Done()
		})
	}
}

// readRequest reads a single request from a connection.
func (s *Server) readRequest(codec rpc.ServerCodec) (req *rpc.Request, m method,
	args proto.Message, err error) {
	req = &rpc.Request{}
	if err = codec.ReadRequestHeader(req); err != nil {
		return
	}

	s.mu.RLock()
	var ok bool
	m, ok = s.methods[req.ServiceMethod]
	s.mu.RUnlock()

	// If we found the method, construct a request protobuf and parse into it.
	// If not, consume and discard the input by passing nil to ReadRequestBody.
	if ok {
		args = reflect.New(m.reqType.Elem()).Interface().(proto.Message)
	}
	err = codec.ReadRequestBody(args)
	return
}

// sendResponses sends a stream of responses on a connection, and
// exits when the channel is closed.
func (s *Server) sendResponses(codec rpc.ServerCodec, responses <-chan serverResponse) {
	for resp := range responses {
		rpcResp := rpc.Response{
			ServiceMethod: resp.req.ServiceMethod,
			Seq:           resp.req.Seq,
		}
		if resp.err != nil {
			rpcResp.Error = resp.err.Error()
		}
		if err := codec.WriteResponse(&rpcResp, resp.reply); err != nil {
			log.Warningf("rpc: write response failed")
			// TODO(bdarnell): what to do at this point? close the connection?
			// net/rpc just swallows the error.
		}
	}
}
<commit_msg>Use some value types<commit_after>// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Spencer Kimball (spencer.kimball@gmail.com)

package rpc

import (
	"fmt"
	"io"
	"net"
	"net/http"
	"net/rpc"
	"reflect"
	"strings"
	"sync"

	"github.com/cockroachdb/cockroach/rpc/codec"
	"github.com/cockroachdb/cockroach/security"
	"github.com/cockroachdb/cockroach/util"
	"github.com/cockroachdb/cockroach/util/log"
	"github.com/gogo/protobuf/proto"
)

type method struct {
	handler func(proto.Message, func(proto.Message, error))
	reqType reflect.Type
}

type serverResponse struct {
	req   rpc.Request
	reply proto.Message
	err   error
}

type syncAdapter func(proto.Message) (proto.Message, error)

func (s syncAdapter) exec(args proto.Message, callback func(proto.Message, error)) {
	go func() {
		callback(s(args))
	}()
}

// Server is a Cockroach-specific RPC server. By default it handles a simple
// heartbeat protocol to measure link health. It also supports close callbacks.
//
// TODO(spencer): heartbeat protocol should also measure link latency.
type Server struct {
	listener net.Listener // Server listener

	activeConns map[net.Conn]struct{}
	handler     http.Handler

	context *Context

	mu             sync.RWMutex          // Mutex protects the fields below
	addr           net.Addr              // Server address; may change if picking unused port
	closed         bool                  // Set upon invocation of Close()
	closeCallbacks []func(conn net.Conn) // Slice of callbacks to invoke on conn close
	methods        map[string]method
}

// NewServer creates a new instance of Server.
func NewServer(addr net.Addr, context *Context) *Server {
	s := &Server{
		context: context,
		addr:    addr,
		methods: map[string]method{},
	}
	heartbeat := &HeartbeatService{
		clock:              context.localClock,
		remoteClockMonitor: context.RemoteClocks,
	}
	if err := heartbeat.Register(s); err != nil {
		log.Fatalf("unable to register heartbeat service with RPC server: %s", err)
	}
	return s
}

// Register a new method handler. `name` is a qualified name of the
// form "Service.Name". `handler` is a function that takes an
// argument of the same type as `reqPrototype`. Both the argument and
// return value of 'handler' should be a pointer to a protocol message
// type. The handler function will be executed in a new goroutine.
func (s *Server) Register(name string, handler func(proto.Message) (proto.Message, error),
	reqPrototype proto.Message) error {
	return s.RegisterAsync(name, syncAdapter(handler).exec, reqPrototype)
}

// RegisterAsync registers an asynchronous method handler. Instead of
// returning a (proto.Message, error) tuple, an asynchronous handler
// receives a callback which it must execute when it is complete. Note
// that async handlers are started in the RPC server's goroutine and
// must not block (i.e. they must start a goroutine or write to a
// channel promptly). However, the fact that they are started in the
// RPC server's goroutine guarantees that the order of requests as
// they were read from the connection is preserved.
func (s *Server) RegisterAsync(name string, handler func(proto.Message, func(proto.Message, error)),
	reqPrototype proto.Message) error {
	s.mu.Lock()
	defer s.mu.Unlock()

	if _, ok := s.methods[name]; ok {
		return util.Errorf("method %s already registered", name)
	}
	reqType := reflect.TypeOf(reqPrototype)
	if reqType.Kind() != reflect.Ptr {
		// net/rpc supports non-pointer requests, but we always use pointers
		// and things are a little simpler this way.
		return util.Errorf("request type not a pointer")
	}
	s.methods[name] = method{
		handler: handler,
		reqType: reqType,
	}
	return nil
}

// AddCloseCallback adds a callback to the closeCallbacks slice to
// be invoked when a connection is closed.
func (s *Server) AddCloseCallback(cb func(conn net.Conn)) {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.closeCallbacks = append(s.closeCallbacks, cb)
}

// Can connect to RPC service using HTTP CONNECT to rpcPath.
var connected = "200 Connected to Go RPC"

// ServeHTTP implements an http.Handler that answers RPC requests.
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path != rpc.DefaultRPCPath {
		if s.handler != nil {
			s.handler.ServeHTTP(w, r)
			return
		}
		http.NotFound(w, r)
		return
	}

	// Note: this code was adapted from net/rpc.Server.ServeHTTP.
	if r.Method != "CONNECT" {
		http.Error(w, "405 must CONNECT", http.StatusMethodNotAllowed)
		return
	}

	// Construct an authentication hook for this security mode and TLS state.
	authHook, err := security.AuthenticationHook(s.context.Insecure, r.TLS)
	if err != nil {
		http.Error(w, err.Error(), http.StatusUnauthorized)
		return
	}

	conn, _, err := w.(http.Hijacker).Hijack()
	if err != nil {
		log.Infof("rpc hijacking %s: %s", r.RemoteAddr, err)
		return
	}
	security.LogTLSState("RPC", r.TLS)
	io.WriteString(conn, "HTTP/1.0 "+connected+"\n\n")

	codec := codec.NewServerCodec(conn, authHook)
	responses := make(chan serverResponse)
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		s.sendResponses(codec, responses)
		wg.Done()
	}()
	s.readRequests(codec, responses)
	wg.Wait()

	codec.Close()

	s.mu.Lock()
	if s.closeCallbacks != nil {
		for _, cb := range s.closeCallbacks {
			cb(conn)
		}
	}
	s.mu.Unlock()
	conn.Close()
}

// Listen listens on the configured address but does not start
// accepting connections until Serve is called.
func (s *Server) Listen() error {
	s.mu.Lock()
	defer s.mu.Unlock()
	tlsConfig, err := s.context.GetServerTLSConfig()
	if err != nil {
		return err
	}
	ln, err := tlsListen(s.addr.Network(), s.addr.String(), tlsConfig)
	if err != nil {
		return err
	}
	s.listener = ln

	addr, err := updatedAddr(s.addr, ln.Addr())
	if err != nil {
		s.Close()
		return err
	}
	s.addr = addr

	return nil
}

// Serve accepts and services connections on the already started
// listener.
func (s *Server) Serve(handler http.Handler) {
	s.handler = handler
	s.activeConns = make(map[net.Conn]struct{})

	server := &http.Server{
		Handler: s,
		ConnState: func(conn net.Conn, state http.ConnState) {
			s.mu.Lock()
			defer s.mu.Unlock()

			switch state {
			case http.StateNew:
				if s.closed {
					conn.Close()
					return
				}
				s.activeConns[conn] = struct{}{}
			case http.StateClosed:
				delete(s.activeConns, conn)
			}
		},
	}

	s.context.Stopper.RunWorker(func() {
		if err := server.Serve(s.listener); err != nil {
			if !strings.HasSuffix(err.Error(), "use of closed network connection") {
				log.Fatal(err)
			}
		}
	})

	s.context.Stopper.RunWorker(func() {
		<-s.context.Stopper.ShouldStop()
		s.Close()
	})
}

// Start runs the RPC server. After this method returns, the socket
// will have been bound. Use Server.Addr() to ascertain server address.
func (s *Server) Start() error {
	if err := s.Listen(); err != nil {
		return err
	}
	s.Serve(s)
	return nil
}

// updatedAddr returns our "official" address based on the address we asked for
// (oldAddr) and the address we successfully bound to (newAddr). It's kind of
// hacky, but necessary to make TLS work.
func updatedAddr(oldAddr, newAddr net.Addr) (net.Addr, error) {
	switch oldAddr.Network() {
	case "tcp", "tcp4", "tcp6":
		// After binding, it's possible that our host and/or port will be
		// different from what we requested. If the hostname is different, we
		// want to keep the original one since it's more likely to match our
		// TLS certificate. But if the port is different, it should be because
		// we asked for ":0" and got an arbitrary unused port; that needs to be
		// reflected in our addr.
		host, oldPort, err := net.SplitHostPort(util.EnsureHost(oldAddr.String()))
		if err != nil {
			return nil, fmt.Errorf("unable to parse original addr '%s': %v",
				oldAddr.String(), err)
		}
		_, newPort, err := net.SplitHostPort(newAddr.String())
		if err != nil {
			return nil, fmt.Errorf("unable to parse new addr '%s': %v",
				newAddr.String(), err)
		}

		if newPort != oldPort && oldPort != "0" {
			log.Warningf("asked for port %s, got %s", oldPort, newPort)
		}

		return util.MakeUnresolvedAddr("tcp", net.JoinHostPort(host, newPort)), nil

	case "unix":
		if oldAddr.String() != newAddr.String() {
			return nil, fmt.Errorf("asked for unix addr %s, got %s", oldAddr, newAddr)
		}
		return newAddr, nil

	default:
		return nil, fmt.Errorf("unexpected network type: %s", oldAddr.Network())
	}
}

// Addr returns the server's network address.
func (s *Server) Addr() net.Addr {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.addr
}

// Close closes the listener.
func (s *Server) Close() {
	// If the server didn't start properly, it might not have a listener.
	if s.listener != nil {
		s.listener.Close()
	}

	s.mu.Lock()
	defer s.mu.Unlock()
	s.closed = true

	for conn := range s.activeConns {
		conn.Close()
	}
}

// readRequests synchronously reads a stream of requests from a
// connection. Each request is handled in a new background goroutine;
// when the handler finishes the response is written to the responses
// channel. When the connection is closed (and any pending requests
// have finished), we close the responses channel.
func (s *Server) readRequests(codec rpc.ServerCodec, responses chan<- serverResponse) {
	var wg sync.WaitGroup
	defer func() {
		wg.Wait()
		close(responses)
	}()

	for {
		req, meth, args, err := s.readRequest(codec)
		if err == io.EOF || err == io.ErrUnexpectedEOF {
			return
		} else if err != nil {
			log.Warningf("rpc: server cannot decode request: %s", err)
			return
		}

		if meth.handler == nil {
			responses <- serverResponse{
				req: req,
				err: util.Errorf("rpc: couldn't find method: %s", req.ServiceMethod),
			}
			continue
		}

		wg.Add(1)
		meth.handler(args, func(reply proto.Message, err error) {
			responses <- serverResponse{
				req:   req,
				reply: reply,
				err:   err,
			}
			wg.Done()
		})
	}
}

// readRequest reads a single request from a connection.
func (s *Server) readRequest(codec rpc.ServerCodec) (req rpc.Request, m method,
	args proto.Message, err error) {
	if err = codec.ReadRequestHeader(&req); err != nil {
		return
	}

	s.mu.RLock()
	var ok bool
	m, ok = s.methods[req.ServiceMethod]
	s.mu.RUnlock()

	// If we found the method, construct a request protobuf and parse into it.
	// If not, consume and discard the input by passing nil to ReadRequestBody.
	if ok {
		args = reflect.New(m.reqType.Elem()).Interface().(proto.Message)
	}
	err = codec.ReadRequestBody(args)
	return
}

// sendResponses sends a stream of responses on a connection, and
// exits when the channel is closed.
func (s *Server) sendResponses(codec rpc.ServerCodec, responses <-chan serverResponse) {
	for resp := range responses {
		rpcResp := rpc.Response{
			ServiceMethod: resp.req.ServiceMethod,
			Seq:           resp.req.Seq,
		}
		if resp.err != nil {
			rpcResp.Error = resp.err.Error()
		}
		if err := codec.WriteResponse(&rpcResp, resp.reply); err != nil {
			log.Warningf("rpc: write response failed")
			// TODO(bdarnell): what to do at this point? close the connection?
			// net/rpc just swallows the error.
		}
	}
}
<|endoftext|>


<commit_before>/* vim: set sw=4 sts=4 et foldmethod=syntax : */

/*
 * Copyright (c) 2011 Alexander Færøy <ahf@0x90.dk>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * * Redistributions of source code must retain the above copyright notice, this
 *   list of conditions and the following disclaimer.
 *
 * * Redistributions in binary form must reproduce the above copyright notice,
 *   this list of conditions and the following disclaimer in the documentation
 *   and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package main

import (
    "testing"
)

type stripTestCase struct {
    input string
    expected string
}

var stripTests = []stripTestCase {
    stripTestCase{"", ""},
    stripTestCase{"a", "a"},
    stripTestCase{"abc", "abc"},
    stripTestCase{"\r\n", ""},
    stripTestCase{"foobar\r\n  ", "foobar"},
    stripTestCase{"\nfoobar\n", "foobar"},
    stripTestCase{"\rfoobar\n   ", "foobar"},
    stripTestCase{"    \rfoobar\n", "foobar"},
    stripTestCase{"goat\r\r\r", "goat"},
}

func TestStrip(t *testing.T) {
    for i := range stripTests {
        test := stripTests[i]
        result := Strip(test.input)

        if result != test.expected {
            t.Errorf("Strip('%s') = '%s', want '%s'.", test.input, result, test.expected)
        }
    }
}
<commit_msg>Add test cases for the Join() function.<commit_after>/* vim: set sw=4 sts=4 et foldmethod=syntax : */

/*
 * Copyright (c) 2011 Alexander Færøy <ahf@0x90.dk>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * * Redistributions of source code must retain the above copyright notice, this
 *   list of conditions and the following disclaimer.
 *
 * * Redistributions in binary form must reproduce the above copyright notice,
 *   this list of conditions and the following disclaimer in the documentation
 *   and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package main

import (
    "testing"
)

type stripTestCase struct {
    input string
    expected string
}

var stripTests = []stripTestCase {
    stripTestCase{"", ""},
    stripTestCase{"a", "a"},
    stripTestCase{"abc", "abc"},
    stripTestCase{"\r\n", ""},
    stripTestCase{"foobar\r\n  ", "foobar"},
    stripTestCase{"\nfoobar\n", "foobar"},
    stripTestCase{"\rfoobar\n   ", "foobar"},
    stripTestCase{"    \rfoobar\n", "foobar"},
    stripTestCase{"goat\r\r\r", "goat"},
}

type joinTestCase struct {
    input []string
    expected string
}

var joinTests = []joinTestCase {
    joinTestCase{[]string{}, "[]"},
    joinTestCase{[]string{"a"}, "[a]"},
    joinTestCase{[]string{"a", "b", "c"}, "[a, b, c]"},
    joinTestCase{[]string{"foo", "bar", "baz"}, "[foo, bar, baz]"},
}

func TestStrip(t *testing.T) {
    for i := range stripTests {
        test := stripTests[i]
        result := Strip(test.input)

        if result != test.expected {
            t.Errorf("Strip('%s') = '%s', want '%s'.", test.input, result, test.expected)
        }
    }
}

func TestJoin(t *testing.T) {
    for i := range joinTests {
        test := joinTests[i]
        result := Join(test.input)

        if result != test.expected {
            t.Errorf("Join('%s') = '%s', want '%s'.", Join(test.input), result, test.expected)
        }
    }
}
<|endoftext|>


<commit_before>package stick

import (
	"errors"
	"fmt"
	"regexp"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestF(t *testing.T) {
	err := F("foo %d", 42)

	str := err.Error()
	assert.Equal(t, `foo 42`, str)

	str = fmt.Sprintf("%s", err)
	assert.Equal(t, `foo 42`, str)

	str = fmt.Sprintf("%v", err)
	assert.Equal(t, `foo 42`, str)

	str = fmt.Sprintf("%+v", err)
	assert.Equal(t, []string{
		"foo 42",
		"github.com/256dpi/fire/stick.F",
		"  /Users/256dpi/Development/GitHub/256dpi/fire/stick/error.go:LN",
		"github.com/256dpi/fire/stick.TestF",
		"  /Users/256dpi/Development/GitHub/256dpi/fire/stick/error_test.go:LN",
		"testing.tRunner",
		"  /usr/local/Cellar/go/1.14.1/libexec/src/testing/testing.go:LN",
		"runtime.goexit",
		"  /usr/local/Cellar/go/1.14.1/libexec/src/runtime/asm_amd64.s:LN",
	}, splitTrace(str))
}

func TestWF(t *testing.T) {
	err := errors.New("foo")
	err = WF(err, "bar %d", 42)

	str := err.Error()
	assert.Equal(t, `bar 42: foo`, str)

	str = fmt.Sprintf("%s", err)
	assert.Equal(t, `bar 42: foo`, str)

	str = fmt.Sprintf("%v", err)
	assert.Equal(t, `bar 42: foo`, str)

	str = fmt.Sprintf("%+v", err)
	assert.Equal(t, []string{
		"foo",
		"bar 42",
		"github.com/256dpi/fire/stick.WF",
		"  /Users/256dpi/Development/GitHub/256dpi/fire/stick/error.go:LN",
		"github.com/256dpi/fire/stick.TestWF",
		"  /Users/256dpi/Development/GitHub/256dpi/fire/stick/error_test.go:LN",
		"testing.tRunner",
		"  /usr/local/Cellar/go/1.14.1/libexec/src/testing/testing.go:LN",
		"runtime.goexit",
		"  /usr/local/Cellar/go/1.14.1/libexec/src/runtime/asm_amd64.s:LN",
	}, splitTrace(str))
}

func TestE(t *testing.T) {
	err := E("foo")
	assert.True(t, IsSafe(err))

	str := err.Error()
	assert.Equal(t, `foo`, str)

	str = fmt.Sprintf("%s", err)
	assert.Equal(t, `foo`, str)

	str = fmt.Sprintf("%v", err)
	assert.Equal(t, `foo`, str)

	str = fmt.Sprintf("%+v", err)
	assert.Equal(t, []string{
		"foo",
		"github.com/256dpi/fire/stick.F",
		"  /Users/256dpi/Development/GitHub/256dpi/fire/stick/error.go:LN",
		"github.com/256dpi/fire/stick.E",
		"  /Users/256dpi/Development/GitHub/256dpi/fire/stick/error.go:LN",
		"github.com/256dpi/fire/stick.TestE",
		"  /Users/256dpi/Development/GitHub/256dpi/fire/stick/error_test.go:LN",
		"testing.tRunner",
		"  /usr/local/Cellar/go/1.14.1/libexec/src/testing/testing.go:LN",
		"runtime.goexit",
		"  /usr/local/Cellar/go/1.14.1/libexec/src/runtime/asm_amd64.s:LN",
	}, splitTrace(str))

	/* wrapped */

	err = WF(err, "bar")
	assert.True(t, IsSafe(err))

	str = err.Error()
	assert.Equal(t, `bar: foo`, str)

	str = fmt.Sprintf("%s", err)
	assert.Equal(t, `bar: foo`, str)

	str = fmt.Sprintf("%v", err)
	assert.Equal(t, `bar: foo`, str)

	str = fmt.Sprintf("%+v", err)
	assert.Equal(t, []string{
		"foo",
		"github.com/256dpi/fire/stick.F",
		"  /Users/256dpi/Development/GitHub/256dpi/fire/stick/error.go:LN",
		"github.com/256dpi/fire/stick.E",
		"  /Users/256dpi/Development/GitHub/256dpi/fire/stick/error.go:LN",
		"github.com/256dpi/fire/stick.TestE",
		"  /Users/256dpi/Development/GitHub/256dpi/fire/stick/error_test.go:LN",
		"testing.tRunner",
		"  /usr/local/Cellar/go/1.14.1/libexec/src/testing/testing.go:LN",
		"runtime.goexit",
		"  /usr/local/Cellar/go/1.14.1/libexec/src/runtime/asm_amd64.s:LN",
		"bar",
		"github.com/256dpi/fire/stick.WF",
		"  /Users/256dpi/Development/GitHub/256dpi/fire/stick/error.go:LN",
		"github.com/256dpi/fire/stick.TestE",
		"  /Users/256dpi/Development/GitHub/256dpi/fire/stick/error_test.go:LN",
		"testing.tRunner",
		"  /usr/local/Cellar/go/1.14.1/libexec/src/testing/testing.go:LN",
		"runtime.goexit",
		"  /usr/local/Cellar/go/1.14.1/libexec/src/runtime/asm_amd64.s:LN",
	}, splitTrace(str))
}

func TestSafeError(t *testing.T) {
	err1 := errors.New("foo")
	assert.False(t, IsSafe(err1))
	assert.Equal(t, "foo", err1.Error())
	assert.Nil(t, AsSafe(err1))

	err2 := Safe(err1)
	assert.True(t, IsSafe(err2))
	assert.Equal(t, "foo", err2.Error())
	assert.Equal(t, err2, AsSafe(err2))

	err3 := WF(err2, "bar")
	assert.True(t, IsSafe(err3))
	assert.Equal(t, "bar: foo", err3.Error())
	assert.Equal(t, err2, AsSafe(err3))
}

func splitTrace(str string) []string {
	str = strings.ReplaceAll(str, "\t", "  ")
	str = regexp.MustCompile(":\\d+").ReplaceAllString(str, ":LN")
	return strings.Split(str, "\n")
}
<commit_msg>use custorm formatter<commit_after>package stick

import (
	"fmt"
	"regexp"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestF(t *testing.T) {
	err := F("foo %d", 42)

	str := err.Error()
	assert.Equal(t, `foo 42`, str)

	str = fmt.Sprintf("%s", err)
	assert.Equal(t, `foo 42`, str)

	str = fmt.Sprintf("%v", err)
	assert.Equal(t, `foo 42`, str)

	str = fmt.Sprintf("%+v", err)
	assert.Equal(t, []string{
		"foo 42",
		"github.com/256dpi/fire/stick.F",
		"  /Users/256dpi/Development/GitHub/256dpi/fire/stick/error.go:LN",
		"github.com/256dpi/fire/stick.TestF",
		"  /Users/256dpi/Development/GitHub/256dpi/fire/stick/error_test.go:LN",
		"testing.tRunner",
		"  /usr/local/Cellar/go/1.14.1/libexec/src/testing/testing.go:LN",
		"runtime.goexit",
		"  /usr/local/Cellar/go/1.14.1/libexec/src/runtime/asm_amd64.s:LN",
	}, splitTrace(str))
}

func TestWF(t *testing.T) {
	err := F("foo")
	err = WF(err, "bar %d", 42)

	str := err.Error()
	assert.Equal(t, `bar 42: foo`, str)

	str = fmt.Sprintf("%s", err)
	assert.Equal(t, `bar 42: foo`, str)

	str = fmt.Sprintf("%v", err)
	assert.Equal(t, `bar 42: foo`, str)

	str = fmt.Sprintf("%+v", err)
	assert.Equal(t, []string{
		"foo",
		"github.com/256dpi/fire/stick.F",
		"  /Users/256dpi/Development/GitHub/256dpi/fire/stick/error.go:LN",
		"github.com/256dpi/fire/stick.TestWF",
		"  /Users/256dpi/Development/GitHub/256dpi/fire/stick/error_test.go:LN",
		"testing.tRunner",
		"  /usr/local/Cellar/go/1.14.1/libexec/src/testing/testing.go:LN",
		"runtime.goexit",
		"  /usr/local/Cellar/go/1.14.1/libexec/src/runtime/asm_amd64.s:LN",
		"bar 42",
		"github.com/256dpi/fire/stick.WF",
		"  /Users/256dpi/Development/GitHub/256dpi/fire/stick/error.go:LN",
		"github.com/256dpi/fire/stick.TestWF",
		"  /Users/256dpi/Development/GitHub/256dpi/fire/stick/error_test.go:LN",
		"testing.tRunner",
		"  /usr/local/Cellar/go/1.14.1/libexec/src/testing/testing.go:LN",
		"runtime.goexit",
		"  /usr/local/Cellar/go/1.14.1/libexec/src/runtime/asm_amd64.s:LN",
	}, splitTrace(str))
}

func TestE(t *testing.T) {
	err := E("foo")
	assert.True(t, IsSafe(err))

	str := err.Error()
	assert.Equal(t, `foo`, str)

	str = fmt.Sprintf("%s", err)
	assert.Equal(t, `foo`, str)

	str = fmt.Sprintf("%v", err)
	assert.Equal(t, `foo`, str)

	str = fmt.Sprintf("%+v", err)
	assert.Equal(t, []string{
		"foo",
		"github.com/256dpi/fire/stick.F",
		"  /Users/256dpi/Development/GitHub/256dpi/fire/stick/error.go:LN",
		"github.com/256dpi/fire/stick.E",
		"  /Users/256dpi/Development/GitHub/256dpi/fire/stick/error.go:LN",
		"github.com/256dpi/fire/stick.TestE",
		"  /Users/256dpi/Development/GitHub/256dpi/fire/stick/error_test.go:LN",
		"testing.tRunner",
		"  /usr/local/Cellar/go/1.14.1/libexec/src/testing/testing.go:LN",
		"runtime.goexit",
		"  /usr/local/Cellar/go/1.14.1/libexec/src/runtime/asm_amd64.s:LN",
	}, splitTrace(str))

	/* wrapped */

	err = WF(err, "bar")
	assert.True(t, IsSafe(err))

	str = err.Error()
	assert.Equal(t, `bar: foo`, str)

	str = fmt.Sprintf("%s", err)
	assert.Equal(t, `bar: foo`, str)

	str = fmt.Sprintf("%v", err)
	assert.Equal(t, `bar: foo`, str)

	str = fmt.Sprintf("%+v", err)
	assert.Equal(t, []string{
		"foo",
		"github.com/256dpi/fire/stick.F",
		"  /Users/256dpi/Development/GitHub/256dpi/fire/stick/error.go:LN",
		"github.com/256dpi/fire/stick.E",
		"  /Users/256dpi/Development/GitHub/256dpi/fire/stick/error.go:LN",
		"github.com/256dpi/fire/stick.TestE",
		"  /Users/256dpi/Development/GitHub/256dpi/fire/stick/error_test.go:LN",
		"testing.tRunner",
		"  /usr/local/Cellar/go/1.14.1/libexec/src/testing/testing.go:LN",
		"runtime.goexit",
		"  /usr/local/Cellar/go/1.14.1/libexec/src/runtime/asm_amd64.s:LN",
		"bar",
		"github.com/256dpi/fire/stick.WF",
		"  /Users/256dpi/Development/GitHub/256dpi/fire/stick/error.go:LN",
		"github.com/256dpi/fire/stick.TestE",
		"  /Users/256dpi/Development/GitHub/256dpi/fire/stick/error_test.go:LN",
		"testing.tRunner",
		"  /usr/local/Cellar/go/1.14.1/libexec/src/testing/testing.go:LN",
		"runtime.goexit",
		"  /usr/local/Cellar/go/1.14.1/libexec/src/runtime/asm_amd64.s:LN",
	}, splitTrace(str))
}

func TestSafeError(t *testing.T) {
	err1 := F("foo")
	assert.False(t, IsSafe(err1))
	assert.Equal(t, "foo", err1.Error())
	assert.Nil(t, AsSafe(err1))

	err2 := Safe(err1)
	assert.True(t, IsSafe(err2))
	assert.Equal(t, "foo", err2.Error())
	assert.Equal(t, err2, AsSafe(err2))

	err3 := WF(err2, "bar")
	assert.True(t, IsSafe(err3))
	assert.Equal(t, "bar: foo", err3.Error())
	assert.Equal(t, err2, AsSafe(err3))
}

func splitTrace(str string) []string {
	str = strings.ReplaceAll(str, "\t", "  ")
	str = regexp.MustCompile(":\\d+").ReplaceAllString(str, ":LN")
	return strings.Split(str, "\n")
}
<|endoftext|>


<commit_before>package stick

import (
	"fmt"
	"reflect"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"unicode"
	"unicode/utf8"

	"github.com/256dpi/xo"
	"github.com/asaskevich/govalidator"
)

// Validatable represents a type that can be validated.
type Validatable interface {
	Validate() error
}

// NoValidation can be embedded in a struct to provide a no-op validation method.
type NoValidation struct{}

// Validate will perform no validation.
func (*NoValidation) Validate() error {
	return nil
}

// ValidationError describes a validation error.
type ValidationError map[error][]string

// Error implements the error interface.
func (e ValidationError) Error() string {
	// collect messages
	var messages []string
	for err, path := range e {
		// get message
		msg := "error"
		if xo.IsSafe(err) {
			msg = err.Error()
		}

		// add message
		messages = append(messages, fmt.Sprintf("%s: %s", strings.Join(path, "."), msg))
	}

	// sort messages
	sort.Strings(messages)

	// combine messages
	err := strings.Join(messages, "; ")

	return err
}

// Validator is used to validate an object.
type Validator struct {
	obj   Accessible
	path  []string
	error ValidationError
}

// Validate will validate the provided accessible using the specified validator
// function.
func Validate(obj Accessible, fn func(v *Validator)) error {
	// prepare validator
	val := &Validator{obj: obj}

	// run validator
	fn(val)

	return xo.SW(val.Error())
}

// Nest nest validation under the specified field.
func (v *Validator) Nest(field string, fn func()) {
	// push
	v.path = append(v.path, field)

	// yield
	fn()

	// pop
	v.path = v.path[:len(v.path)-1]
}

// Value will validate the value at the named field using the provided rules.
// If the value is optional it will be skipped if nil or unwrapped if present.
func (v *Validator) Value(name string, optional bool, rules ...Rule) {
	// get value
	value := MustGet(v.obj, name)

	// prepare subject
	sub := Subject{
		IValue: value,
		RValue: reflect.ValueOf(value),
	}

	// handle optionals
	if optional {
		// skip if nil
		if sub.IsNil() {
			return
		}

		// otherwise unwrap pointer once
		sub.RValue = sub.RValue.Elem()
		sub.IValue = sub.RValue.Interface()
	}

	// execute rules
	for _, rule := range rules {
		err := rule(sub)
		if err != nil {
			v.Report(name, err)
		}
	}
}

// Items will validate each item of the slice at the named field using the
// provided rules.
func (v *Validator) Items(name string, rules ...Rule) {
	// get slice
	slice := reflect.ValueOf(MustGet(v.obj, name))

	// execute rules for each item
	v.Nest(name, func() {
		for i := 0; i < slice.Len(); i++ {
			// get item
			item := slice.Index(i)

			// prepare subject
			sub := Subject{
				IValue: item.Interface(),
				RValue: item,
			}

			// execute rules
			for _, rule := range rules {
				err := rule(sub)
				if err != nil {
					v.Report(strconv.Itoa(i), err)
				}
			}
		}
	})
}

// Report will report a validation error.
func (v *Validator) Report(name string, err error) {
	// ensure error
	if v.error == nil {
		v.error = ValidationError{}
	}

	// copy path
	path := append([]string{}, v.path...)
	path = append(path, name)

	// add error
	v.error[xo.W(err)] = path
}

// Error will return the validation error or nil of no errors have yet been
// reported.
func (v *Validator) Error() error {
	// check error
	if v.error != nil {
		return v.error
	}

	return nil
}

// Subject carries the to be validated value.
type Subject struct {
	IValue interface{}
	RValue reflect.Value
}

// IsNil returns true if the value is nil or a typed nil (zero pointer).
func (s *Subject) IsNil() bool {
	// check plain nil
	if s.IValue == nil {
		return true
	}

	// check typed nils
	switch s.RValue.Kind() {
	case reflect.Ptr, reflect.Slice, reflect.Map, reflect.Interface:
		return s.RValue.IsNil()
	}

	return false
}

// Unwrap will unwrap all pointers and return whether a value is available.
func (s *Subject) Unwrap() bool {
	// check nil
	if s.IsNil() {
		return false
	}

	// unwrap pointers
	var unwrapped bool
	for s.RValue.Kind() == reflect.Ptr && !s.RValue.IsNil() {
		s.RValue = s.RValue.Elem()
		unwrapped = true
	}
	if unwrapped {
		s.IValue = s.RValue.Interface()
	}

	return !s.IsNil()
}

// Rule is a single validation rule.
type Rule func(sub Subject) error

func isZero(sub Subject) bool {
	// check nil
	if sub.IsNil() {
		return true
	}

	// check using IsZero method
	type isZero interface {
		IsZero() bool
	}
	if v, ok := sub.IValue.(isZero); ok {
		// check zeroness
		if !v.IsZero() {
			return false
		}

		return true
	}

	// check using Zero method
	type zero interface {
		Zero() bool
	}
	if v, ok := sub.IValue.(zero); ok {
		// check zeroness
		if !v.Zero() {
			return false
		}

		return true
	}

	// unwrap
	if !sub.Unwrap() {
		return true
	}

	// check zeroness
	if !sub.RValue.IsZero() {
		return false
	}

	return true
}

// IsZero will check if the provided value is zero. It will determine zeroness
// using IsZero() or Zero() if implemented. A nil pointer, slice, array or map
// is also considered as zero.
func IsZero(sub Subject) error {
	// check zeroness
	if !isZero(sub) {
		return xo.SF("not zero")
	}

	return nil
}

// IsNotZero will check if the provided value is not zero. It will determine
// zeroness using IsZero() or Zero() if implemented. A nil pointer, slice, array
// or map is also considered as zero.
func IsNotZero(sub Subject) error {
	// check zeroness
	if isZero(sub) {
		return xo.SF("zero")
	}

	return nil
}

func isEmpty(sub Subject) bool {
	// unwrap
	if !sub.Unwrap() {
		return true
	}

	// check slice and map length
	switch sub.RValue.Kind() {
	case reflect.Slice, reflect.Map:
		return sub.RValue.Len() == 0
	}

	panic(fmt.Sprintf("stick: cannot check length of %T", sub.IValue))
}

// IsEmpty will check if the provided value is empty. Emptiness can only be
// checked for slices and maps.
func IsEmpty(sub Subject) error {
	// check emptiness
	if !isEmpty(sub) {
		return xo.SF("not empty")
	}

	return nil
}

// IsNotEmpty will check if the provided value is not empty. Emptiness can only
// be checked for slices and maps.
func IsNotEmpty(sub Subject) error {
	// check emptiness
	if isEmpty(sub) {
		return xo.SF("empty")
	}

	return nil
}

// IsValid will check if the value is valid by calling Validate(), IsValid() or
// Valid().
func IsValid(sub Subject) error {
	// check using Validate() method
	if v, ok := sub.IValue.(Validatable); ok {
		return v.Validate()
	}

	// check using IsValid() method
	type isValid interface {
		IsValid() bool
	}
	if v, ok := sub.IValue.(isValid); ok {
		// check validity
		if !v.IsValid() {
			return xo.SF("invalid")
		}

		return nil
	}

	// check using Valid() method
	type valid interface {
		Valid() bool
	}
	if v, ok := sub.IValue.(valid); ok {
		// check validity
		if !v.Valid() {
			return xo.SF("invalid")
		}

		return nil
	}

	panic(fmt.Sprintf("stick: cannot check validity of %T", sub.IValue))
}

// IsMinLen checks whether the value has at least the specified length.
func IsMinLen(min int) Rule {
	return func(sub Subject) error {
		// unwrap
		if !sub.Unwrap() {
			return nil
		}

		// check length
		if sub.RValue.Len() < min {
			return xo.SF("too short")
		}

		return nil
	}
}

// IsMaxLen checks whether the value does not exceed the specified length.
func IsMaxLen(max int) Rule {
	return func(sub Subject) error {
		// unwrap
		if !sub.Unwrap() {
			return nil
		}

		// check length
		if sub.RValue.Len() > max {
			return xo.SF("too long")
		}

		return nil
	}
}

// IsMinInt checks whether the value satisfies the provided minimum.
func IsMinInt(min int64) Rule {
	return func(sub Subject) error {
		// unwrap
		if !sub.Unwrap() {
			return nil
		}

		// check value
		switch sub.RValue.Kind() {
		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		default:
			panic("stick: expected int value")
		}

		// check min
		if sub.RValue.Int() < min {
			return xo.SF("too small")
		}

		return nil
	}
}

// IsMaxInt checks whether the value satisfies the provided maximum.
func IsMaxInt(max int64) Rule {
	return func(sub Subject) error {
		// unwrap
		if !sub.Unwrap() {
			return nil
		}

		// check value
		switch sub.RValue.Kind() {
		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		default:
			panic("stick: expected int value")
		}

		// check min
		if sub.RValue.Int() > max {
			return xo.SF("too big")
		}

		return nil
	}
}

// IsMinUint checks whether the value satisfies the provided minimum.
func IsMinUint(min uint64) Rule {
	return func(sub Subject) error {
		// unwrap
		if !sub.Unwrap() {
			return nil
		}

		// check value
		switch sub.RValue.Kind() {
		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		default:
			panic("stick: expected uint value")
		}

		// check range
		if sub.RValue.Uint() < min {
			return xo.SF("too small")
		}

		return nil
	}
}

// IsMaxUint checks whether the value satisfies the provided maximum.
func IsMaxUint(max uint64) Rule {
	return func(sub Subject) error {
		// unwrap
		if !sub.Unwrap() {
			return nil
		}

		// check value
		switch sub.RValue.Kind() {
		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		default:
			panic("stick: expected uint value")
		}

		// check max
		if sub.RValue.Uint() > max {
			return xo.SF("too big")
		}

		return nil
	}
}

// IsMinFloat checks whether the value satisfies the provided minimum.
func IsMinFloat(min float64) Rule {
	return func(sub Subject) error {
		// unwrap
		if !sub.Unwrap() {
			return nil
		}

		// check value
		switch sub.RValue.Kind() {
		case reflect.Float32, reflect.Float64:
		default:
			panic("stick: expected float value")
		}

		// check min
		if sub.RValue.Float() < min {
			return xo.SF("too small")
		}

		return nil
	}
}

// IsMaxFloat checks whether the value satisfies the provided maximum.
func IsMaxFloat(max float64) Rule {
	return func(sub Subject) error {
		// unwrap
		if !sub.Unwrap() {
			return nil
		}

		// check value
		switch sub.RValue.Kind() {
		case reflect.Float32, reflect.Float64:
		default:
			panic("stick: expected float value")
		}

		// check max
		if sub.RValue.Float() > max {
			return xo.SF("too big")
		}

		return nil
	}
}

// IsFormat will check of the value corresponds to the format determined by the
// provided string format checker.
func IsFormat(fns ...func(string) bool) Rule {
	return func(sub Subject) error {
		// unwrap
		if !sub.Unwrap() {
			return nil
		}

		// check value
		if sub.RValue.Kind() != reflect.String {
			panic("stick: expected string value")
		}

		// get string
		str := sub.RValue.String()

		// check zero
		if str == "" {
			return nil
		}

		// check validity
		for _, fn := range fns {
			if !fn(str) {
				return xo.SF("invalid format")
			}
		}

		return nil
	}
}

// IsRegexMatch will check if a string matches a regular expression.
func IsRegexMatch(reg *regexp.Regexp) Rule {
	return IsFormat(reg.MatchString)
}

// IsPatternMatch will check if a string matches a regular expression pattern.
func IsPatternMatch(pattern string) Rule {
	return IsRegexMatch(regexp.MustCompile(pattern))
}

// IsEmail will check if a string is a valid email.
var IsEmail = IsFormat(govalidator.IsEmail)

// IsURL will check if a string is a valid URL.
var IsURL = IsFormat(govalidator.IsURL)

// IsHost will check if a string is a valid host.
var IsHost = IsFormat(govalidator.IsHost)

// IsDNSName will check if a string is a valid DNS name.
var IsDNSName = IsFormat(govalidator.IsDNSName)

// IsIPAddress will check if a string is a valid IP address.
var IsIPAddress = IsFormat(govalidator.IsIP)

// IsNumeric will check if a string is numeric.
var IsNumeric = IsFormat(govalidator.IsNumeric)

// IsValidUTF8 will check if a string is valid utf8.
var IsValidUTF8 = IsFormat(utf8.ValidString)

// IsVisible will check if a string is visible.
var IsVisible = IsFormat(utf8.ValidString, func(s string) bool {
	// count characters and whitespace
	c := 0
	w := 0
	for _, r := range s {
		c++
		if unicode.IsSpace(r) {
			w++
		}
	}

	return w < c
})
<commit_msg>stick: moved safe wrap<commit_after>package stick

import (
	"fmt"
	"reflect"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"unicode"
	"unicode/utf8"

	"github.com/256dpi/xo"
	"github.com/asaskevich/govalidator"
)

// Validatable represents a type that can be validated.
type Validatable interface {
	Validate() error
}

// NoValidation can be embedded in a struct to provide a no-op validation method.
type NoValidation struct{}

// Validate will perform no validation.
func (*NoValidation) Validate() error {
	return nil
}

// ValidationError describes a validation error.
type ValidationError map[error][]string

// Error implements the error interface.
func (e ValidationError) Error() string {
	// collect messages
	var messages []string
	for err, path := range e {
		// get message
		msg := "error"
		if xo.IsSafe(err) {
			msg = err.Error()
		}

		// add message
		messages = append(messages, fmt.Sprintf("%s: %s", strings.Join(path, "."), msg))
	}

	// sort messages
	sort.Strings(messages)

	// combine messages
	err := strings.Join(messages, "; ")

	return err
}

// Validator is used to validate an object.
type Validator struct {
	obj   Accessible
	path  []string
	error ValidationError
}

// Validate will validate the provided accessible using the specified validator
// function.
func Validate(obj Accessible, fn func(v *Validator)) error {
	// prepare validator
	val := &Validator{obj: obj}

	// run validator
	fn(val)

	return val.Error()
}

// Nest nest validation under the specified field.
func (v *Validator) Nest(field string, fn func()) {
	// push
	v.path = append(v.path, field)

	// yield
	fn()

	// pop
	v.path = v.path[:len(v.path)-1]
}

// Value will validate the value at the named field using the provided rules.
// If the value is optional it will be skipped if nil or unwrapped if present.
func (v *Validator) Value(name string, optional bool, rules ...Rule) {
	// get value
	value := MustGet(v.obj, name)

	// prepare subject
	sub := Subject{
		IValue: value,
		RValue: reflect.ValueOf(value),
	}

	// handle optionals
	if optional {
		// skip if nil
		if sub.IsNil() {
			return
		}

		// otherwise unwrap pointer once
		sub.RValue = sub.RValue.Elem()
		sub.IValue = sub.RValue.Interface()
	}

	// execute rules
	for _, rule := range rules {
		err := rule(sub)
		if err != nil {
			v.Report(name, err)
		}
	}
}

// Items will validate each item of the slice at the named field using the
// provided rules.
func (v *Validator) Items(name string, rules ...Rule) {
	// get slice
	slice := reflect.ValueOf(MustGet(v.obj, name))

	// execute rules for each item
	v.Nest(name, func() {
		for i := 0; i < slice.Len(); i++ {
			// get item
			item := slice.Index(i)

			// prepare subject
			sub := Subject{
				IValue: item.Interface(),
				RValue: item,
			}

			// execute rules
			for _, rule := range rules {
				err := rule(sub)
				if err != nil {
					v.Report(strconv.Itoa(i), err)
				}
			}
		}
	})
}

// Report will report a validation error.
func (v *Validator) Report(name string, err error) {
	// ensure error
	if v.error == nil {
		v.error = ValidationError{}
	}

	// copy path
	path := append([]string{}, v.path...)
	path = append(path, name)

	// add error
	v.error[xo.W(err)] = path
}

// Error will return the validation error or nil of no errors have yet been
// reported.
func (v *Validator) Error() error {
	// check error
	if v.error != nil {
		return xo.SW(v.error)
	}

	return nil
}

// Subject carries the to be validated value.
type Subject struct {
	IValue interface{}
	RValue reflect.Value
}

// IsNil returns true if the value is nil or a typed nil (zero pointer).
func (s *Subject) IsNil() bool {
	// check plain nil
	if s.IValue == nil {
		return true
	}

	// check typed nils
	switch s.RValue.Kind() {
	case reflect.Ptr, reflect.Slice, reflect.Map, reflect.Interface:
		return s.RValue.IsNil()
	}

	return false
}

// Unwrap will unwrap all pointers and return whether a value is available.
func (s *Subject) Unwrap() bool {
	// check nil
	if s.IsNil() {
		return false
	}

	// unwrap pointers
	var unwrapped bool
	for s.RValue.Kind() == reflect.Ptr && !s.RValue.IsNil() {
		s.RValue = s.RValue.Elem()
		unwrapped = true
	}
	if unwrapped {
		s.IValue = s.RValue.Interface()
	}

	return !s.IsNil()
}

// Rule is a single validation rule.
type Rule func(sub Subject) error

func isZero(sub Subject) bool {
	// check nil
	if sub.IsNil() {
		return true
	}

	// check using IsZero method
	type isZero interface {
		IsZero() bool
	}
	if v, ok := sub.IValue.(isZero); ok {
		// check zeroness
		if !v.IsZero() {
			return false
		}

		return true
	}

	// check using Zero method
	type zero interface {
		Zero() bool
	}
	if v, ok := sub.IValue.(zero); ok {
		// check zeroness
		if !v.Zero() {
			return false
		}

		return true
	}

	// unwrap
	if !sub.Unwrap() {
		return true
	}

	// check zeroness
	if !sub.RValue.IsZero() {
		return false
	}

	return true
}

// IsZero will check if the provided value is zero. It will determine zeroness
// using IsZero() or Zero() if implemented. A nil pointer, slice, array or map
// is also considered as zero.
func IsZero(sub Subject) error {
	// check zeroness
	if !isZero(sub) {
		return xo.SF("not zero")
	}

	return nil
}

// IsNotZero will check if the provided value is not zero. It will determine
// zeroness using IsZero() or Zero() if implemented. A nil pointer, slice, array
// or map is also considered as zero.
func IsNotZero(sub Subject) error {
	// check zeroness
	if isZero(sub) {
		return xo.SF("zero")
	}

	return nil
}

func isEmpty(sub Subject) bool {
	// unwrap
	if !sub.Unwrap() {
		return true
	}

	// check slice and map length
	switch sub.RValue.Kind() {
	case reflect.Slice, reflect.Map:
		return sub.RValue.Len() == 0
	}

	panic(fmt.Sprintf("stick: cannot check length of %T", sub.IValue))
}

// IsEmpty will check if the provided value is empty. Emptiness can only be
// checked for slices and maps.
func IsEmpty(sub Subject) error {
	// check emptiness
	if !isEmpty(sub) {
		return xo.SF("not empty")
	}

	return nil
}

// IsNotEmpty will check if the provided value is not empty. Emptiness can only
// be checked for slices and maps.
func IsNotEmpty(sub Subject) error {
	// check emptiness
	if isEmpty(sub) {
		return xo.SF("empty")
	}

	return nil
}

// IsValid will check if the value is valid by calling Validate(), IsValid() or
// Valid().
func IsValid(sub Subject) error {
	// check using Validate() method
	if v, ok := sub.IValue.(Validatable); ok {
		return v.Validate()
	}

	// check using IsValid() method
	type isValid interface {
		IsValid() bool
	}
	if v, ok := sub.IValue.(isValid); ok {
		// check validity
		if !v.IsValid() {
			return xo.SF("invalid")
		}

		return nil
	}

	// check using Valid() method
	type valid interface {
		Valid() bool
	}
	if v, ok := sub.IValue.(valid); ok {
		// check validity
		if !v.Valid() {
			return xo.SF("invalid")
		}

		return nil
	}

	panic(fmt.Sprintf("stick: cannot check validity of %T", sub.IValue))
}

// IsMinLen checks whether the value has at least the specified length.
func IsMinLen(min int) Rule {
	return func(sub Subject) error {
		// unwrap
		if !sub.Unwrap() {
			return nil
		}

		// check length
		if sub.RValue.Len() < min {
			return xo.SF("too short")
		}

		return nil
	}
}

// IsMaxLen checks whether the value does not exceed the specified length.
func IsMaxLen(max int) Rule {
	return func(sub Subject) error {
		// unwrap
		if !sub.Unwrap() {
			return nil
		}

		// check length
		if sub.RValue.Len() > max {
			return xo.SF("too long")
		}

		return nil
	}
}

// IsMinInt checks whether the value satisfies the provided minimum.
func IsMinInt(min int64) Rule {
	return func(sub Subject) error {
		// unwrap
		if !sub.Unwrap() {
			return nil
		}

		// check value
		switch sub.RValue.Kind() {
		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		default:
			panic("stick: expected int value")
		}

		// check min
		if sub.RValue.Int() < min {
			return xo.SF("too small")
		}

		return nil
	}
}

// IsMaxInt checks whether the value satisfies the provided maximum.
func IsMaxInt(max int64) Rule {
	return func(sub Subject) error {
		// unwrap
		if !sub.Unwrap() {
			return nil
		}

		// check value
		switch sub.RValue.Kind() {
		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		default:
			panic("stick: expected int value")
		}

		// check min
		if sub.RValue.Int() > max {
			return xo.SF("too big")
		}

		return nil
	}
}

// IsMinUint checks whether the value satisfies the provided minimum.
func IsMinUint(min uint64) Rule {
	return func(sub Subject) error {
		// unwrap
		if !sub.Unwrap() {
			return nil
		}

		// check value
		switch sub.RValue.Kind() {
		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		default:
			panic("stick: expected uint value")
		}

		// check range
		if sub.RValue.Uint() < min {
			return xo.SF("too small")
		}

		return nil
	}
}

// IsMaxUint checks whether the value satisfies the provided maximum.
func IsMaxUint(max uint64) Rule {
	return func(sub Subject) error {
		// unwrap
		if !sub.Unwrap() {
			return nil
		}

		// check value
		switch sub.RValue.Kind() {
		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		default:
			panic("stick: expected uint value")
		}

		// check max
		if sub.RValue.Uint() > max {
			return xo.SF("too big")
		}

		return nil
	}
}

// IsMinFloat checks whether the value satisfies the provided minimum.
func IsMinFloat(min float64) Rule {
	return func(sub Subject) error {
		// unwrap
		if !sub.Unwrap() {
			return nil
		}

		// check value
		switch sub.RValue.Kind() {
		case reflect.Float32, reflect.Float64:
		default:
			panic("stick: expected float value")
		}

		// check min
		if sub.RValue.Float() < min {
			return xo.SF("too small")
		}

		return nil
	}
}

// IsMaxFloat checks whether the value satisfies the provided maximum.
func IsMaxFloat(max float64) Rule {
	return func(sub Subject) error {
		// unwrap
		if !sub.Unwrap() {
			return nil
		}

		// check value
		switch sub.RValue.Kind() {
		case reflect.Float32, reflect.Float64:
		default:
			panic("stick: expected float value")
		}

		// check max
		if sub.RValue.Float() > max {
			return xo.SF("too big")
		}

		return nil
	}
}

// IsFormat will check of the value corresponds to the format determined by the
// provided string format checker.
func IsFormat(fns ...func(string) bool) Rule {
	return func(sub Subject) error {
		// unwrap
		if !sub.Unwrap() {
			return nil
		}

		// check value
		if sub.RValue.Kind() != reflect.String {
			panic("stick: expected string value")
		}

		// get string
		str := sub.RValue.String()

		// check zero
		if str == "" {
			return nil
		}

		// check validity
		for _, fn := range fns {
			if !fn(str) {
				return xo.SF("invalid format")
			}
		}

		return nil
	}
}

// IsRegexMatch will check if a string matches a regular expression.
func IsRegexMatch(reg *regexp.Regexp) Rule {
	return IsFormat(reg.MatchString)
}

// IsPatternMatch will check if a string matches a regular expression pattern.
func IsPatternMatch(pattern string) Rule {
	return IsRegexMatch(regexp.MustCompile(pattern))
}

// IsEmail will check if a string is a valid email.
var IsEmail = IsFormat(govalidator.IsEmail)

// IsURL will check if a string is a valid URL.
var IsURL = IsFormat(govalidator.IsURL)

// IsHost will check if a string is a valid host.
var IsHost = IsFormat(govalidator.IsHost)

// IsDNSName will check if a string is a valid DNS name.
var IsDNSName = IsFormat(govalidator.IsDNSName)

// IsIPAddress will check if a string is a valid IP address.
var IsIPAddress = IsFormat(govalidator.IsIP)

// IsNumeric will check if a string is numeric.
var IsNumeric = IsFormat(govalidator.IsNumeric)

// IsValidUTF8 will check if a string is valid utf8.
var IsValidUTF8 = IsFormat(utf8.ValidString)

// IsVisible will check if a string is visible.
var IsVisible = IsFormat(utf8.ValidString, func(s string) bool {
	// count characters and whitespace
	c := 0
	w := 0
	for _, r := range s {
		c++
		if unicode.IsSpace(r) {
			w++
		}
	}

	return w < c
})
<|endoftext|>


<commit_before>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package tikv

import (
	"context"

	"github.com/juju/errors"
	"github.com/pingcap/kvproto/pkg/kvrpcpb"
	"github.com/pingcap/pd/pd-client"
)

// RawKVClient is a client of TiKV server which is used as a key-value storage,
// only GET/PUT/DELETE commands are supported.
type RawKVClient struct {
	clusterID   uint64
	regionCache *RegionCache
	rpcClient   Client
}

// NewRawKVClient creates a client with PD cluster addrs.
func NewRawKVClient(pdAddrs []string) (*RawKVClient, error) {
	pdCli, err := pd.NewClient(pdAddrs)
	if err != nil {
		return nil, errors.Trace(err)
	}
	return &RawKVClient{
		clusterID:   pdCli.GetClusterID(),
		regionCache: NewRegionCache(pdCli),
		rpcClient:   newRPCClient(),
	}, nil
}

// ClusterID returns the TiKV cluster ID.
func (c *RawKVClient) ClusterID() uint64 {
	return c.clusterID
}

// Get queries value with the key. When the key does not exist, it returns
// `nil, nil`, while `[]byte{}, nil` means an empty value.
func (c *RawKVClient) Get(key []byte) ([]byte, error) {
	req := &kvrpcpb.Request{
		Type: kvrpcpb.MessageType_CmdRawGet,
		CmdRawGetReq: &kvrpcpb.CmdRawGetRequest{
			Key: key,
		},
	}
	resp, err := c.sendKVReq(key, req)
	if err != nil {
		return nil, errors.Trace(err)
	}
	cmdResp := resp.GetCmdRawGetResp()
	if cmdResp == nil {
		return nil, errors.Trace(errBodyMissing)
	}
	if cmdResp.GetError() != "" {
		return nil, errors.New(cmdResp.GetError())
	}
	return cmdResp.Value, nil
}

// Put stores a key-value pair to TiKV.
func (c *RawKVClient) Put(key, value []byte) error {
	req := &kvrpcpb.Request{
		Type: kvrpcpb.MessageType_CmdRawPut,
		CmdRawPutReq: &kvrpcpb.CmdRawPutRequest{
			Key:   key,
			Value: value,
		},
	}
	resp, err := c.sendKVReq(key, req)
	if err != nil {
		return errors.Trace(err)
	}
	cmdResp := resp.GetCmdRawPutResp()
	if cmdResp == nil {
		return errors.Trace(errBodyMissing)
	}
	if cmdResp.GetError() != "" {
		return errors.New(cmdResp.GetError())
	}
	return nil
}

// Delete deletes a key-value pair from TiKV.
func (c *RawKVClient) Delete(key []byte) error {
	req := &kvrpcpb.Request{
		Type: kvrpcpb.MessageType_CmdRawDelete,
		CmdRawDeleteReq: &kvrpcpb.CmdRawDeleteRequest{
			Key: key,
		},
	}
	resp, err := c.sendKVReq(key, req)
	if err != nil {
		return errors.Trace(err)
	}
	cmdResp := resp.GetCmdRawDeleteResp()
	if cmdResp == nil {
		return errors.Trace(errBodyMissing)
	}
	if cmdResp.GetError() != "" {
		return errors.New(cmdResp.GetError())
	}
	return nil
}

func (c *RawKVClient) sendKVReq(key []byte, req *kvrpcpb.Request) (*kvrpcpb.Response, error) {
	bo := NewBackoffer(rawkvMaxBackoff, context.Background())
	for {
		region, err := c.regionCache.GetRegion(bo, key)
		if err != nil {
			return nil, errors.Trace(err)
		}
		resp, err := sendKVReq(c.regionCache, c.rpcClient, bo, req, region.VerID(), readTimeoutShort)
		if err != nil {
			return nil, errors.Trace(err)
		}
		if regionErr := resp.GetRegionError(); regionErr != nil {
			err := bo.Backoff(boRegionMiss, errors.New(regionErr.String()))
			if err != nil {
				return nil, errors.Trace(err)
			}
			continue
		}
		return resp, nil
	}
}
<commit_msg>fix "context" import. (#2116)<commit_after>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package tikv

import (
	"github.com/juju/errors"
	"github.com/pingcap/kvproto/pkg/kvrpcpb"
	"github.com/pingcap/pd/pd-client"
	"golang.org/x/net/context"
)

// RawKVClient is a client of TiKV server which is used as a key-value storage,
// only GET/PUT/DELETE commands are supported.
type RawKVClient struct {
	clusterID   uint64
	regionCache *RegionCache
	rpcClient   Client
}

// NewRawKVClient creates a client with PD cluster addrs.
func NewRawKVClient(pdAddrs []string) (*RawKVClient, error) {
	pdCli, err := pd.NewClient(pdAddrs)
	if err != nil {
		return nil, errors.Trace(err)
	}
	return &RawKVClient{
		clusterID:   pdCli.GetClusterID(),
		regionCache: NewRegionCache(pdCli),
		rpcClient:   newRPCClient(),
	}, nil
}

// ClusterID returns the TiKV cluster ID.
func (c *RawKVClient) ClusterID() uint64 {
	return c.clusterID
}

// Get queries value with the key. When the key does not exist, it returns
// `nil, nil`, while `[]byte{}, nil` means an empty value.
func (c *RawKVClient) Get(key []byte) ([]byte, error) {
	req := &kvrpcpb.Request{
		Type: kvrpcpb.MessageType_CmdRawGet,
		CmdRawGetReq: &kvrpcpb.CmdRawGetRequest{
			Key: key,
		},
	}
	resp, err := c.sendKVReq(key, req)
	if err != nil {
		return nil, errors.Trace(err)
	}
	cmdResp := resp.GetCmdRawGetResp()
	if cmdResp == nil {
		return nil, errors.Trace(errBodyMissing)
	}
	if cmdResp.GetError() != "" {
		return nil, errors.New(cmdResp.GetError())
	}
	return cmdResp.Value, nil
}

// Put stores a key-value pair to TiKV.
func (c *RawKVClient) Put(key, value []byte) error {
	req := &kvrpcpb.Request{
		Type: kvrpcpb.MessageType_CmdRawPut,
		CmdRawPutReq: &kvrpcpb.CmdRawPutRequest{
			Key:   key,
			Value: value,
		},
	}
	resp, err := c.sendKVReq(key, req)
	if err != nil {
		return errors.Trace(err)
	}
	cmdResp := resp.GetCmdRawPutResp()
	if cmdResp == nil {
		return errors.Trace(errBodyMissing)
	}
	if cmdResp.GetError() != "" {
		return errors.New(cmdResp.GetError())
	}
	return nil
}

// Delete deletes a key-value pair from TiKV.
func (c *RawKVClient) Delete(key []byte) error {
	req := &kvrpcpb.Request{
		Type: kvrpcpb.MessageType_CmdRawDelete,
		CmdRawDeleteReq: &kvrpcpb.CmdRawDeleteRequest{
			Key: key,
		},
	}
	resp, err := c.sendKVReq(key, req)
	if err != nil {
		return errors.Trace(err)
	}
	cmdResp := resp.GetCmdRawDeleteResp()
	if cmdResp == nil {
		return errors.Trace(errBodyMissing)
	}
	if cmdResp.GetError() != "" {
		return errors.New(cmdResp.GetError())
	}
	return nil
}

func (c *RawKVClient) sendKVReq(key []byte, req *kvrpcpb.Request) (*kvrpcpb.Response, error) {
	bo := NewBackoffer(rawkvMaxBackoff, context.Background())
	for {
		region, err := c.regionCache.GetRegion(bo, key)
		if err != nil {
			return nil, errors.Trace(err)
		}
		resp, err := sendKVReq(c.regionCache, c.rpcClient, bo, req, region.VerID(), readTimeoutShort)
		if err != nil {
			return nil, errors.Trace(err)
		}
		if regionErr := resp.GetRegionError(); regionErr != nil {
			err := bo.Backoff(boRegionMiss, errors.New(regionErr.String()))
			if err != nil {
				return nil, errors.Trace(err)
			}
			continue
		}
		return resp, nil
	}
}
<|endoftext|>