<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package rand

import (
	"math";
	"fmt";
	"os";
	"testing";
)

const (
	numTestSamples = 10000;
)

type statsResults struct {
	mean		float64;
	stddev		float64;
	closeEnough	float64;
	maxError	float64;
}

func max(a, b float64) float64 {
	if a > b {
		return a
	}
	return b;
}

func nearEqual(a, b, closeEnough, maxError float64) bool {
	absDiff := math.Fabs(a - b);
	if absDiff < closeEnough {	// Necessary when one value is zero and one value is close to zero.
		return true
	}
	return absDiff/max(math.Fabs(a), math.Fabs(b)) < maxError;
}

var testSeeds = []int64{1, 1754801282, 1698661970, 1550503961}

// checkSimilarDistribution returns success if the mean and stddev of the
// two statsResults are similar.
func (this *statsResults) checkSimilarDistribution(expected *statsResults) os.Error {
	if !nearEqual(this.mean, expected.mean, expected.closeEnough, expected.maxError) {
		s := fmt.Sprintf("mean %v != %v (allowed error %v, %v)", this.mean, expected.mean, expected.closeEnough, expected.maxError);
		fmt.Println(s);
		return os.ErrorString(s);
	}
	if !nearEqual(this.stddev, expected.stddev, 0, expected.maxError) {
		s := fmt.Sprintf("stddev %v != %v (allowed error %v, %v)", this.stddev, expected.stddev, expected.closeEnough, expected.maxError);
		fmt.Println(s);
		return os.ErrorString(s);
	}
	return nil;
}

func getStatsResults(samples []float64) *statsResults {
	res := new(statsResults);
	var sum float64;
	for i := range samples {
		sum += samples[i]
	}
	res.mean = sum / float64(len(samples));
	var devsum float64;
	for i := range samples {
		devsum += math.Pow(samples[i]-res.mean, 2)
	}
	res.stddev = math.Sqrt(devsum / float64(len(samples)));
	return res;
}

func checkSampleDistribution(t *testing.T, samples []float64, expected *statsResults) {
	actual := getStatsResults(samples);
	err := actual.checkSimilarDistribution(expected);
	if err != nil {
		t.Errorf(err.String())
	}
}

func checkSampleSliceDistributions(t *testing.T, samples []float64, nslices int, expected *statsResults) {
	chunk := len(samples) / nslices;
	for i := 0; i < nslices; i++ {
		low := i * chunk;
		var high int;
		if i == nslices-1 {
			high = len(samples) - 1
		} else {
			high = (i + 1) * chunk
		}
		checkSampleDistribution(t, samples[low:high], expected);
	}
}

//
// Normal distribution tests
//

func generateNormalSamples(nsamples int, mean, stddev float64, seed int64) []float64 {
	r := New(NewSource(seed));
	samples := make([]float64, nsamples);
	for i := range samples {
		samples[i] = r.NormFloat64()*stddev + mean
	}
	return samples;
}

func testNormalDistribution(t *testing.T, nsamples int, mean, stddev float64, seed int64) {
	//fmt.Printf("testing nsamples=%v mean=%v stddev=%v seed=%v\n", nsamples, mean, stddev, seed);

	samples := generateNormalSamples(nsamples, mean, stddev, seed);
	errorScale := max(1.0, stddev);	// Error scales with stddev
	expected := &statsResults{mean, stddev, 0.10 * errorScale, 0.08 * errorScale};

	// Make sure that the entire set matches the expected distribution.
	checkSampleDistribution(t, samples, expected);

	// Make sure that each half of the set matches the expected distribution.
	checkSampleSliceDistributions(t, samples, 2, expected);

	// Make sure that each 7th of the set matches the expected distribution.
	checkSampleSliceDistributions(t, samples, 7, expected);
}

// Actual tests

func TestStandardNormalValues(t *testing.T) {
	for _, seed := range testSeeds {
		testNormalDistribution(t, numTestSamples, 0, 1, seed)
	}
}

func TestNonStandardNormalValues(t *testing.T) {
	for sd := float64(0.5); sd < 1000; sd *= 2 {
		for m := float64(0.5); m < 1000; m *= 2 {
			for _, seed := range testSeeds {
				testNormalDistribution(t, numTestSamples, m, sd, seed)
			}
		}
	}
}

//
// Exponential distribution tests
//

func generateExponentialSamples(nsamples int, rate float64, seed int64) []float64 {
	r := New(NewSource(seed));
	samples := make([]float64, nsamples);
	for i := range samples {
		samples[i] = r.ExpFloat64() / rate
	}
	return samples;
}

func testExponentialDistribution(t *testing.T, nsamples int, rate float64, seed int64) {
	//fmt.Printf("testing nsamples=%v rate=%v seed=%v\n", nsamples, rate, seed);

	mean := 1 / rate;
	stddev := mean;

	samples := generateExponentialSamples(nsamples, rate, seed);
	errorScale := max(1.0, 1/rate);	// Error scales with the inverse of the rate
	expected := &statsResults{mean, stddev, 0.10 * errorScale, 0.20 * errorScale};

	// Make sure that the entire set matches the expected distribution.
	checkSampleDistribution(t, samples, expected);

	// Make sure that each half of the set matches the expected distribution.
	checkSampleSliceDistributions(t, samples, 2, expected);

	// Make sure that each 7th of the set matches the expected distribution.
	checkSampleSliceDistributions(t, samples, 7, expected);
}

// Actual tests

func TestStandardExponentialValues(t *testing.T) {
	for _, seed := range testSeeds {
		testExponentialDistribution(t, numTestSamples, 1, seed)
	}
}

func TestNonStandardExponentialValues(t *testing.T) {
	for rate := float64(0.05); rate < 10; rate *= 2 {
		for _, seed := range testSeeds {
			testExponentialDistribution(t, numTestSamples, rate, seed)
		}
	}
}

//
// Table generation tests
//

func initNorm() (testKn []uint32, testWn, testFn []float32) {
	const m1 = 1 << 31;
	var (
		dn	float64	= rn;
		tn		= dn;
		vn	float64	= 9.91256303526217e-3;
	)

	testKn = make([]uint32, 128);
	testWn = make([]float32, 128);
	testFn = make([]float32, 128);

	q := vn / math.Exp(-0.5*dn*dn);
	testKn[0] = uint32((dn / q) * m1);
	testKn[1] = 0;
	testWn[0] = float32(q / m1);
	testWn[127] = float32(dn / m1);
	testFn[0] = 1.0;
	testFn[127] = float32(math.Exp(-0.5 * dn * dn));
	for i := 126; i >= 1; i-- {
		dn = math.Sqrt(-2.0 * math.Log(vn/dn+math.Exp(-0.5*dn*dn)));
		testKn[i+1] = uint32((dn / tn) * m1);
		tn = dn;
		testFn[i] = float32(math.Exp(-0.5 * dn * dn));
		testWn[i] = float32(dn / m1);
	}
	return;
}

func initExp() (testKe []uint32, testWe, testFe []float32) {
	const m2 = 1 << 32;
	var (
		de	float64	= re;
		te		= de;
		ve	float64	= 3.9496598225815571993e-3;
	)

	testKe = make([]uint32, 256);
	testWe = make([]float32, 256);
	testFe = make([]float32, 256);

	q := ve / math.Exp(-de);
	testKe[0] = uint32((de / q) * m2);
	testKe[1] = 0;
	testWe[0] = float32(q / m2);
	testWe[255] = float32(de / m2);
	testFe[0] = 1.0;
	testFe[255] = float32(math.Exp(-de));
	for i := 254; i >= 1; i-- {
		de = -math.Log(ve/de + math.Exp(-de));
		testKe[i+1] = uint32((de / te) * m2);
		te = de;
		testFe[i] = float32(math.Exp(-de));
		testWe[i] = float32(de / m2);
	}
	return;
}

// compareUint32Slices returns the first index where the two slices
// disagree, or <0 if the lengths are the same and all elements
// are identical.
func compareUint32Slices(s1, s2 []uint32) int {
	if len(s1) != len(s2) {
		if len(s1) > len(s2) {
			return len(s2) + 1
		}
		return len(s1) + 1;
	}
	for i := range s1 {
		if s1[i] != s2[i] {
			return i
		}
	}
	return -1;
}

// compareFloat32Slices returns the first index where the two slices
// disagree, or <0 if the lengths are the same and all elements
// are identical.
func compareFloat32Slices(s1, s2 []float32) int {
	if len(s1) != len(s2) {
		if len(s1) > len(s2) {
			return len(s2) + 1
		}
		return len(s1) + 1;
	}
	for i := range s1 {
		if !nearEqual(float64(s1[i]), float64(s2[i]), 0, 1e-7) {
			return i
		}
	}
	return -1;
}

func TestNormTables(t *testing.T) {
	testKn, testWn, testFn := initNorm();
	if i := compareUint32Slices(kn[0:], testKn); i >= 0 {
		t.Errorf("kn disagrees at index %v; %v != %v\n", i, kn[i], testKn[i])
	}
	if i := compareFloat32Slices(wn[0:], testWn); i >= 0 {
		t.Errorf("wn disagrees at index %v; %v != %v\n", i, wn[i], testWn[i])
	}
	if i := compareFloat32Slices(fn[0:], testFn); i >= 0 {
		t.Errorf("fn disagrees at index %v; %v != %v\n", i, fn[i], testFn[i])
	}
}

func TestExpTables(t *testing.T) {
	testKe, testWe, testFe := initExp();
	if i := compareUint32Slices(ke[0:], testKe); i >= 0 {
		t.Errorf("ke disagrees at index %v; %v != %v\n", i, ke[i], testKe[i])
	}
	if i := compareFloat32Slices(we[0:], testWe); i >= 0 {
		t.Errorf("we disagrees at index %v; %v != %v\n", i, we[i], testWe[i])
	}
	if i := compareFloat32Slices(fe[0:], testFe); i >= 0 {
		t.Errorf("fe disagrees at index %v; %v != %v\n", i, fe[i], testFe[i])
	}
}
<commit_msg>added benchmarks to rand_test.go; removed superfluous field name in lockedSource.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package rand

import (
	"math";
	"fmt";
	"os";
	"testing";
)

const (
	numTestSamples = 10000;
)

type statsResults struct {
	mean		float64;
	stddev		float64;
	closeEnough	float64;
	maxError	float64;
}

func max(a, b float64) float64 {
	if a > b {
		return a
	}
	return b;
}

func nearEqual(a, b, closeEnough, maxError float64) bool {
	absDiff := math.Fabs(a - b);
	if absDiff < closeEnough {	// Necessary when one value is zero and one value is close to zero.
		return true
	}
	return absDiff/max(math.Fabs(a), math.Fabs(b)) < maxError;
}

var testSeeds = []int64{1, 1754801282, 1698661970, 1550503961}

// checkSimilarDistribution returns success if the mean and stddev of the
// two statsResults are similar.
func (this *statsResults) checkSimilarDistribution(expected *statsResults) os.Error {
	if !nearEqual(this.mean, expected.mean, expected.closeEnough, expected.maxError) {
		s := fmt.Sprintf("mean %v != %v (allowed error %v, %v)", this.mean, expected.mean, expected.closeEnough, expected.maxError);
		fmt.Println(s);
		return os.ErrorString(s);
	}
	if !nearEqual(this.stddev, expected.stddev, 0, expected.maxError) {
		s := fmt.Sprintf("stddev %v != %v (allowed error %v, %v)", this.stddev, expected.stddev, expected.closeEnough, expected.maxError);
		fmt.Println(s);
		return os.ErrorString(s);
	}
	return nil;
}

func getStatsResults(samples []float64) *statsResults {
	res := new(statsResults);
	var sum float64;
	for i := range samples {
		sum += samples[i]
	}
	res.mean = sum / float64(len(samples));
	var devsum float64;
	for i := range samples {
		devsum += math.Pow(samples[i]-res.mean, 2)
	}
	res.stddev = math.Sqrt(devsum / float64(len(samples)));
	return res;
}

func checkSampleDistribution(t *testing.T, samples []float64, expected *statsResults) {
	actual := getStatsResults(samples);
	err := actual.checkSimilarDistribution(expected);
	if err != nil {
		t.Errorf(err.String())
	}
}

func checkSampleSliceDistributions(t *testing.T, samples []float64, nslices int, expected *statsResults) {
	chunk := len(samples) / nslices;
	for i := 0; i < nslices; i++ {
		low := i * chunk;
		var high int;
		if i == nslices-1 {
			high = len(samples) - 1
		} else {
			high = (i + 1) * chunk
		}
		checkSampleDistribution(t, samples[low:high], expected);
	}
}

//
// Normal distribution tests
//

func generateNormalSamples(nsamples int, mean, stddev float64, seed int64) []float64 {
	r := New(NewSource(seed));
	samples := make([]float64, nsamples);
	for i := range samples {
		samples[i] = r.NormFloat64()*stddev + mean
	}
	return samples;
}

func testNormalDistribution(t *testing.T, nsamples int, mean, stddev float64, seed int64) {
	//fmt.Printf("testing nsamples=%v mean=%v stddev=%v seed=%v\n", nsamples, mean, stddev, seed);

	samples := generateNormalSamples(nsamples, mean, stddev, seed);
	errorScale := max(1.0, stddev);	// Error scales with stddev
	expected := &statsResults{mean, stddev, 0.10 * errorScale, 0.08 * errorScale};

	// Make sure that the entire set matches the expected distribution.
	checkSampleDistribution(t, samples, expected);

	// Make sure that each half of the set matches the expected distribution.
	checkSampleSliceDistributions(t, samples, 2, expected);

	// Make sure that each 7th of the set matches the expected distribution.
	checkSampleSliceDistributions(t, samples, 7, expected);
}

// Actual tests

func TestStandardNormalValues(t *testing.T) {
	for _, seed := range testSeeds {
		testNormalDistribution(t, numTestSamples, 0, 1, seed)
	}
}

func TestNonStandardNormalValues(t *testing.T) {
	for sd := float64(0.5); sd < 1000; sd *= 2 {
		for m := float64(0.5); m < 1000; m *= 2 {
			for _, seed := range testSeeds {
				testNormalDistribution(t, numTestSamples, m, sd, seed)
			}
		}
	}
}

//
// Exponential distribution tests
//

func generateExponentialSamples(nsamples int, rate float64, seed int64) []float64 {
	r := New(NewSource(seed));
	samples := make([]float64, nsamples);
	for i := range samples {
		samples[i] = r.ExpFloat64() / rate
	}
	return samples;
}

func testExponentialDistribution(t *testing.T, nsamples int, rate float64, seed int64) {
	//fmt.Printf("testing nsamples=%v rate=%v seed=%v\n", nsamples, rate, seed);

	mean := 1 / rate;
	stddev := mean;

	samples := generateExponentialSamples(nsamples, rate, seed);
	errorScale := max(1.0, 1/rate);	// Error scales with the inverse of the rate
	expected := &statsResults{mean, stddev, 0.10 * errorScale, 0.20 * errorScale};

	// Make sure that the entire set matches the expected distribution.
	checkSampleDistribution(t, samples, expected);

	// Make sure that each half of the set matches the expected distribution.
	checkSampleSliceDistributions(t, samples, 2, expected);

	// Make sure that each 7th of the set matches the expected distribution.
	checkSampleSliceDistributions(t, samples, 7, expected);
}

// Actual tests

func TestStandardExponentialValues(t *testing.T) {
	for _, seed := range testSeeds {
		testExponentialDistribution(t, numTestSamples, 1, seed)
	}
}

func TestNonStandardExponentialValues(t *testing.T) {
	for rate := float64(0.05); rate < 10; rate *= 2 {
		for _, seed := range testSeeds {
			testExponentialDistribution(t, numTestSamples, rate, seed)
		}
	}
}

//
// Table generation tests
//

func initNorm() (testKn []uint32, testWn, testFn []float32) {
	const m1 = 1 << 31;
	var (
		dn	float64	= rn;
		tn		= dn;
		vn	float64	= 9.91256303526217e-3;
	)

	testKn = make([]uint32, 128);
	testWn = make([]float32, 128);
	testFn = make([]float32, 128);

	q := vn / math.Exp(-0.5*dn*dn);
	testKn[0] = uint32((dn / q) * m1);
	testKn[1] = 0;
	testWn[0] = float32(q / m1);
	testWn[127] = float32(dn / m1);
	testFn[0] = 1.0;
	testFn[127] = float32(math.Exp(-0.5 * dn * dn));
	for i := 126; i >= 1; i-- {
		dn = math.Sqrt(-2.0 * math.Log(vn/dn+math.Exp(-0.5*dn*dn)));
		testKn[i+1] = uint32((dn / tn) * m1);
		tn = dn;
		testFn[i] = float32(math.Exp(-0.5 * dn * dn));
		testWn[i] = float32(dn / m1);
	}
	return;
}

func initExp() (testKe []uint32, testWe, testFe []float32) {
	const m2 = 1 << 32;
	var (
		de	float64	= re;
		te		= de;
		ve	float64	= 3.9496598225815571993e-3;
	)

	testKe = make([]uint32, 256);
	testWe = make([]float32, 256);
	testFe = make([]float32, 256);

	q := ve / math.Exp(-de);
	testKe[0] = uint32((de / q) * m2);
	testKe[1] = 0;
	testWe[0] = float32(q / m2);
	testWe[255] = float32(de / m2);
	testFe[0] = 1.0;
	testFe[255] = float32(math.Exp(-de));
	for i := 254; i >= 1; i-- {
		de = -math.Log(ve/de + math.Exp(-de));
		testKe[i+1] = uint32((de / te) * m2);
		te = de;
		testFe[i] = float32(math.Exp(-de));
		testWe[i] = float32(de / m2);
	}
	return;
}

// compareUint32Slices returns the first index where the two slices
// disagree, or <0 if the lengths are the same and all elements
// are identical.
func compareUint32Slices(s1, s2 []uint32) int {
	if len(s1) != len(s2) {
		if len(s1) > len(s2) {
			return len(s2) + 1
		}
		return len(s1) + 1;
	}
	for i := range s1 {
		if s1[i] != s2[i] {
			return i
		}
	}
	return -1;
}

// compareFloat32Slices returns the first index where the two slices
// disagree, or <0 if the lengths are the same and all elements
// are identical.
func compareFloat32Slices(s1, s2 []float32) int {
	if len(s1) != len(s2) {
		if len(s1) > len(s2) {
			return len(s2) + 1
		}
		return len(s1) + 1;
	}
	for i := range s1 {
		if !nearEqual(float64(s1[i]), float64(s2[i]), 0, 1e-7) {
			return i
		}
	}
	return -1;
}

func TestNormTables(t *testing.T) {
	testKn, testWn, testFn := initNorm();
	if i := compareUint32Slices(kn[0:], testKn); i >= 0 {
		t.Errorf("kn disagrees at index %v; %v != %v\n", i, kn[i], testKn[i])
	}
	if i := compareFloat32Slices(wn[0:], testWn); i >= 0 {
		t.Errorf("wn disagrees at index %v; %v != %v\n", i, wn[i], testWn[i])
	}
	if i := compareFloat32Slices(fn[0:], testFn); i >= 0 {
		t.Errorf("fn disagrees at index %v; %v != %v\n", i, fn[i], testFn[i])
	}
}

func TestExpTables(t *testing.T) {
	testKe, testWe, testFe := initExp();
	if i := compareUint32Slices(ke[0:], testKe); i >= 0 {
		t.Errorf("ke disagrees at index %v; %v != %v\n", i, ke[i], testKe[i])
	}
	if i := compareFloat32Slices(we[0:], testWe); i >= 0 {
		t.Errorf("we disagrees at index %v; %v != %v\n", i, we[i], testWe[i])
	}
	if i := compareFloat32Slices(fe[0:], testFe); i >= 0 {
		t.Errorf("fe disagrees at index %v; %v != %v\n", i, fe[i], testFe[i])
	}
}

// Benchmarks

func BenchmarkInt63Threadsafe(b *testing.B) {
	for n := b.N; n > 0; n-- {
		Int63()
	}
}

func BenchmarkInt63Unthreadsafe(b *testing.B) {
	r := New(NewSource(1));
	for n := b.N; n > 0; n-- {
		r.Int63()
	}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package strings

import (
	"errors"
	"io"
	"unicode/utf8"
)

// A Reader implements the io.Reader, io.Seeker, io.ByteScanner, and
// io.RuneScanner interfaces by reading from a string.
type Reader struct {
	s        string
	i        int // current reading index
	prevRune int // index of previous rune; or < 0
}

// Len returns the number of bytes of the unread portion of the
// string.
func (r *Reader) Len() int {
	if r.i >= len(r.s) {
		return 0
	}
	return len(r.s) - r.i
}

func (r *Reader) Read(b []byte) (n int, err error) {
	if len(b) == 0 {
		return 0, nil
	}
	if r.i >= len(r.s) {
		return 0, io.EOF
	}
	n = copy(b, r.s[r.i:])
	r.i += n
	r.prevRune = -1
	return
}

func (r *Reader) ReadByte() (b byte, err error) {
	if r.i >= len(r.s) {
		return 0, io.EOF
	}
	b = r.s[r.i]
	r.i++
	r.prevRune = -1
	return
}

// UnreadByte moves the reading position back by one byte.
// It is an error to call UnreadByte if nothing has been
// read yet.
func (r *Reader) UnreadByte() error {
	if r.i <= 0 {
		return errors.New("strings.Reader: at beginning of string")
	}
	r.i--
	r.prevRune = -1
	return nil
}

// ReadRune reads and returns the next UTF-8-encoded
// Unicode code point from the buffer.
// If no bytes are available, the error returned is io.EOF.
// If the bytes are an erroneous UTF-8 encoding, it
// consumes one byte and returns U+FFFD, 1.
func (r *Reader) ReadRune() (ch rune, size int, err error) {
	if r.i >= len(r.s) {
		return 0, 0, io.EOF
	}
	r.prevRune = r.i
	if c := r.s[r.i]; c < utf8.RuneSelf {
		r.i++
		return rune(c), 1, nil
	}
	ch, size = utf8.DecodeRuneInString(r.s[r.i:])
	r.i += size
	return
}

// UnreadRune causes the next call to ReadRune to return the same rune
// as the previous call to ReadRune.
// The last method called on r must have been ReadRune.
func (r *Reader) UnreadRune() error {
	if r.prevRune < 0 {
		return errors.New("strings.Reader: previous operation was not ReadRune")
	}
	r.i = r.prevRune
	r.prevRune = -1
	return nil
}

// Seek implements the io.Seeker interface.
func (r *Reader) Seek(offset int64, whence int) (int64, error) {
	var abs int64
	switch whence {
	case 0:
		abs = offset
	case 1:
		abs = int64(r.i) + offset
	case 2:
		abs = int64(len(r.s)) + offset
	default:
		return 0, errors.New("strings: invalid whence")
	}
	if abs < 0 {
		return 0, errors.New("strings: negative position")
	}
	if abs >= 1<<31 {
		return 0, errors.New("strings: position out of range")
	}
	r.i = int(abs)
	return abs, nil
}

// NewReader returns a new Reader reading from s.
// It is similar to bytes.NewBufferString but more efficient and read-only.
func NewReader(s string) *Reader { return &Reader{s, 0, -1} }
<commit_msg>strings: delete method comments implied by interface satisfaction<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package strings

import (
	"errors"
	"io"
	"unicode/utf8"
)

// A Reader implements the io.Reader, io.Seeker, io.ByteScanner, and
// io.RuneScanner interfaces by reading from a string.
type Reader struct {
	s        string
	i        int // current reading index
	prevRune int // index of previous rune; or < 0
}

// Len returns the number of bytes of the unread portion of the
// string.
func (r *Reader) Len() int {
	if r.i >= len(r.s) {
		return 0
	}
	return len(r.s) - r.i
}

func (r *Reader) Read(b []byte) (n int, err error) {
	if len(b) == 0 {
		return 0, nil
	}
	if r.i >= len(r.s) {
		return 0, io.EOF
	}
	n = copy(b, r.s[r.i:])
	r.i += n
	r.prevRune = -1
	return
}

func (r *Reader) ReadByte() (b byte, err error) {
	if r.i >= len(r.s) {
		return 0, io.EOF
	}
	b = r.s[r.i]
	r.i++
	r.prevRune = -1
	return
}

func (r *Reader) UnreadByte() error {
	if r.i <= 0 {
		return errors.New("strings.Reader: at beginning of string")
	}
	r.i--
	r.prevRune = -1
	return nil
}

func (r *Reader) ReadRune() (ch rune, size int, err error) {
	if r.i >= len(r.s) {
		return 0, 0, io.EOF
	}
	r.prevRune = r.i
	if c := r.s[r.i]; c < utf8.RuneSelf {
		r.i++
		return rune(c), 1, nil
	}
	ch, size = utf8.DecodeRuneInString(r.s[r.i:])
	r.i += size
	return
}

func (r *Reader) UnreadRune() error {
	if r.prevRune < 0 {
		return errors.New("strings.Reader: previous operation was not ReadRune")
	}
	r.i = r.prevRune
	r.prevRune = -1
	return nil
}

// Seek implements the io.Seeker interface.
func (r *Reader) Seek(offset int64, whence int) (int64, error) {
	var abs int64
	switch whence {
	case 0:
		abs = offset
	case 1:
		abs = int64(r.i) + offset
	case 2:
		abs = int64(len(r.s)) + offset
	default:
		return 0, errors.New("strings: invalid whence")
	}
	if abs < 0 {
		return 0, errors.New("strings: negative position")
	}
	if abs >= 1<<31 {
		return 0, errors.New("strings: position out of range")
	}
	r.i = int(abs)
	return abs, nil
}

// NewReader returns a new Reader reading from s.
// It is similar to bytes.NewBufferString but more efficient and read-only.
func NewReader(s string) *Reader { return &Reader{s, 0, -1} }
<|endoftext|>


<commit_before>package moon

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"strings"
)

const ()

var nodes = map[tokenType]func(p *parser) node{
	t_string:       func(p *parser) node { return new(stringNode) },
	t_real_number:  func(p *parser) node { return new(numberNode) },
	t_list_start:   func(p *parser) node { p.next(); return &listNode{} },
	t_object_start: func(p *parser) node { p.next(); return &objectNode{} },
	t_variable:     func(p *parser) node { return new(variableNode) },
	t_bool:         func(p *parser) node { return new(boolNode) },
	t_duration:     func(p *parser) node { return new(durationNode) },
}

var DefaultPath = "./config.moon"

func bail(status int, t string, args ...interface{}) {
	if status == 0 {
		fmt.Fprintf(os.Stdout, t+"\n", args...)
	} else {
		fmt.Fprintf(os.Stderr, t+"\n", args...)
	}
	os.Exit(status)
}

func Parse(dest interface{}) {
	cliArgs, err := parseArgs(os.Args, dest)
	if err != nil {
		bail(1, "unable to parse cli args: %s", err)
	}
	f, err := os.Open(DefaultPath)
	if err != nil {
		bail(1, "unable to open moon config file at path %s: %s", DefaultPath, err)
	}
	defer f.Close()

	doc, err := Read(f)
	if err != nil {
		bail(1, "unable to parse moon config file at path %s: %s", DefaultPath, err)
	}

	for k, v := range cliArgs {
		doc.items[k] = v
	}

	if err := doc.Fill(dest); err != nil {
		bail(1, "unable to fill moon config values: %s", err)
	}
}

// Reads a moon document from a given io.Reader. The io.Reader is advanced to
// EOF. The reader is not closed after reading, since it's an io.Reader and not
// an io.ReadCloser. In the event of error, the state that the source reader
// will be left in is undefined.
func Read(r io.Reader) (*Doc, error) {
	tree, err := parse(r)
	if err != nil {
		return nil, err
	}
	ctx := newContext()
	if _, err := tree.eval(ctx); err != nil {
		return nil, fmt.Errorf("eval error: %s\n", err)
	}
	return &Doc{items: ctx.public}, nil
}

// Reads a moon document from a string. This is purely a convenience method;
// all it does is create a buffer and call the moon.Read function.
func ReadString(source string) (*Doc, error) {
	return Read(strings.NewReader(source))
}

// Reads a moon document from a slice of bytes. This is purely a concenience
// method; like ReadString, it simply creates a buffer and calls moon.Read
func ReadBytes(b []byte) (*Doc, error) {
	return Read(bytes.NewBuffer(b))
}

func ReadFile(path string) (*Doc, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	return Read(f)
}

func parse(r io.Reader) (node, error) {
	p := &parser{
		root:   newRootNode(),
		input:  lex(r),
		backup: make([]token, 0, 8),
	}
	if err := p.parse(); err != nil {
		return nil, err
	}
	return p.root, nil
}

// parser (little p) is an actual parser.  It actually does the parsing of a
// moon document.
type parser struct {
	root   node
	input  chan token
	backup []token
}

func (p *parser) parse() error {
	if p.root == nil {
		p.root = newRootNode()
	}
	return p.root.parse(p)
}

// returns the next token and advances the input stream
func (p *parser) next() token {
	if len(p.backup) > 0 {
		t := p.backup[len(p.backup)-1]
		p.backup = p.backup[:len(p.backup)-1]
		return t
	}
SKIP_COMMENTS:
	t, ok := <-p.input
	if !ok {
		return token{t_eof, "eof"}
	}
	if t.t == t_comment {
		goto SKIP_COMMENTS
	}
	return t
}

func (p *parser) peek() token {
	t := p.next()
	p.unread(t)
	return t
}

func (p *parser) unread(t token) {
	if p.backup == nil {
		p.backup = make([]token, 0, 8)
	}
	p.backup = append(p.backup, t)
}

func (p *parser) ensureNext(tt tokenType, context string) error {
	if p.peek().t != tt {
		return fmt.Errorf("unexpected %v in %s: expected %v", p.peek().t, context, tt)
	}
	return nil
}

// parse the next value.  This is to be executed in a context where we know we
// want something that is a value to come next, such as after an equals sign.
func (p *parser) parseValue() (node, error) {
	for {
		t := p.peek()
		switch t.t {
		case t_error:
			return nil, fmt.Errorf("parse error: saw lex error when looking for value: %v", t.s)
		case t_eof:
			return nil, fmt.Errorf("parse error: unexpected eof when looking for value")
		}

		fn, ok := nodes[t.t]
		if !ok {
			return nil, fmt.Errorf("parse error: unexpected %v token while looking for value", t.t)
		}
		n := fn(p)
		if err := n.parse(p); err != nil {
			return nil, err
		}
		return n, nil
	}
}
<commit_msg>don't fail if default config doesn't exist<commit_after>package moon

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"strings"
)

const ()

var nodes = map[tokenType]func(p *parser) node{
	t_string:       func(p *parser) node { return new(stringNode) },
	t_real_number:  func(p *parser) node { return new(numberNode) },
	t_list_start:   func(p *parser) node { p.next(); return &listNode{} },
	t_object_start: func(p *parser) node { p.next(); return &objectNode{} },
	t_variable:     func(p *parser) node { return new(variableNode) },
	t_bool:         func(p *parser) node { return new(boolNode) },
	t_duration:     func(p *parser) node { return new(durationNode) },
}

var DefaultPath = "./config.moon"

func bail(status int, t string, args ...interface{}) {
	if status == 0 {
		fmt.Fprintf(os.Stdout, t+"\n", args...)
	} else {
		fmt.Fprintf(os.Stderr, t+"\n", args...)
	}
	os.Exit(status)
}

func Parse(dest interface{}) {
	cliArgs, err := parseArgs(os.Args, dest)
	if err != nil {
		bail(1, "unable to parse cli args: %s", err)
	}

    var doc *Doc

	f, err := os.Open(DefaultPath)
	if err == nil {
        defer f.Close()
        d, err := Read(f)
        if err != nil {
            bail(1, "unable to parse moon config file at path %s: %s", DefaultPath, err)
        }
        doc = d
	} else {
        doc = &Doc{items: make(map[string]interface{})}
    }

	for k, v := range cliArgs {
		doc.items[k] = v
	}

	if err := doc.Fill(dest); err != nil {
		bail(1, "unable to fill moon config values: %s", err)
	}
}

// Reads a moon document from a given io.Reader. The io.Reader is advanced to
// EOF. The reader is not closed after reading, since it's an io.Reader and not
// an io.ReadCloser. In the event of error, the state that the source reader
// will be left in is undefined.
func Read(r io.Reader) (*Doc, error) {
	tree, err := parse(r)
	if err != nil {
		return nil, err
	}
	ctx := newContext()
	if _, err := tree.eval(ctx); err != nil {
		return nil, fmt.Errorf("eval error: %s\n", err)
	}
	return &Doc{items: ctx.public}, nil
}

// Reads a moon document from a string. This is purely a convenience method;
// all it does is create a buffer and call the moon.Read function.
func ReadString(source string) (*Doc, error) {
	return Read(strings.NewReader(source))
}

// Reads a moon document from a slice of bytes. This is purely a concenience
// method; like ReadString, it simply creates a buffer and calls moon.Read
func ReadBytes(b []byte) (*Doc, error) {
	return Read(bytes.NewBuffer(b))
}

func ReadFile(path string) (*Doc, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	return Read(f)
}

func parse(r io.Reader) (node, error) {
	p := &parser{
		root:   newRootNode(),
		input:  lex(r),
		backup: make([]token, 0, 8),
	}
	if err := p.parse(); err != nil {
		return nil, err
	}
	return p.root, nil
}

// parser (little p) is an actual parser.  It actually does the parsing of a
// moon document.
type parser struct {
	root   node
	input  chan token
	backup []token
}

func (p *parser) parse() error {
	if p.root == nil {
		p.root = newRootNode()
	}
	return p.root.parse(p)
}

// returns the next token and advances the input stream
func (p *parser) next() token {
	if len(p.backup) > 0 {
		t := p.backup[len(p.backup)-1]
		p.backup = p.backup[:len(p.backup)-1]
		return t
	}
SKIP_COMMENTS:
	t, ok := <-p.input
	if !ok {
		return token{t_eof, "eof"}
	}
	if t.t == t_comment {
		goto SKIP_COMMENTS
	}
	return t
}

func (p *parser) peek() token {
	t := p.next()
	p.unread(t)
	return t
}

func (p *parser) unread(t token) {
	if p.backup == nil {
		p.backup = make([]token, 0, 8)
	}
	p.backup = append(p.backup, t)
}

func (p *parser) ensureNext(tt tokenType, context string) error {
	if p.peek().t != tt {
		return fmt.Errorf("unexpected %v in %s: expected %v", p.peek().t, context, tt)
	}
	return nil
}

// parse the next value.  This is to be executed in a context where we know we
// want something that is a value to come next, such as after an equals sign.
func (p *parser) parseValue() (node, error) {
	for {
		t := p.peek()
		switch t.t {
		case t_error:
			return nil, fmt.Errorf("parse error: saw lex error when looking for value: %v", t.s)
		case t_eof:
			return nil, fmt.Errorf("parse error: unexpected eof when looking for value")
		}

		fn, ok := nodes[t.t]
		if !ok {
			return nil, fmt.Errorf("parse error: unexpected %v token while looking for value", t.t)
		}
		n := fn(p)
		if err := n.parse(p); err != nil {
			return nil, err
		}
		return n, nil
	}
}
<|endoftext|>


<commit_before>package cli

import (
	"context"
	"fmt"
	"path/filepath"
	"strings"
	"time"

	"github.com/kopia/kopia/fs"
	"github.com/kopia/kopia/internal/units"
	"github.com/kopia/kopia/snapshot"
	"github.com/kopia/kopia/snapshot/policy"
	"github.com/kopia/kopia/snapshot/snapshotfs"
	"github.com/kopia/repo"
	"github.com/kopia/repo/object"
)

var (
	snapshotListCommand              = snapshotCommands.Command("list", "List snapshots of files and directories.").Alias("ls")
	snapshotListPath                 = snapshotListCommand.Arg("source", "File or directory to show history of.").String()
	snapshotListIncludeIncomplete    = snapshotListCommand.Flag("incomplete", "Include incomplete.").Short('i').Bool()
	snapshotListShowHumanReadable    = snapshotListCommand.Flag("human-readable", "Show human-readable units").Default("true").Bool()
	snapshotListShowDelta            = snapshotListCommand.Flag("delta", "Include deltas.").Short('d').Bool()
	snapshotListShowItemID           = snapshotListCommand.Flag("manifest-id", "Include manifest item ID.").Short('m').Bool()
	snapshotListShowHashCache        = snapshotListCommand.Flag("hashcache", "Include hashcache object ID.").Bool()
	snapshotListShowRetentionReasons = snapshotListCommand.Flag("retention", "Include retention reasons.").Default("true").Bool()
	snapshotListShowModTime          = snapshotListCommand.Flag("mtime", "Include file mod time").Bool()
	shapshotListShowOwner            = snapshotListCommand.Flag("owner", "Include owner").Bool()
	snapshotListShowIdentical        = snapshotListCommand.Flag("show-identical", "Show identical snapshots").Short('l').Bool()
	snapshotListShowAll              = snapshotListCommand.Flag("all", "Show all shapshots (not just current username/host)").Short('a').Bool()
	maxResultsPerPath                = snapshotListCommand.Flag("max-results", "Maximum number of entries per source.").Default("100").Short('n').Int()
)

func findSnapshotsForSource(ctx context.Context, rep *repo.Repository, sourceInfo snapshot.SourceInfo) (manifestIDs []string, relPath string, err error) {
	for len(sourceInfo.Path) > 0 {
		list, err := snapshot.ListSnapshotManifests(ctx, rep, &sourceInfo)
		if err != nil {
			return nil, "", err
		}

		if len(list) > 0 {
			return list, relPath, nil
		}

		if len(relPath) > 0 {
			relPath = filepath.Base(sourceInfo.Path) + "/" + relPath
		} else {
			relPath = filepath.Base(sourceInfo.Path)
		}

		log.Debugf("No snapshots of %v@%v:%v", sourceInfo.UserName, sourceInfo.Host, sourceInfo.Path)

		parentPath := filepath.Dir(sourceInfo.Path)
		if parentPath == sourceInfo.Path {
			break
		}
		sourceInfo.Path = parentPath
	}

	return nil, "", nil
}

func findManifestIDs(ctx context.Context, rep *repo.Repository, source string) ([]string, string, error) {
	if source == "" {
		man, err := snapshot.ListSnapshotManifests(ctx, rep, nil)
		return man, "", err
	}

	si, err := snapshot.ParseSourceInfo(source, getHostName(), getUserName())
	if err != nil {
		return nil, "", fmt.Errorf("invalid directory: '%s': %s", source, err)
	}

	manifestIDs, relPath, err := findSnapshotsForSource(ctx, rep, si)
	if relPath != "" {
		relPath = "/" + relPath
	}

	return manifestIDs, relPath, err
}

func runSnapshotsCommand(ctx context.Context, rep *repo.Repository) error {
	manifestIDs, relPath, err := findManifestIDs(ctx, rep, *snapshotListPath)
	if err != nil {
		return err
	}

	manifests, err := snapshot.LoadSnapshots(ctx, rep, manifestIDs)
	if err != nil {
		return err
	}

	return outputManifestGroups(ctx, rep, manifests, strings.Split(relPath, "/"))
}

func shouldOutputSnapshotSource(src snapshot.SourceInfo) bool {
	if *snapshotListShowAll {
		return true
	}

	if src.Host != getHostName() {
		return false

	}

	return src.UserName == getUserName()
}

func outputManifestGroups(ctx context.Context, rep *repo.Repository, manifests []*snapshot.Manifest, relPathParts []string) error {
	separator := ""
	for _, snapshotGroup := range snapshot.GroupBySource(manifests) {
		src := snapshotGroup[0].Source
		if !shouldOutputSnapshotSource(src) {
			continue
		}
		fmt.Printf("%v%v\n", separator, src)
		separator = "\n"

		pol, _, err := policy.GetEffectivePolicy(ctx, rep, src)
		if err != nil {
			log.Warningf("unable to determine effective policy for %v", src)
		} else {
			pol.RetentionPolicy.ComputeRetentionReasons(snapshotGroup)
		}
		if err := outputManifestFromSingleSource(ctx, rep, snapshotGroup, relPathParts); err != nil {
			return err
		}
	}

	return nil
}

//nolint:gocyclo
func outputManifestFromSingleSource(ctx context.Context, rep *repo.Repository, manifests []*snapshot.Manifest, parts []string) error {
	var count int
	var lastTotalFileSize int64

	manifests = snapshot.SortByTime(manifests, false)
	if len(manifests) > *maxResultsPerPath {
		manifests = manifests[len(manifests)-*maxResultsPerPath:]
	}

	var previousOID object.ID
	var elidedCount int
	var maxElidedTime time.Time

	outputElided := func() {
		if elidedCount > 0 {
			fmt.Printf(
				"  + %v identical snapshots until %v\n",
				elidedCount,
				formatTimestamp(maxElidedTime),
			)
		}
	}

	for _, m := range manifests {
		root, err := snapshotfs.SnapshotRoot(rep, m)
		if err != nil {
			fmt.Printf("  %v <ERROR> %v\n", formatTimestamp(m.StartTime), err)
			continue
		}
		ent, err := getNestedEntry(ctx, root, parts)
		if err != nil {
			fmt.Printf("  %v <ERROR> %v\n", formatTimestamp(m.StartTime), err)
			continue
		}

		if _, ok := ent.(object.HasObjectID); !ok {
			log.Warningf("entry does not have object ID: %v", ent, err)
			continue
		}

		var bits []string
		if m.IncompleteReason != "" {
			if !*snapshotListIncludeIncomplete {
				continue
			}
			bits = append(bits, "incomplete:"+m.IncompleteReason)
		}

		bits = append(bits, maybeHumanReadableBytes(*snapshotListShowHumanReadable, ent.Size()))
		bits = append(bits, fmt.Sprintf("%v", ent.Mode()))
		if *shapshotListShowOwner {
			bits = append(bits, fmt.Sprintf("uid:%v", ent.Owner().UserID))
			bits = append(bits, fmt.Sprintf("gid:%v", ent.Owner().GroupID))
		}
		if *snapshotListShowModTime {
			bits = append(bits, fmt.Sprintf("modified:%v", formatTimestamp(ent.ModTime())))
		}

		if *snapshotListShowItemID {
			bits = append(bits, "manifest:"+m.ID)
		}
		if *snapshotListShowHashCache {
			bits = append(bits, "hashcache:"+m.HashCacheID.String())
		}

		if *snapshotListShowDelta {
			bits = append(bits, deltaBytes(ent.Size()-lastTotalFileSize))
		}

		if d, ok := ent.(fs.Directory); ok {
			s := d.Summary()
			if s != nil {
				bits = append(bits, fmt.Sprintf("files:%v", s.TotalFileCount))
				bits = append(bits, fmt.Sprintf("dirs:%v", s.TotalDirCount))
			}
		}

		if *snapshotListShowRetentionReasons {
			if len(m.RetentionReasons) > 0 {
				bits = append(bits, "("+strings.Join(m.RetentionReasons, ",")+")")
			}
		}

		oid := ent.(object.HasObjectID).ObjectID()
		if !*snapshotListShowIdentical && oid == previousOID {
			elidedCount++
			maxElidedTime = m.StartTime
			continue
		}

		previousOID = oid

		outputElided()
		elidedCount = 0

		fmt.Printf(
			"  %v %v %v\n",
			formatTimestamp(m.StartTime),
			oid,
			strings.Join(bits, " "),
		)

		count++
		if m.IncompleteReason == "" {
			lastTotalFileSize = m.Stats.TotalFileSize
		}
	}
	outputElided()

	return nil
}

func deltaBytes(b int64) string {
	if b > 0 {
		return "(+" + units.BytesStringBase10(b) + ")"
	}

	return ""
}

func init() {
	snapshotListCommand.Action(repositoryAction(runSnapshotsCommand))
}
<commit_msg>cli: added warning to 'snapshot list' when no snapshots are found<commit_after>package cli

import (
	"context"
	"fmt"
	"path/filepath"
	"strings"
	"time"

	"github.com/kopia/kopia/fs"
	"github.com/kopia/kopia/internal/units"
	"github.com/kopia/kopia/snapshot"
	"github.com/kopia/kopia/snapshot/policy"
	"github.com/kopia/kopia/snapshot/snapshotfs"
	"github.com/kopia/repo"
	"github.com/kopia/repo/object"
)

var (
	snapshotListCommand              = snapshotCommands.Command("list", "List snapshots of files and directories.").Alias("ls")
	snapshotListPath                 = snapshotListCommand.Arg("source", "File or directory to show history of.").String()
	snapshotListIncludeIncomplete    = snapshotListCommand.Flag("incomplete", "Include incomplete.").Short('i').Bool()
	snapshotListShowHumanReadable    = snapshotListCommand.Flag("human-readable", "Show human-readable units").Default("true").Bool()
	snapshotListShowDelta            = snapshotListCommand.Flag("delta", "Include deltas.").Short('d').Bool()
	snapshotListShowItemID           = snapshotListCommand.Flag("manifest-id", "Include manifest item ID.").Short('m').Bool()
	snapshotListShowHashCache        = snapshotListCommand.Flag("hashcache", "Include hashcache object ID.").Bool()
	snapshotListShowRetentionReasons = snapshotListCommand.Flag("retention", "Include retention reasons.").Default("true").Bool()
	snapshotListShowModTime          = snapshotListCommand.Flag("mtime", "Include file mod time").Bool()
	shapshotListShowOwner            = snapshotListCommand.Flag("owner", "Include owner").Bool()
	snapshotListShowIdentical        = snapshotListCommand.Flag("show-identical", "Show identical snapshots").Short('l').Bool()
	snapshotListShowAll              = snapshotListCommand.Flag("all", "Show all shapshots (not just current username/host)").Short('a').Bool()
	maxResultsPerPath                = snapshotListCommand.Flag("max-results", "Maximum number of entries per source.").Default("100").Short('n').Int()
)

func findSnapshotsForSource(ctx context.Context, rep *repo.Repository, sourceInfo snapshot.SourceInfo) (manifestIDs []string, relPath string, err error) {
	for len(sourceInfo.Path) > 0 {
		list, err := snapshot.ListSnapshotManifests(ctx, rep, &sourceInfo)
		if err != nil {
			return nil, "", err
		}

		if len(list) > 0 {
			return list, relPath, nil
		}

		if len(relPath) > 0 {
			relPath = filepath.Base(sourceInfo.Path) + "/" + relPath
		} else {
			relPath = filepath.Base(sourceInfo.Path)
		}

		log.Debugf("No snapshots of %v@%v:%v", sourceInfo.UserName, sourceInfo.Host, sourceInfo.Path)

		parentPath := filepath.Dir(sourceInfo.Path)
		if parentPath == sourceInfo.Path {
			break
		}
		sourceInfo.Path = parentPath
	}

	return nil, "", nil
}

func findManifestIDs(ctx context.Context, rep *repo.Repository, source string) ([]string, string, error) {
	if source == "" {
		man, err := snapshot.ListSnapshotManifests(ctx, rep, nil)
		return man, "", err
	}

	si, err := snapshot.ParseSourceInfo(source, getHostName(), getUserName())
	if err != nil {
		return nil, "", fmt.Errorf("invalid directory: '%s': %s", source, err)
	}

	manifestIDs, relPath, err := findSnapshotsForSource(ctx, rep, si)
	if relPath != "" {
		relPath = "/" + relPath
	}

	return manifestIDs, relPath, err
}

func runSnapshotsCommand(ctx context.Context, rep *repo.Repository) error {
	manifestIDs, relPath, err := findManifestIDs(ctx, rep, *snapshotListPath)
	if err != nil {
		return err
	}

	manifests, err := snapshot.LoadSnapshots(ctx, rep, manifestIDs)
	if err != nil {
		return err
	}

	return outputManifestGroups(ctx, rep, manifests, strings.Split(relPath, "/"))
}

func shouldOutputSnapshotSource(src snapshot.SourceInfo) bool {
	if *snapshotListShowAll {
		return true
	}

	if src.Host != getHostName() {
		return false

	}

	return src.UserName == getUserName()
}

func outputManifestGroups(ctx context.Context, rep *repo.Repository, manifests []*snapshot.Manifest, relPathParts []string) error {
	separator := ""
	var anyOutput bool
	for _, snapshotGroup := range snapshot.GroupBySource(manifests) {
		src := snapshotGroup[0].Source
		if !shouldOutputSnapshotSource(src) {
			log.Debugf("skipping %v", src)
			continue
		}
		fmt.Printf("%v%v\n", separator, src)
		separator = "\n"
		anyOutput = true

		pol, _, err := policy.GetEffectivePolicy(ctx, rep, src)
		if err != nil {
			log.Warningf("unable to determine effective policy for %v", src)
		} else {
			pol.RetentionPolicy.ComputeRetentionReasons(snapshotGroup)
		}
		if err := outputManifestFromSingleSource(ctx, rep, snapshotGroup, relPathParts); err != nil {
			return err
		}
	}

	if !anyOutput && !*snapshotListShowAll && len(manifests) > 0 {
		printStderr("No snapshots found. Pass --all to show snapshots from all users/hosts.\n")
	}

	return nil
}

//nolint:gocyclo
func outputManifestFromSingleSource(ctx context.Context, rep *repo.Repository, manifests []*snapshot.Manifest, parts []string) error {
	var count int
	var lastTotalFileSize int64

	manifests = snapshot.SortByTime(manifests, false)
	if len(manifests) > *maxResultsPerPath {
		manifests = manifests[len(manifests)-*maxResultsPerPath:]
	}

	var previousOID object.ID
	var elidedCount int
	var maxElidedTime time.Time

	outputElided := func() {
		if elidedCount > 0 {
			fmt.Printf(
				"  + %v identical snapshots until %v\n",
				elidedCount,
				formatTimestamp(maxElidedTime),
			)
		}
	}

	for _, m := range manifests {
		root, err := snapshotfs.SnapshotRoot(rep, m)
		if err != nil {
			fmt.Printf("  %v <ERROR> %v\n", formatTimestamp(m.StartTime), err)
			continue
		}
		ent, err := getNestedEntry(ctx, root, parts)
		if err != nil {
			fmt.Printf("  %v <ERROR> %v\n", formatTimestamp(m.StartTime), err)
			continue
		}

		if _, ok := ent.(object.HasObjectID); !ok {
			log.Warningf("entry does not have object ID: %v", ent, err)
			continue
		}

		var bits []string
		if m.IncompleteReason != "" {
			if !*snapshotListIncludeIncomplete {
				continue
			}
			bits = append(bits, "incomplete:"+m.IncompleteReason)
		}

		bits = append(bits, maybeHumanReadableBytes(*snapshotListShowHumanReadable, ent.Size()))
		bits = append(bits, fmt.Sprintf("%v", ent.Mode()))
		if *shapshotListShowOwner {
			bits = append(bits, fmt.Sprintf("uid:%v", ent.Owner().UserID))
			bits = append(bits, fmt.Sprintf("gid:%v", ent.Owner().GroupID))
		}
		if *snapshotListShowModTime {
			bits = append(bits, fmt.Sprintf("modified:%v", formatTimestamp(ent.ModTime())))
		}

		if *snapshotListShowItemID {
			bits = append(bits, "manifest:"+m.ID)
		}
		if *snapshotListShowHashCache {
			bits = append(bits, "hashcache:"+m.HashCacheID.String())
		}

		if *snapshotListShowDelta {
			bits = append(bits, deltaBytes(ent.Size()-lastTotalFileSize))
		}

		if d, ok := ent.(fs.Directory); ok {
			s := d.Summary()
			if s != nil {
				bits = append(bits, fmt.Sprintf("files:%v", s.TotalFileCount))
				bits = append(bits, fmt.Sprintf("dirs:%v", s.TotalDirCount))
			}
		}

		if *snapshotListShowRetentionReasons {
			if len(m.RetentionReasons) > 0 {
				bits = append(bits, "("+strings.Join(m.RetentionReasons, ",")+")")
			}
		}

		oid := ent.(object.HasObjectID).ObjectID()
		if !*snapshotListShowIdentical && oid == previousOID {
			elidedCount++
			maxElidedTime = m.StartTime
			continue
		}

		previousOID = oid

		outputElided()
		elidedCount = 0

		fmt.Printf(
			"  %v %v %v\n",
			formatTimestamp(m.StartTime),
			oid,
			strings.Join(bits, " "),
		)

		count++
		if m.IncompleteReason == "" {
			lastTotalFileSize = m.Stats.TotalFileSize
		}
	}
	outputElided()

	return nil
}

func deltaBytes(b int64) string {
	if b > 0 {
		return "(+" + units.BytesStringBase10(b) + ")"
	}

	return ""
}

func init() {
	snapshotListCommand.Action(repositoryAction(runSnapshotsCommand))
}
<|endoftext|>


<commit_before>package defaul

import (
	"github.com/clawio/clawiod/config"
)

// DefaultDirectives represents the default configuration used by Server. This default configuration
// must work out-of-the-box without using user supplied config files.
var DefaultDirectives = config.Directives{
	Server: config.Server{
		CPU:                           "100%",
		BaseURL:                       "/api/v1/",
		Port:                          1502,
		JWTSecret:                     "you must change me",
		JWTSigningMethod:              "HS256",
		AppLog:                        "stdout",
		AppLogLevel:                   "info",
		AppLogMaxSize:                 100, // MiB
		HTTPAccessLog:                 "stdout",
		HTTPAccessLogLevel:            "info",
		HTTPAccessLogMaxSize:          100, // MiB
		ShutdownTimeout:               10,
		EnabledServices:               []string{"authentication", "metadata", "data", "webdav", "ocwebdav"},
		CORSEnabled:                   true,
		CORSAccessControlAllowOrigin:  []string{},
		CORSAccessControlAllowMethods: []string{"GET", "POST", "HEAD", "PUT", "DELETE"},
		CORSAccessControlAllowHeaders: []string{"*"},
		CORSEnabledServices:           []string{"authentication", "metadata", "data"},
	},

	Authentication: config.Authentication{
		BaseURL: "/authentication/",
		Type:    "memory",

		Memory: config.AuthenticationMemory{
			Users: getDefaultMemoryUsers(),
		},

		SQL: config.AuthenticationSQL{
			Driver: "mysql",
			DSN:    "root:passwd@tcp(localhost:3306)/users",
		},
	},

	MetaData: config.MetaData{
		BaseURL: "/metadata/",
		Type:    "simple",

		Simple: config.MetaDataSimple{
			Namespace:          "/tmp/clawio-namespace",
			TemporaryNamespace: "/tmp/clawio-temporary-namespace",
		},

		OCSQL: config.MetaDataOCSQL{
			Namespace:                   "/tmp/clawio-namespace",
			TemporaryNamespace:          "/tmp/clawio-temporary-namespace",
			MaxSQLIdleConnections:       1024,
			MaxSQLConcurrentConnections: 1024,
			SQLLog:        "stdout",
			SQLLogEnabled: true,
			SQLLogMaxSize: 100, // MiB
		},
	},

	Data: config.Data{
		BaseURL:           "/data/",
		Type:              "simple",
		UploadMaxFileSize: 8589934592, // 8 GiB

		Simple: config.DataSimple{
			Namespace:          "/tmp/clawio-namespace",
			TemporaryNamespace: "/tmp/clawio-temporary-namespace",
		},

		OCSQL: config.DataOCSQL{
			Namespace:          "/tmp/clawio-namespace",
			TemporaryNamespace: "/tmp/clawio-temporary-namespace",
		},
	},

	WebDAV: config.WebDAV{
		BaseURL:            "/webdav/",
		UploadMaxFileSize:  8589934592, // 8 GiB
		DataController:     "simple",
		MetaDataController: "simple",
	},

	OCWebDAV: config.OCWebDAV{
		BaseURL:                  "/ocwebdav/",
		UploadMaxFileSize:        8589934592, // 8 GiB
		DataController:           "ocsql",
		MetaDataController:       "ocsql",
		ChunksNamespace:          "/tmp/clawio-oc-chunks-namespace",
		ChunksTemporaryNamespace: "/tmp/clawio-oc-chunks-temporary-namespace",
	},
}

type conf struct{}

// New returns a source that always loads the default configuration.
func New() config.Source {
	return &conf{}
}

// LoadDirectives returns the configuration directives.
func (c *conf) LoadDirectives() (*config.Directives, error) {
	return &DefaultDirectives, nil
}
func getDefaultMemoryUsers() interface{} {
	users := []map[string]interface{}{}
	user := map[string]interface{}{}
	user["username"] = "demo"
	user["email"] = "demo@example.com"
	user["display_name"] = "Demo User"
	user["password"] = "demo"
	users = append(users, user)
	return users
}
<commit_msg>OCWebDAV API is not enabled by default<commit_after>package defaul

import (
	"github.com/clawio/clawiod/config"
)

// DefaultDirectives represents the default configuration used by Server. This default configuration
// must work out-of-the-box without using user supplied config files.
var DefaultDirectives = config.Directives{
	Server: config.Server{
		CPU:                           "100%",
		BaseURL:                       "/api/v1/",
		Port:                          1502,
		JWTSecret:                     "you must change me",
		JWTSigningMethod:              "HS256",
		AppLog:                        "stdout",
		AppLogLevel:                   "info",
		AppLogMaxSize:                 100, // MiB
		HTTPAccessLog:                 "stdout",
		HTTPAccessLogLevel:            "info",
		HTTPAccessLogMaxSize:          100, // MiB
		ShutdownTimeout:               10,
		EnabledServices:               []string{"authentication", "metadata", "data", "webdav"},
		CORSEnabled:                   true,
		CORSAccessControlAllowOrigin:  []string{},
		CORSAccessControlAllowMethods: []string{"GET", "POST", "HEAD", "PUT", "DELETE"},
		CORSAccessControlAllowHeaders: []string{"*"},
		CORSEnabledServices:           []string{"authentication", "metadata", "data"},
	},

	Authentication: config.Authentication{
		BaseURL: "/authentication/",
		Type:    "memory",

		Memory: config.AuthenticationMemory{
			Users: getDefaultMemoryUsers(),
		},

		SQL: config.AuthenticationSQL{
			Driver: "mysql",
			DSN:    "root:passwd@tcp(localhost:3306)/users",
		},
	},

	MetaData: config.MetaData{
		BaseURL: "/metadata/",
		Type:    "simple",

		Simple: config.MetaDataSimple{
			Namespace:          "/tmp/clawio-namespace",
			TemporaryNamespace: "/tmp/clawio-temporary-namespace",
		},

		OCSQL: config.MetaDataOCSQL{
			Namespace:                   "/tmp/clawio-namespace",
			TemporaryNamespace:          "/tmp/clawio-temporary-namespace",
			MaxSQLIdleConnections:       1024,
			MaxSQLConcurrentConnections: 1024,
			SQLLog:        "stdout",
			SQLLogEnabled: false,
			SQLLogMaxSize: 100, // MiB
		},
	},

	Data: config.Data{
		BaseURL:           "/data/",
		Type:              "simple",
		UploadMaxFileSize: 8589934592, // 8 GiB

		Simple: config.DataSimple{
			Namespace:          "/tmp/clawio-namespace",
			TemporaryNamespace: "/tmp/clawio-temporary-namespace",
		},

		OCSQL: config.DataOCSQL{
			Namespace:          "/tmp/clawio-namespace",
			TemporaryNamespace: "/tmp/clawio-temporary-namespace",
		},
	},

	WebDAV: config.WebDAV{
		BaseURL:            "/webdav/",
		UploadMaxFileSize:  8589934592, // 8 GiB
		DataController:     "simple",
		MetaDataController: "simple",
	},

	OCWebDAV: config.OCWebDAV{
		BaseURL:                  "/ocwebdav/",
		UploadMaxFileSize:        8589934592, // 8 GiB
		DataController:           "ocsql",
		MetaDataController:       "ocsql",
		ChunksNamespace:          "/tmp/clawio-oc-chunks-namespace",
		ChunksTemporaryNamespace: "/tmp/clawio-oc-chunks-temporary-namespace",
	},
}

type conf struct{}

// New returns a source that always loads the default configuration.
func New() config.Source {
	return &conf{}
}

// LoadDirectives returns the configuration directives.
func (c *conf) LoadDirectives() (*config.Directives, error) {
	return &DefaultDirectives, nil
}
func getDefaultMemoryUsers() interface{} {
	users := []map[string]interface{}{}
	user := map[string]interface{}{}
	user["username"] = "demo"
	user["email"] = "demo@example.com"
	user["display_name"] = "Demo User"
	user["password"] = "demo"
	users = append(users, user)
	return users
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package cbft

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"

	log "github.com/couchbaselabs/clog"
)

func docIDLookup(req *http.Request) string {
	return muxVariableLookup(req, "docID")
}

func indexNameLookup(req *http.Request) string {
	return muxVariableLookup(req, "indexName")
}

func pindexNameLookup(req *http.Request) string {
	return muxVariableLookup(req, "pindexName")
}

// ------------------------------------------------------------------

type ListIndexHandler struct {
	mgr *Manager
}

func NewListIndexHandler(mgr *Manager) *ListIndexHandler {
	return &ListIndexHandler{mgr: mgr}
}

func (h *ListIndexHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	indexDefs, _, err := h.mgr.GetIndexDefs(false)
	if err != nil {
		showError(w, req, "could not retrieve index defs", 500)
		return
	}

	rv := struct {
		Status    string     `json:"status"`
		IndexDefs *IndexDefs `json:"indexDefs"`
	}{
		Status:    "ok",
		IndexDefs: indexDefs,
	}
	mustEncode(w, rv)
}

// ---------------------------------------------------

type GetIndexHandler struct {
	mgr *Manager
}

func NewGetIndexHandler(mgr *Manager) *GetIndexHandler {
	return &GetIndexHandler{mgr: mgr}
}

func (h *GetIndexHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	indexName := indexNameLookup(req)
	if indexName == "" {
		showError(w, req, "index name is required", 400)
		return
	}

	_, indexDefsByName, err := h.mgr.GetIndexDefs(false)
	if err != nil {
		showError(w, req, "could not retrieve index defs", 500)
		return
	}

	indexDef, exists := indexDefsByName[indexName]
	if !exists || indexDef == nil {
		showError(w, req, "not an index", 400)
		return
	}

	indexUUID := req.FormValue("indexUUID")
	if indexUUID != "" && indexUUID != indexDef.UUID {
		showError(w, req, "wrong index UUID", 400)
		return
	}

	planPIndexes, planPIndexesByName, err := h.mgr.GetPlanPIndexes(false)
	if err != nil {
		showError(w, req, fmt.Sprintf("GetPlanPIndexes, err: %v", err), 400)
	}

	mustEncode(w, struct {
		Status       string        `json:"status"`
		IndexDef     *IndexDef     `json:"indexDef"`
		PlanPIndexes []*PlanPIndex `json:"planPIndexes"`
		Warnings     []string      `json:"warnings"`
	}{
		Status:       "ok",
		IndexDef:     indexDef,
		PlanPIndexes: planPIndexesByName[indexName],
		Warnings:     planPIndexes.Warnings[indexName],
	})
}

// ---------------------------------------------------

type CountHandler struct {
	mgr *Manager
}

func NewCountHandler(mgr *Manager) *CountHandler {
	return &CountHandler{mgr: mgr}
}

func (h *CountHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	indexName := indexNameLookup(req)
	if indexName == "" {
		showError(w, req, "index name is required", 400)
		return
	}

	indexUUID := req.FormValue("indexUUID")

	pindexImplType, err := PIndexImplTypeForIndex(h.mgr.Cfg(), indexName)
	if err != nil || pindexImplType.Count == nil {
		showError(w, req, fmt.Sprintf("rest.Count,"+
			" no pindexImplType, indexName: %s, err: %v", indexName, err), 400)
		return
	}

	count, err := pindexImplType.Count(h.mgr, indexName, indexUUID)
	if err != nil {
		showError(w, req, fmt.Sprintf("rest.Count,"+
			" indexName: %s, err: %v", indexName, err), 500)
		return
	}

	rv := struct {
		Status string `json:"status"`
		Count  uint64 `json:"count"`
	}{
		Status: "ok",
		Count:  count,
	}
	mustEncode(w, rv)
}

// ---------------------------------------------------

type QueryHandler struct {
	mgr *Manager
}

func NewQueryHandler(mgr *Manager) *QueryHandler {
	return &QueryHandler{mgr: mgr}
}

func (h *QueryHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	indexName := indexNameLookup(req)
	if indexName == "" {
		showError(w, req, "index name is required", 400)
		return
	}

	indexUUID := req.FormValue("indexUUID")

	requestBody, err := ioutil.ReadAll(req.Body)
	if err != nil {
		showError(w, req, fmt.Sprintf("rest.Query,"+
			" could not read request body, indexName: %s", indexName), 400)
		return
	}

	pindexImplType, err := PIndexImplTypeForIndex(h.mgr.Cfg(), indexName)
	if err != nil || pindexImplType.Query == nil {
		showError(w, req, fmt.Sprintf("rest.Query,"+
			" no pindexImplType, indexName: %s, err: %v", indexName, err), 400)
		return
	}

	log.Printf("rest.Query indexName: %s, requestBody: %s", indexName, requestBody)

	err = pindexImplType.Query(h.mgr, indexName, indexUUID, requestBody, w)
	if err != nil {
		if errCW, ok := err.(*ErrorConsistencyWait); ok {
			rv := struct {
				Status       string              `json:"status"`
				Message      string              `json:"message"`
				StartEndSeqs map[string][]uint64 `json:"startEndSeqs"`
			}{
				Status: errCW.Status,
				Message: fmt.Sprintf("rest.Query,"+
					" indexName: %s, requestBody: %s, req: %#v, err: %v",
					indexName, requestBody, req, err),
				StartEndSeqs: errCW.StartEndSeqs,
			}
			buf, err := json.Marshal(rv)
			if err == nil && buf != nil {
				showError(w, req, string(buf), 408)
				return
			}
		}

		showError(w, req, fmt.Sprintf("rest.Query,"+
			" indexName: %s, requestBody: %s, req: %#v, err: %v",
			indexName, requestBody, req, err), 400)
		return
	}

	log.Printf("rest.Query indexName: %s, DONE, requestBody: %s", indexName, requestBody)
}

// ---------------------------------------------------

type CountPIndexHandler struct {
	mgr *Manager
}

func NewCountPIndexHandler(mgr *Manager) *CountPIndexHandler {
	return &CountPIndexHandler{mgr: mgr}
}

func (h *CountPIndexHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	pindexName := pindexNameLookup(req)
	if pindexName == "" {
		showError(w, req, "pindex name is required", 400)
		return
	}

	pindex := h.mgr.GetPIndex(pindexName)
	if pindex == nil {
		showError(w, req, fmt.Sprintf("rest.CountPIndex,"+
			" no pindex, pindexName: %s", pindexName), 400)
		return
	}
	if pindex.Dest == nil {
		showError(w, req, fmt.Sprintf("rest.CountPIndex,"+
			" no pindex.Dest, pindexName: %s", pindexName), 400)
		return
	}

	pindexUUID := req.FormValue("pindexUUID")
	if pindexUUID != "" && pindex.UUID != pindexUUID {
		showError(w, req, fmt.Sprintf("rest.CountPIndex,"+
			" wrong pindexUUID: %s, pindex.UUID: %s, pindexName: %s",
			pindexUUID, pindex.UUID, pindexName), 400)
		return
	}

	var cancelCh chan string // TODO: Support request timeout and cancellation.

	log.Printf("rest.CountPIndex pindexName: %s", pindexName)

	count, err := pindex.Dest.Count(pindex, cancelCh)
	if err != nil {
		showError(w, req, fmt.Sprintf("rest.CountPIndex,"+
			" pindexName: %s, req: %#v, err: %v", pindexName, req, err), 400)
		return
	}

	rv := struct {
		Status string `json:"status"`
		Count  uint64 `json:"count"`
	}{
		Status: "ok",
		Count:  count,
	}
	mustEncode(w, rv)
}

// ---------------------------------------------------

type QueryPIndexHandler struct {
	mgr *Manager
}

func NewQueryPIndexHandler(mgr *Manager) *QueryPIndexHandler {
	return &QueryPIndexHandler{mgr: mgr}
}

func (h *QueryPIndexHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	pindexName := pindexNameLookup(req)
	if pindexName == "" {
		showError(w, req, "pindex name is required", 400)
		return
	}

	pindex := h.mgr.GetPIndex(pindexName)
	if pindex == nil {
		showError(w, req, fmt.Sprintf("rest.QueryPIndex,"+
			" no pindex, pindexName: %s", pindexName), 400)
		return
	}
	if pindex.Dest == nil {
		showError(w, req, fmt.Sprintf("rest.QueryPIndex,"+
			" no pindex.Dest, pindexName: %s", pindexName), 400)
		return
	}

	pindexUUID := req.FormValue("pindexUUID")
	if pindexUUID != "" && pindex.UUID != pindexUUID {
		showError(w, req, fmt.Sprintf("rest.QueryPIndex,"+
			" wrong pindexUUID: %s, pindex.UUID: %s, pindexName: %s",
			pindexUUID, pindex.UUID, pindexName), 400)
		return
	}

	requestBody, err := ioutil.ReadAll(req.Body)
	if err != nil {
		showError(w, req, fmt.Sprintf("rest.QueryPIndex,"+
			" could not read request body, pindexName: %s", pindexName), 400)
		return
	}

	var cancelCh chan string // TODO: Support request timeout and cancellation.

	log.Printf("rest.QueryPIndex pindexName: %s, requestBody: %s",
		pindexName, requestBody)

	err = pindex.Dest.Query(pindex, requestBody, w, cancelCh)
	if err != nil {
		if errCW, ok := err.(*ErrorConsistencyWait); ok {
			rv := struct {
				Status       string              `json:"status"`
				Message      string              `json:"message"`
				StartEndSeqs map[string][]uint64 `json:"startEndSeqs"`
			}{
				Status: errCW.Status,
				Message: fmt.Sprintf("rest.QueryPIndex,"+
					" pindexName: %s, requestBody: %s, req: %#v, err: %v",
					pindexName, requestBody, req, err),
				StartEndSeqs: errCW.StartEndSeqs,
			}
			buf, err := json.Marshal(rv)
			if err == nil && buf != nil {
				showError(w, req, string(buf), 408)
				return
			}
		}

		showError(w, req, fmt.Sprintf("rest.QueryPIndex,"+
			" pindexName: %s, requestBody: %s, req: %#v, err: %v",
			pindexName, requestBody, req, err), 400)
		return
	}

	log.Printf("rest.QueryPIndex pindexName: %s, DONE, requestBody: %s",
		pindexName, requestBody)
}
<commit_msg>split long lines<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package cbft

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"

	log "github.com/couchbaselabs/clog"
)

func docIDLookup(req *http.Request) string {
	return muxVariableLookup(req, "docID")
}

func indexNameLookup(req *http.Request) string {
	return muxVariableLookup(req, "indexName")
}

func pindexNameLookup(req *http.Request) string {
	return muxVariableLookup(req, "pindexName")
}

// ------------------------------------------------------------------

type ListIndexHandler struct {
	mgr *Manager
}

func NewListIndexHandler(mgr *Manager) *ListIndexHandler {
	return &ListIndexHandler{mgr: mgr}
}

func (h *ListIndexHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	indexDefs, _, err := h.mgr.GetIndexDefs(false)
	if err != nil {
		showError(w, req, "could not retrieve index defs", 500)
		return
	}

	rv := struct {
		Status    string     `json:"status"`
		IndexDefs *IndexDefs `json:"indexDefs"`
	}{
		Status:    "ok",
		IndexDefs: indexDefs,
	}
	mustEncode(w, rv)
}

// ---------------------------------------------------

type GetIndexHandler struct {
	mgr *Manager
}

func NewGetIndexHandler(mgr *Manager) *GetIndexHandler {
	return &GetIndexHandler{mgr: mgr}
}

func (h *GetIndexHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	indexName := indexNameLookup(req)
	if indexName == "" {
		showError(w, req, "index name is required", 400)
		return
	}

	_, indexDefsByName, err := h.mgr.GetIndexDefs(false)
	if err != nil {
		showError(w, req, "could not retrieve index defs", 500)
		return
	}

	indexDef, exists := indexDefsByName[indexName]
	if !exists || indexDef == nil {
		showError(w, req, "not an index", 400)
		return
	}

	indexUUID := req.FormValue("indexUUID")
	if indexUUID != "" && indexUUID != indexDef.UUID {
		showError(w, req, "wrong index UUID", 400)
		return
	}

	planPIndexes, planPIndexesByName, err := h.mgr.GetPlanPIndexes(false)
	if err != nil {
		showError(w, req, fmt.Sprintf("GetPlanPIndexes, err: %v", err), 400)
	}

	mustEncode(w, struct {
		Status       string        `json:"status"`
		IndexDef     *IndexDef     `json:"indexDef"`
		PlanPIndexes []*PlanPIndex `json:"planPIndexes"`
		Warnings     []string      `json:"warnings"`
	}{
		Status:       "ok",
		IndexDef:     indexDef,
		PlanPIndexes: planPIndexesByName[indexName],
		Warnings:     planPIndexes.Warnings[indexName],
	})
}

// ---------------------------------------------------

type CountHandler struct {
	mgr *Manager
}

func NewCountHandler(mgr *Manager) *CountHandler {
	return &CountHandler{mgr: mgr}
}

func (h *CountHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	indexName := indexNameLookup(req)
	if indexName == "" {
		showError(w, req, "index name is required", 400)
		return
	}

	indexUUID := req.FormValue("indexUUID")

	pindexImplType, err := PIndexImplTypeForIndex(h.mgr.Cfg(), indexName)
	if err != nil || pindexImplType.Count == nil {
		showError(w, req, fmt.Sprintf("rest.Count,"+
			" no pindexImplType, indexName: %s, err: %v", indexName, err), 400)
		return
	}

	count, err := pindexImplType.Count(h.mgr, indexName, indexUUID)
	if err != nil {
		showError(w, req, fmt.Sprintf("rest.Count,"+
			" indexName: %s, err: %v", indexName, err), 500)
		return
	}

	rv := struct {
		Status string `json:"status"`
		Count  uint64 `json:"count"`
	}{
		Status: "ok",
		Count:  count,
	}
	mustEncode(w, rv)
}

// ---------------------------------------------------

type QueryHandler struct {
	mgr *Manager
}

func NewQueryHandler(mgr *Manager) *QueryHandler {
	return &QueryHandler{mgr: mgr}
}

func (h *QueryHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	indexName := indexNameLookup(req)
	if indexName == "" {
		showError(w, req, "index name is required", 400)
		return
	}

	indexUUID := req.FormValue("indexUUID")

	requestBody, err := ioutil.ReadAll(req.Body)
	if err != nil {
		showError(w, req, fmt.Sprintf("rest.Query,"+
			" could not read request body, indexName: %s", indexName), 400)
		return
	}

	pindexImplType, err := PIndexImplTypeForIndex(h.mgr.Cfg(), indexName)
	if err != nil || pindexImplType.Query == nil {
		showError(w, req, fmt.Sprintf("rest.Query,"+
			" no pindexImplType, indexName: %s, err: %v", indexName, err), 400)
		return
	}

	log.Printf("rest.Query indexName: %s, requestBody: %s", indexName, requestBody)

	err = pindexImplType.Query(h.mgr, indexName, indexUUID, requestBody, w)
	if err != nil {
		if errCW, ok := err.(*ErrorConsistencyWait); ok {
			rv := struct {
				Status       string              `json:"status"`
				Message      string              `json:"message"`
				StartEndSeqs map[string][]uint64 `json:"startEndSeqs"`
			}{
				Status: errCW.Status,
				Message: fmt.Sprintf("rest.Query,"+
					" indexName: %s, requestBody: %s, req: %#v, err: %v",
					indexName, requestBody, req, err),
				StartEndSeqs: errCW.StartEndSeqs,
			}
			buf, err := json.Marshal(rv)
			if err == nil && buf != nil {
				showError(w, req, string(buf), 408)
				return
			}
		}

		showError(w, req, fmt.Sprintf("rest.Query,"+
			" indexName: %s, requestBody: %s, req: %#v, err: %v",
			indexName, requestBody, req, err), 400)
		return
	}

	log.Printf("rest.Query indexName: %s, DONE, requestBody: %s",
		indexName, requestBody)
}

// ---------------------------------------------------

type CountPIndexHandler struct {
	mgr *Manager
}

func NewCountPIndexHandler(mgr *Manager) *CountPIndexHandler {
	return &CountPIndexHandler{mgr: mgr}
}

func (h *CountPIndexHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	pindexName := pindexNameLookup(req)
	if pindexName == "" {
		showError(w, req, "pindex name is required", 400)
		return
	}

	pindex := h.mgr.GetPIndex(pindexName)
	if pindex == nil {
		showError(w, req, fmt.Sprintf("rest.CountPIndex,"+
			" no pindex, pindexName: %s", pindexName), 400)
		return
	}
	if pindex.Dest == nil {
		showError(w, req, fmt.Sprintf("rest.CountPIndex,"+
			" no pindex.Dest, pindexName: %s", pindexName), 400)
		return
	}

	pindexUUID := req.FormValue("pindexUUID")
	if pindexUUID != "" && pindex.UUID != pindexUUID {
		showError(w, req, fmt.Sprintf("rest.CountPIndex,"+
			" wrong pindexUUID: %s, pindex.UUID: %s, pindexName: %s",
			pindexUUID, pindex.UUID, pindexName), 400)
		return
	}

	var cancelCh chan string // TODO: Support request timeout and cancellation.

	log.Printf("rest.CountPIndex pindexName: %s", pindexName)

	count, err := pindex.Dest.Count(pindex, cancelCh)
	if err != nil {
		showError(w, req, fmt.Sprintf("rest.CountPIndex,"+
			" pindexName: %s, req: %#v, err: %v", pindexName, req, err), 400)
		return
	}

	rv := struct {
		Status string `json:"status"`
		Count  uint64 `json:"count"`
	}{
		Status: "ok",
		Count:  count,
	}
	mustEncode(w, rv)
}

// ---------------------------------------------------

type QueryPIndexHandler struct {
	mgr *Manager
}

func NewQueryPIndexHandler(mgr *Manager) *QueryPIndexHandler {
	return &QueryPIndexHandler{mgr: mgr}
}

func (h *QueryPIndexHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	pindexName := pindexNameLookup(req)
	if pindexName == "" {
		showError(w, req, "pindex name is required", 400)
		return
	}

	pindex := h.mgr.GetPIndex(pindexName)
	if pindex == nil {
		showError(w, req, fmt.Sprintf("rest.QueryPIndex,"+
			" no pindex, pindexName: %s", pindexName), 400)
		return
	}
	if pindex.Dest == nil {
		showError(w, req, fmt.Sprintf("rest.QueryPIndex,"+
			" no pindex.Dest, pindexName: %s", pindexName), 400)
		return
	}

	pindexUUID := req.FormValue("pindexUUID")
	if pindexUUID != "" && pindex.UUID != pindexUUID {
		showError(w, req, fmt.Sprintf("rest.QueryPIndex,"+
			" wrong pindexUUID: %s, pindex.UUID: %s, pindexName: %s",
			pindexUUID, pindex.UUID, pindexName), 400)
		return
	}

	requestBody, err := ioutil.ReadAll(req.Body)
	if err != nil {
		showError(w, req, fmt.Sprintf("rest.QueryPIndex,"+
			" could not read request body, pindexName: %s", pindexName), 400)
		return
	}

	var cancelCh chan string // TODO: Support request timeout and cancellation.

	log.Printf("rest.QueryPIndex pindexName: %s, requestBody: %s",
		pindexName, requestBody)

	err = pindex.Dest.Query(pindex, requestBody, w, cancelCh)
	if err != nil {
		if errCW, ok := err.(*ErrorConsistencyWait); ok {
			rv := struct {
				Status       string              `json:"status"`
				Message      string              `json:"message"`
				StartEndSeqs map[string][]uint64 `json:"startEndSeqs"`
			}{
				Status: errCW.Status,
				Message: fmt.Sprintf("rest.QueryPIndex,"+
					" pindexName: %s, requestBody: %s, req: %#v, err: %v",
					pindexName, requestBody, req, err),
				StartEndSeqs: errCW.StartEndSeqs,
			}
			buf, err := json.Marshal(rv)
			if err == nil && buf != nil {
				showError(w, req, string(buf), 408)
				return
			}
		}

		showError(w, req, fmt.Sprintf("rest.QueryPIndex,"+
			" pindexName: %s, requestBody: %s, req: %#v, err: %v",
			pindexName, requestBody, req, err), 400)
		return
	}

	log.Printf("rest.QueryPIndex pindexName: %s, DONE, requestBody: %s",
		pindexName, requestBody)
}
<|endoftext|>


<commit_before>package gps

import (
	"reflect"
	"testing"
)

func TestLockedProjectSorting(t *testing.T) {
	// version doesn't matter here
	lps := []LockedProject{
		NewLockedProject(mkPI("github.com/sdboyer/gps"), NewVersion("v0.10.0"), nil),
		NewLockedProject(mkPI("foo"), NewVersion("nada"), nil),
		NewLockedProject(mkPI("bar"), NewVersion("zip"), nil),
		NewLockedProject(mkPI("qux"), NewVersion("zilch"), nil),
	}
	lps2 := make([]LockedProject, len(lps))
	copy(lps2, lps)

	SortLockedProjects(lps2)

	// only the two should have switched positions
	lps[0], lps[2] = lps[2], lps[0]
	if !reflect.DeepEqual(lps, lps2) {
		t.Errorf("SortLockedProject did not sort as expected:\n\t(GOT) %s\n\t(WNT) %s", lps2, lps)
	}
}

func TestLockedProjectsEq(t *testing.T) {
	lps := []LockedProject{
		NewLockedProject(mkPI("github.com/sdboyer/gps"), NewVersion("v0.10.0"), []string{"gps"}),
		NewLockedProject(mkPI("github.com/sdboyer/gps"), NewVersion("v0.10.0"), nil),
		NewLockedProject(mkPI("github.com/sdboyer/gps"), NewVersion("v0.10.0"), []string{"gps", "flugle"}),
		NewLockedProject(mkPI("foo"), NewVersion("nada"), []string{"foo"}),
		NewLockedProject(mkPI("github.com/sdboyer/gps"), NewVersion("v0.10.0"), []string{"flugle", "gps"}),
	}

	if !lps[0].Eq(lps[0]) {
		t.Error("lp does not eq self")
	}

	if lps[0].Eq(lps[1]) {
		t.Error("lp should not eq when other pkg list is empty")
	}
	if lps[1].Eq(lps[0]) {
		t.Fail()
	}

	if lps[0].Eq(lps[2]) {
		t.Error("lp should not eq when other pkg list is longer")
	}
	if lps[2].Eq(lps[0]) {
		t.Fail()
	}

	if lps[1].Eq(lps[2]) {
		t.Fail()
	}
	if lps[2].Eq(lps[1]) {
		t.Fail()
	}

	if lps[2].Eq(lps[4]) {
		t.Error("should not eq if pkgs are out of order")
	}

	if lps[0].Eq(lps[3]) {
		t.Error("lp should not eq totally different lp")
	}
}
<commit_msg>Switch to table-based test<commit_after>package gps

import (
	"reflect"
	"testing"
)

func TestLockedProjectSorting(t *testing.T) {
	// version doesn't matter here
	lps := []LockedProject{
		NewLockedProject(mkPI("github.com/sdboyer/gps"), NewVersion("v0.10.0"), nil),
		NewLockedProject(mkPI("foo"), NewVersion("nada"), nil),
		NewLockedProject(mkPI("bar"), NewVersion("zip"), nil),
		NewLockedProject(mkPI("qux"), NewVersion("zilch"), nil),
	}
	lps2 := make([]LockedProject, len(lps))
	copy(lps2, lps)

	SortLockedProjects(lps2)

	// only the two should have switched positions
	lps[0], lps[2] = lps[2], lps[0]
	if !reflect.DeepEqual(lps, lps2) {
		t.Errorf("SortLockedProject did not sort as expected:\n\t(GOT) %s\n\t(WNT) %s", lps2, lps)
	}
}

func TestLockedProjectsEq(t *testing.T) {
	lps := []LockedProject{
		NewLockedProject(mkPI("github.com/sdboyer/gps"), NewVersion("v0.10.0"), []string{"gps"}),
		NewLockedProject(mkPI("github.com/sdboyer/gps"), NewVersion("v0.10.0"), nil),
		NewLockedProject(mkPI("github.com/sdboyer/gps"), NewVersion("v0.10.0"), []string{"gps", "flugle"}),
		NewLockedProject(mkPI("foo"), NewVersion("nada"), []string{"foo"}),
		NewLockedProject(mkPI("github.com/sdboyer/gps"), NewVersion("v0.10.0"), []string{"flugle", "gps"}),
		NewLockedProject(mkPI("github.com/sdboyer/gps"), NewVersion("v0.10.0").Is("278a227dfc3d595a33a77ff3f841fd8ca1bc8cd0"), []string{"gps"}),
	}

	fix := []struct {
		l1, l2   int
		shouldeq bool
		err      string
	}{
		{0, 0, true, "lp does not eq self"},
		{0, 5, false, "should not eq with different rev"},
		{0, 1, false, "should not eq when other pkg list is empty"},
		{0, 2, false, "should not eq when other pkg list is longer"},
		{0, 4, false, "should not eq when pkg lists are out of order"},
		{0, 3, false, "should not eq totally different lp"},
	}

	for _, f := range fix {
		if f.shouldeq {
			if !lps[f.l1].Eq(lps[f.l2]) {
				t.Error(f.err)
			}
			if !lps[f.l2].Eq(lps[f.l1]) {
				t.Error(f.err + (" (reversed)"))
			}
		} else {
			if lps[f.l1].Eq(lps[f.l2]) {
				t.Error(f.err)
			}
			if lps[f.l2].Eq(lps[f.l1]) {
				t.Error(f.err + (" (reversed)"))
			}

		}
	}
}
<|endoftext|>


<commit_before>package naive

import (
	"fmt"
	"github.com/sjwhitworth/golearn/base"
	"math"
)

// A Bernoulli Naive Bayes Classifier. Naive Bayes classifiers assumes
// that features probabilities are independent. In order to classify an
// instance, it is calculated the probability that it was generated by
// each known class, that is, for each class C, the following
// probability is calculated.
//
// p(C|F1, F2, F3... Fn)
//
// Being F1, F2... Fn the instance features. Using the bayes theorem
// this can be written as:
//
// \frac{p(C) \times p(F1, F2... Fn|C)}{p(F1, F2... Fn)}
//
// In the Bernoulli Naive Bayes features are considered independent
// booleans, this means that the likelihood of a document given a class
// C is given by:
//
// p(F1, F2... Fn) =
// \prod_{i=1}^{n}{[F_i \times p(f_i|C)) + (1-F_i)(1 - p(f_i|C)))]}
//
// where
//     - F_i equals to 1 if feature is present in vector and zero
//       otherwise
//     - p(f_i|C) the probability of class C generating the feature
//       f_i
//
// For more information:
//
// C.D. Manning, P. Raghavan and H. Schuetze (2008). Introduction to
// Information Retrieval. Cambridge University Press, pp. 234-265.
// http://nlp.stanford.edu/IR-book/html/htmledition/the-bernoulli-model-1.html
type BernoulliNBClassifier struct {
	base.BaseEstimator
	// Conditional probability for each term. This vector should be
	// accessed in the following way: p(f|c) = condProb[c][f].
	// Logarithm is used in order to avoid underflow.
	condProb map[string][]float64
	// Number of instances in each class. This is necessary in order to
	// calculate the laplace smooth value during the Predict step.
	classInstances map[string]int
	// Number of instances used in training.
	trainingInstances int
	// Number of features used in training
	features int
	// Attributes used to Train
	attrs []base.Attribute
	// Instance template
	fitOn base.FixedDataGrid
}

func (nb *BernoulliNBClassifier) GetMetadata() base.ClassifierMetadataV1 {
	return base.ClassifierMetadataV1{
		FormatVersion:      1,
		ClassifierName:     "KNN",
		ClassifierVersion:  "1.0",
		ClassifierMetadata: nil,
	}
}

func (nb *BernoulliNBClassifier) Save(filePath string) error {
	writer, err := base.CreateSerializedClassifierStub(filePath, nb.GetMetadata())
	if err != nil {
		return err
	}
	err = nb.SaveWithPrefix(writer, "")
	writer.Close()
	return err
}

func (nb *BernoulliNBClassifier) Load(filePath string) error {
	reader, err := base.ReadSerializedClassifierStub(filePath)
	if err != nil {
		return err
	}

	return nb.LoadWithPrefix(reader, "")
}

func (nb *BernoulliNBClassifier) LoadWithPrefix(reader *base.ClassifierDeserializer, prefix string) error {

	instances, err := reader.GetInstancesForKey(reader.Prefix(prefix, "INSTANCE_STRUCTURE"))
	if err != nil {
		return base.DescribeError("Unable to read INSTANCE_STRUCTURE", err)
	}

	rawAttrs, err := reader.GetAttributesForKey(reader.Prefix(prefix, "TRAINING_ATTRIBUTES"))
	if err != nil {
		return base.DescribeError("Unable to read training attributes", err)
	}
	attrs, err := base.ReplaceDeserializedAttributesWithVersionsFromInstances(rawAttrs, instances)
	if err != nil {
		return base.DescribeError("Unable to match up attributes", err)
	}

	numFeatures, err := reader.GetU64ForKey(reader.Prefix(prefix, "NUM_FEATURES"))
	if err != nil {
		return base.DescribeError("Unable to read training feature count", err)
	}
	numTrainingInstances, err := reader.GetU64ForKey(reader.Prefix(prefix, "NUM_TRAINING_INSTANCES"))
	if err != nil {
		return base.DescribeError("Unable to read training feature count", err)
	}

	// Save the class instances map
	condProbs := make(map[string][]float64)
	classInstances := make(map[string]int)

	err = reader.GetJSONForKey(reader.Prefix(prefix, "CLASS_INSTANCES"), &classInstances)
	if err != nil {
		return base.DescribeError("Unable to read the number of things in each class", err)
	}
	err = reader.GetJSONForKey(reader.Prefix(prefix, "COND_MAP"), &condProbs)
	if err != nil {
		return base.DescribeError("Unable to read the number of things in each class", err)
	}

	nb.fitOn = instances
	nb.attrs = attrs
	nb.features = int(numFeatures)
	nb.trainingInstances = int(numTrainingInstances)
	nb.classInstances = classInstances
	nb.condProb = condProbs
	return nil
}

func (nb *BernoulliNBClassifier) SaveWithPrefix(writer *base.ClassifierSerializer, prefix string) error {

	// Save the instance template
	err := writer.WriteInstancesForKey(writer.Prefix(prefix, "INSTANCE_STRUCTURE"), nb.fitOn, false)
	if err != nil {
		return base.DescribeError("Unable to write INSTANCE_STRUCTURE", err)
	}

	// Save the attributes used to train
	err = writer.WriteAttributesForKey(writer.Prefix(prefix, "TRAINING_ATTRIBUTES"), nb.attrs)
	if err != nil {
		return base.DescribeError("Unable to write training attributes", err)
	}

	// Save the number of features
	err = writer.WriteU64ForKey(writer.Prefix(prefix, "NUM_FEATURES"), uint64(nb.features))
	if err != nil {
		return base.DescribeError("Unable to write training feature count", err)
	}

	// Save the number of instances
	err = writer.WriteU64ForKey(writer.Prefix(prefix, "NUM_TRAINING_INSTANCES"), uint64(nb.trainingInstances))
	if err != nil {
		return base.DescribeError("Unable to write training feature count", err)
	}

	// Save the class instances map
	err = writer.WriteJSONForKey(writer.Prefix(prefix, "CLASS_INSTANCES"), nb.classInstances)
	if err != nil {
		return base.DescribeError("Unable to save the number of things in each class", err)
	}

	err = writer.WriteJSONForKey(writer.Prefix(prefix, "COND_MAP"), nb.condProb)
	if err != nil {
		return base.DescribeError("Unable to save conditional probability map", err)
	}
	return nil
}

// Create a new Bernoulli Naive Bayes Classifier. The argument 'classes'
// is the number of possible labels in the classification task.
func NewBernoulliNBClassifier() *BernoulliNBClassifier {
	nb := BernoulliNBClassifier{}
	nb.condProb = make(map[string][]float64)
	nb.features = 0
	nb.trainingInstances = 0
	return &nb
}

// Fill data matrix with Bernoulli Naive Bayes model. All values
// necessary for calculating prior probability and p(f_i)
func (nb *BernoulliNBClassifier) Fit(X base.FixedDataGrid) error {

	// Check that all Attributes are binary
	classAttrs := X.AllClassAttributes()
	allAttrs := X.AllAttributes()
	featAttrs := base.AttributeDifference(allAttrs, classAttrs)
	for i := range featAttrs {
		if _, ok := featAttrs[i].(*base.BinaryAttribute); !ok {
			return fmt.Errorf("%v: Should be BinaryAttribute", featAttrs[i])
		}
	}
	featAttrSpecs := base.ResolveAttributes(X, featAttrs)

	// Check that only one classAttribute is defined
	if len(classAttrs) != 1 {
		return fmt.Errorf("Only one class Attribute can be used")
	}

	// Number of features and instances in this training set
	_, nb.trainingInstances = X.Size()
	nb.attrs = featAttrs
	nb.features = len(featAttrs)

	// Number of instances in class
	nb.classInstances = make(map[string]int)

	// Number of documents with given term (by class)
	docsContainingTerm := make(map[string][]int)

	// This algorithm could be vectorized after binarizing the data
	// matrix. Since mat doesn't have this function, a iterative
	// version is used.
	X.MapOverRows(featAttrSpecs, func(docVector [][]byte, r int) (bool, error) {
		class := base.GetClass(X, r)

		// increment number of instances in class
		t, ok := nb.classInstances[class]
		if !ok {
			t = 0
		}
		nb.classInstances[class] = t + 1

		for feat := 0; feat < len(docVector); feat++ {
			v := docVector[feat]
			// In Bernoulli Naive Bayes the presence and absence of
			// features are considered. All non-zero values are
			// treated as presence.
			if v[0] > 0 {
				// Update number of times this feature appeared within
				// given label.
				t, ok := docsContainingTerm[class]
				if !ok {
					t = make([]int, nb.features)
					docsContainingTerm[class] = t
				}
				t[feat] += 1
			}
		}
		return true, nil
	})

	// Pre-calculate conditional probabilities for each class
	for c, _ := range nb.classInstances {
		nb.condProb[c] = make([]float64, nb.features)
		for feat := 0; feat < nb.features; feat++ {
			classTerms, _ := docsContainingTerm[c]
			numDocs := classTerms[feat]
			docsInClass, _ := nb.classInstances[c]

			classCondProb, _ := nb.condProb[c]
			// Calculate conditional probability with laplace smoothing
			classCondProb[feat] = float64(numDocs+1) / float64(docsInClass+1)
		}
	}

	nb.fitOn = base.NewStructuralCopy(X)
	return nil
}

// Use trained model to predict test vector's class. The following
// operation is used in order to score each class:
//
// classScore = log(p(c)) + \sum_{f}{log(p(f|c))}
//
// PredictOne returns the string that represents the predicted class.
//
// IMPORTANT: PredictOne panics if Fit was not called or if the
// document vector and train matrix have a different number of columns.
func (nb *BernoulliNBClassifier) PredictOne(vector [][]byte) string {
	if nb.features == 0 {
		panic("Fit should be called before predicting")
	}

	if len(vector) != nb.features {
		panic("Different dimensions in Train and Test sets")
	}

	// Currently only the predicted class is returned.
	bestScore := -math.MaxFloat64
	bestClass := ""

	for class, classCount := range nb.classInstances {
		// Init classScore with log(prior)
		classScore := math.Log((float64(classCount)) / float64(nb.trainingInstances))
		for f := 0; f < nb.features; f++ {
			if vector[f][0] > 0 {
				// Test document has feature c
				classScore += math.Log(nb.condProb[class][f])
			} else {
				if nb.condProb[class][f] == 1.0 {
					// special case when prob = 1.0, consider laplace
					// smooth
					classScore += math.Log(1.0 / float64(nb.classInstances[class]+1))
				} else {
					classScore += math.Log(1.0 - nb.condProb[class][f])
				}
			}
		}

		if classScore > bestScore {
			bestScore = classScore
			bestClass = class
		}
	}

	return bestClass
}

// Predict is just a wrapper for the PredictOne function.
//
// IMPORTANT: Predict panics if Fit was not called or if the
// document vector and train matrix have a different number of columns.
func (nb *BernoulliNBClassifier) Predict(what base.FixedDataGrid) (base.FixedDataGrid, error) {
	// Generate return vector
	ret := base.GeneratePredictionVector(what)

	// Get the features
	featAttrSpecs := base.ResolveAttributes(what, nb.attrs)

	what.MapOverRows(featAttrSpecs, func(row [][]byte, i int) (bool, error) {
		base.SetClass(ret, i, nb.PredictOne(row))
		return true, nil
	})

	return ret, nil
}
<commit_msg>Added String method to meet Classifier interface<commit_after>package naive

import (
	"fmt"
	"github.com/sjwhitworth/golearn/base"
	"math"
)

// A Bernoulli Naive Bayes Classifier. Naive Bayes classifiers assumes
// that features probabilities are independent. In order to classify an
// instance, it is calculated the probability that it was generated by
// each known class, that is, for each class C, the following
// probability is calculated.
//
// p(C|F1, F2, F3... Fn)
//
// Being F1, F2... Fn the instance features. Using the bayes theorem
// this can be written as:
//
// \frac{p(C) \times p(F1, F2... Fn|C)}{p(F1, F2... Fn)}
//
// In the Bernoulli Naive Bayes features are considered independent
// booleans, this means that the likelihood of a document given a class
// C is given by:
//
// p(F1, F2... Fn) =
// \prod_{i=1}^{n}{[F_i \times p(f_i|C)) + (1-F_i)(1 - p(f_i|C)))]}
//
// where
//     - F_i equals to 1 if feature is present in vector and zero
//       otherwise
//     - p(f_i|C) the probability of class C generating the feature
//       f_i
//
// For more information:
//
// C.D. Manning, P. Raghavan and H. Schuetze (2008). Introduction to
// Information Retrieval. Cambridge University Press, pp. 234-265.
// http://nlp.stanford.edu/IR-book/html/htmledition/the-bernoulli-model-1.html
type BernoulliNBClassifier struct {
	base.BaseEstimator
	// Conditional probability for each term. This vector should be
	// accessed in the following way: p(f|c) = condProb[c][f].
	// Logarithm is used in order to avoid underflow.
	condProb map[string][]float64
	// Number of instances in each class. This is necessary in order to
	// calculate the laplace smooth value during the Predict step.
	classInstances map[string]int
	// Number of instances used in training.
	trainingInstances int
	// Number of features used in training
	features int
	// Attributes used to Train
	attrs []base.Attribute
	// Instance template
	fitOn base.FixedDataGrid
}

func (nb *BernoulliNBClassifier) String() string {
	return "BernoulliNBBClassifier"
}

func (nb *BernoulliNBClassifier) GetMetadata() base.ClassifierMetadataV1 {
	return base.ClassifierMetadataV1{
		FormatVersion:      1,
		ClassifierName:     "KNN",
		ClassifierVersion:  "1.0",
		ClassifierMetadata: nil,
	}
}

func (nb *BernoulliNBClassifier) Save(filePath string) error {
	writer, err := base.CreateSerializedClassifierStub(filePath, nb.GetMetadata())
	if err != nil {
		return err
	}
	err = nb.SaveWithPrefix(writer, "")
	writer.Close()
	return err
}

func (nb *BernoulliNBClassifier) Load(filePath string) error {
	reader, err := base.ReadSerializedClassifierStub(filePath)
	if err != nil {
		return err
	}

	return nb.LoadWithPrefix(reader, "")
}

func (nb *BernoulliNBClassifier) LoadWithPrefix(reader *base.ClassifierDeserializer, prefix string) error {

	instances, err := reader.GetInstancesForKey(reader.Prefix(prefix, "INSTANCE_STRUCTURE"))
	if err != nil {
		return base.DescribeError("Unable to read INSTANCE_STRUCTURE", err)
	}

	rawAttrs, err := reader.GetAttributesForKey(reader.Prefix(prefix, "TRAINING_ATTRIBUTES"))
	if err != nil {
		return base.DescribeError("Unable to read training attributes", err)
	}
	attrs, err := base.ReplaceDeserializedAttributesWithVersionsFromInstances(rawAttrs, instances)
	if err != nil {
		return base.DescribeError("Unable to match up attributes", err)
	}

	numFeatures, err := reader.GetU64ForKey(reader.Prefix(prefix, "NUM_FEATURES"))
	if err != nil {
		return base.DescribeError("Unable to read training feature count", err)
	}
	numTrainingInstances, err := reader.GetU64ForKey(reader.Prefix(prefix, "NUM_TRAINING_INSTANCES"))
	if err != nil {
		return base.DescribeError("Unable to read training feature count", err)
	}

	// Save the class instances map
	condProbs := make(map[string][]float64)
	classInstances := make(map[string]int)

	err = reader.GetJSONForKey(reader.Prefix(prefix, "CLASS_INSTANCES"), &classInstances)
	if err != nil {
		return base.DescribeError("Unable to read the number of things in each class", err)
	}
	err = reader.GetJSONForKey(reader.Prefix(prefix, "COND_MAP"), &condProbs)
	if err != nil {
		return base.DescribeError("Unable to read the number of things in each class", err)
	}

	nb.fitOn = instances
	nb.attrs = attrs
	nb.features = int(numFeatures)
	nb.trainingInstances = int(numTrainingInstances)
	nb.classInstances = classInstances
	nb.condProb = condProbs
	return nil
}

func (nb *BernoulliNBClassifier) SaveWithPrefix(writer *base.ClassifierSerializer, prefix string) error {

	// Save the instance template
	err := writer.WriteInstancesForKey(writer.Prefix(prefix, "INSTANCE_STRUCTURE"), nb.fitOn, false)
	if err != nil {
		return base.DescribeError("Unable to write INSTANCE_STRUCTURE", err)
	}

	// Save the attributes used to train
	err = writer.WriteAttributesForKey(writer.Prefix(prefix, "TRAINING_ATTRIBUTES"), nb.attrs)
	if err != nil {
		return base.DescribeError("Unable to write training attributes", err)
	}

	// Save the number of features
	err = writer.WriteU64ForKey(writer.Prefix(prefix, "NUM_FEATURES"), uint64(nb.features))
	if err != nil {
		return base.DescribeError("Unable to write training feature count", err)
	}

	// Save the number of instances
	err = writer.WriteU64ForKey(writer.Prefix(prefix, "NUM_TRAINING_INSTANCES"), uint64(nb.trainingInstances))
	if err != nil {
		return base.DescribeError("Unable to write training feature count", err)
	}

	// Save the class instances map
	err = writer.WriteJSONForKey(writer.Prefix(prefix, "CLASS_INSTANCES"), nb.classInstances)
	if err != nil {
		return base.DescribeError("Unable to save the number of things in each class", err)
	}

	err = writer.WriteJSONForKey(writer.Prefix(prefix, "COND_MAP"), nb.condProb)
	if err != nil {
		return base.DescribeError("Unable to save conditional probability map", err)
	}
	return nil
}

// Create a new Bernoulli Naive Bayes Classifier. The argument 'classes'
// is the number of possible labels in the classification task.
func NewBernoulliNBClassifier() *BernoulliNBClassifier {
	nb := BernoulliNBClassifier{}
	nb.condProb = make(map[string][]float64)
	nb.features = 0
	nb.trainingInstances = 0
	return &nb
}

// Fill data matrix with Bernoulli Naive Bayes model. All values
// necessary for calculating prior probability and p(f_i)
func (nb *BernoulliNBClassifier) Fit(X base.FixedDataGrid) error {

	// Check that all Attributes are binary
	classAttrs := X.AllClassAttributes()
	allAttrs := X.AllAttributes()
	featAttrs := base.AttributeDifference(allAttrs, classAttrs)
	for i := range featAttrs {
		if _, ok := featAttrs[i].(*base.BinaryAttribute); !ok {
			return fmt.Errorf("%v: Should be BinaryAttribute", featAttrs[i])
		}
	}
	featAttrSpecs := base.ResolveAttributes(X, featAttrs)

	// Check that only one classAttribute is defined
	if len(classAttrs) != 1 {
		return fmt.Errorf("Only one class Attribute can be used")
	}

	// Number of features and instances in this training set
	_, nb.trainingInstances = X.Size()
	nb.attrs = featAttrs
	nb.features = len(featAttrs)

	// Number of instances in class
	nb.classInstances = make(map[string]int)

	// Number of documents with given term (by class)
	docsContainingTerm := make(map[string][]int)

	// This algorithm could be vectorized after binarizing the data
	// matrix. Since mat doesn't have this function, a iterative
	// version is used.
	X.MapOverRows(featAttrSpecs, func(docVector [][]byte, r int) (bool, error) {
		class := base.GetClass(X, r)

		// increment number of instances in class
		t, ok := nb.classInstances[class]
		if !ok {
			t = 0
		}
		nb.classInstances[class] = t + 1

		for feat := 0; feat < len(docVector); feat++ {
			v := docVector[feat]
			// In Bernoulli Naive Bayes the presence and absence of
			// features are considered. All non-zero values are
			// treated as presence.
			if v[0] > 0 {
				// Update number of times this feature appeared within
				// given label.
				t, ok := docsContainingTerm[class]
				if !ok {
					t = make([]int, nb.features)
					docsContainingTerm[class] = t
				}
				t[feat] += 1
			}
		}
		return true, nil
	})

	// Pre-calculate conditional probabilities for each class
	for c, _ := range nb.classInstances {
		nb.condProb[c] = make([]float64, nb.features)
		for feat := 0; feat < nb.features; feat++ {
			classTerms, _ := docsContainingTerm[c]
			numDocs := classTerms[feat]
			docsInClass, _ := nb.classInstances[c]

			classCondProb, _ := nb.condProb[c]
			// Calculate conditional probability with laplace smoothing
			classCondProb[feat] = float64(numDocs+1) / float64(docsInClass+1)
		}
	}

	nb.fitOn = base.NewStructuralCopy(X)
	return nil
}

// Use trained model to predict test vector's class. The following
// operation is used in order to score each class:
//
// classScore = log(p(c)) + \sum_{f}{log(p(f|c))}
//
// PredictOne returns the string that represents the predicted class.
//
// IMPORTANT: PredictOne panics if Fit was not called or if the
// document vector and train matrix have a different number of columns.
func (nb *BernoulliNBClassifier) PredictOne(vector [][]byte) string {
	if nb.features == 0 {
		panic("Fit should be called before predicting")
	}

	if len(vector) != nb.features {
		panic("Different dimensions in Train and Test sets")
	}

	// Currently only the predicted class is returned.
	bestScore := -math.MaxFloat64
	bestClass := ""

	for class, classCount := range nb.classInstances {
		// Init classScore with log(prior)
		classScore := math.Log((float64(classCount)) / float64(nb.trainingInstances))
		for f := 0; f < nb.features; f++ {
			if vector[f][0] > 0 {
				// Test document has feature c
				classScore += math.Log(nb.condProb[class][f])
			} else {
				if nb.condProb[class][f] == 1.0 {
					// special case when prob = 1.0, consider laplace
					// smooth
					classScore += math.Log(1.0 / float64(nb.classInstances[class]+1))
				} else {
					classScore += math.Log(1.0 - nb.condProb[class][f])
				}
			}
		}

		if classScore > bestScore {
			bestScore = classScore
			bestClass = class
		}
	}

	return bestClass
}

// Predict is just a wrapper for the PredictOne function.
//
// IMPORTANT: Predict panics if Fit was not called or if the
// document vector and train matrix have a different number of columns.
func (nb *BernoulliNBClassifier) Predict(what base.FixedDataGrid) (base.FixedDataGrid, error) {
	// Generate return vector
	ret := base.GeneratePredictionVector(what)

	// Get the features
	featAttrSpecs := base.ResolveAttributes(what, nb.attrs)

	what.MapOverRows(featAttrSpecs, func(row [][]byte, i int) (bool, error) {
		base.SetClass(ret, i, nb.PredictOne(row))
		return true, nil
	})

	return ret, nil
}
<|endoftext|>