<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Write out a file of a certain size, close it, then measure the performance
// of doing the following:
//
// 1.  Open the file.
// 2.  Read it from start to end with a configurable buffer size.
//
package main

import (
	"crypto/rand"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"sort"
	"time"
)

var fDir = flag.String("dir", "", "Directory within which to write the file.")
var fFileSize = flag.Int64("file_size", 1<<20, "Size of file to use.")
var fReadSize = flag.Int64("read_size", 1<<14, "Size of each call to read(2).")

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type DurationSlice []time.Duration

func (p DurationSlice) Len() int           { return len(p) }
func (p DurationSlice) Less(i, j int) bool { return p[i] < p[j] }
func (p DurationSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

// REQUIRES: vals is sorted.
// REQUIRES: len(vals) > 0
// REQUIRES: 0 <= n <= 100
func percentile(
	vals DurationSlice,
	n int) (x time.Duration) {
	// Special cases.
	switch {
	case n == 0:
		x = vals[0]
		return

	case n == 100:
		x = vals[len(vals)-1]
		return
	}

	// Find the nearest, truncating (why not).
	index := int((float64(n) / 100) * float64(len(vals)))
	x = vals[index]

	return
}

func formatBytes(v float64) string {
	switch {
	case v >= 1<<30:
		return fmt.Sprintf("%.2f GiB", v/(1<<30))

	case v >= 1<<20:
		return fmt.Sprintf("%.2f MiB", v/(1<<20))

	case v >= 1<<10:
		return fmt.Sprintf("%.2f KiB", v/(1<<10))

	default:
		return fmt.Sprintf("%.2f bytes", v)
	}
}

////////////////////////////////////////////////////////////////////////
// main logic
////////////////////////////////////////////////////////////////////////

func run() (err error) {
	if *fDir == "" {
		err = errors.New("You must set --dir.")
		return
	}

	// Create a temporary file.
	log.Printf("Creating a temporary file in %s.", *fDir)

	f, err := ioutil.TempFile(*fDir, "sequential_read")
	if err != nil {
		err = fmt.Errorf("TempFile: %v", err)
		return
	}

	path := f.Name()

	// Make sure we clean it up later.
	defer func() {
		log.Printf("Deleting %s.", path)
		os.Remove(path)
	}()

	// Fill it with random content.
	log.Printf("Writing %d random bytes.", *fFileSize)
	_, err = io.Copy(f, io.LimitReader(rand.Reader, *fFileSize))
	if err != nil {
		err = fmt.Errorf("Copying random bytes: %v", err)
		return
	}

	// Finish off the file.
	err = f.Close()
	if err != nil {
		err = fmt.Errorf("Closing file: %v", err)
		return
	}

	// Run several iterations.
	const duration = 5 * time.Second
	log.Printf("Measuring for %v...", duration)

	var fullFileRead DurationSlice
	var singleReadCall DurationSlice
	buf := make([]byte, *fReadSize)

	overallStartTime := time.Now()
	for len(fullFileRead) == 0 || time.Since(overallStartTime) < duration {
		// Open the file for reading.
		f, err = os.Open(path)
		if err != nil {
			err = fmt.Errorf("Opening file: %v", err)
			return
		}

		// Read the whole thing.
		fileStartTime := time.Now()
		for err == nil {
			readStartTime := time.Now()
			_, err = f.Read(buf)
			singleReadCall = append(singleReadCall, time.Since(readStartTime))
		}

		fullFileRead = append(fullFileRead, time.Since(fileStartTime))

		switch {
		case err == io.EOF:
			err = nil

		case err != nil:
			err = fmt.Errorf("Reading: %v", err)
			return
		}

		// Close the file.
		err = f.Close()
		if err != nil {
			err = fmt.Errorf("Closing file after reading: %v", err)
			return
		}
	}

	sort.Sort(fullFileRead)
	sort.Sort(singleReadCall)

	// Report.
	ptiles := []int{50, 90, 98}

	reportSlice := func(
		name string,
		bytesPerObservation int64,
		observations DurationSlice) {
		fmt.Printf("\n%s:\n", name)
		for _, ptile := range ptiles {
			d := percentile(observations, ptile)
			seconds := float64(d) / float64(time.Second)
			bandwidthBytesPerSec := float64(bytesPerObservation) / seconds

			fmt.Printf(
				"  %02dth ptile: %10v (%s/s)\n",
				ptile,
				d,
				formatBytes(bandwidthBytesPerSec))
		}
	}

	reportSlice("Full-file read times", *fFileSize, fullFileRead)
	reportSlice("read(2) latencies", *fReadSize, singleReadCall)

	fmt.Println()

	return
}

func main() {
	log.SetFlags(log.Lmicroseconds | log.Lshortfile)
	flag.Parse()

	err := run()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}
<commit_msg>Made the duration configurable.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Write out a file of a certain size, close it, then measure the performance
// of doing the following:
//
// 1.  Open the file.
// 2.  Read it from start to end with a configurable buffer size.
//
package main

import (
	"crypto/rand"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"sort"
	"time"
)

var fDir = flag.String("dir", "", "Directory within which to write the file.")
var fDuration = flag.Duration("duration", 5*time.Second, "How long to run.")
var fFileSize = flag.Int64("file_size", 1<<20, "Size of file to use.")
var fReadSize = flag.Int64("read_size", 1<<14, "Size of each call to read(2).")

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type DurationSlice []time.Duration

func (p DurationSlice) Len() int           { return len(p) }
func (p DurationSlice) Less(i, j int) bool { return p[i] < p[j] }
func (p DurationSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

// REQUIRES: vals is sorted.
// REQUIRES: len(vals) > 0
// REQUIRES: 0 <= n <= 100
func percentile(
	vals DurationSlice,
	n int) (x time.Duration) {
	// Special cases.
	switch {
	case n == 0:
		x = vals[0]
		return

	case n == 100:
		x = vals[len(vals)-1]
		return
	}

	// Find the nearest, truncating (why not).
	index := int((float64(n) / 100) * float64(len(vals)))
	x = vals[index]

	return
}

func formatBytes(v float64) string {
	switch {
	case v >= 1<<30:
		return fmt.Sprintf("%.2f GiB", v/(1<<30))

	case v >= 1<<20:
		return fmt.Sprintf("%.2f MiB", v/(1<<20))

	case v >= 1<<10:
		return fmt.Sprintf("%.2f KiB", v/(1<<10))

	default:
		return fmt.Sprintf("%.2f bytes", v)
	}
}

////////////////////////////////////////////////////////////////////////
// main logic
////////////////////////////////////////////////////////////////////////

func run() (err error) {
	if *fDir == "" {
		err = errors.New("You must set --dir.")
		return
	}

	// Create a temporary file.
	log.Printf("Creating a temporary file in %s.", *fDir)

	f, err := ioutil.TempFile(*fDir, "sequential_read")
	if err != nil {
		err = fmt.Errorf("TempFile: %v", err)
		return
	}

	path := f.Name()

	// Make sure we clean it up later.
	defer func() {
		log.Printf("Deleting %s.", path)
		os.Remove(path)
	}()

	// Fill it with random content.
	log.Printf("Writing %d random bytes.", *fFileSize)
	_, err = io.Copy(f, io.LimitReader(rand.Reader, *fFileSize))
	if err != nil {
		err = fmt.Errorf("Copying random bytes: %v", err)
		return
	}

	// Finish off the file.
	err = f.Close()
	if err != nil {
		err = fmt.Errorf("Closing file: %v", err)
		return
	}

	// Run several iterations.
	log.Printf("Measuring for %v...", *fDuration)

	var fullFileRead DurationSlice
	var singleReadCall DurationSlice
	buf := make([]byte, *fReadSize)

	overallStartTime := time.Now()
	for len(fullFileRead) == 0 || time.Since(overallStartTime) < *fDuration {
		// Open the file for reading.
		f, err = os.Open(path)
		if err != nil {
			err = fmt.Errorf("Opening file: %v", err)
			return
		}

		// Read the whole thing.
		fileStartTime := time.Now()
		for err == nil {
			readStartTime := time.Now()
			_, err = f.Read(buf)
			singleReadCall = append(singleReadCall, time.Since(readStartTime))
		}

		fullFileRead = append(fullFileRead, time.Since(fileStartTime))

		switch {
		case err == io.EOF:
			err = nil

		case err != nil:
			err = fmt.Errorf("Reading: %v", err)
			return
		}

		// Close the file.
		err = f.Close()
		if err != nil {
			err = fmt.Errorf("Closing file after reading: %v", err)
			return
		}
	}

	sort.Sort(fullFileRead)
	sort.Sort(singleReadCall)

	// Report.
	ptiles := []int{50, 90, 98}

	reportSlice := func(
		name string,
		bytesPerObservation int64,
		observations DurationSlice) {
		fmt.Printf("\n%s:\n", name)
		for _, ptile := range ptiles {
			d := percentile(observations, ptile)
			seconds := float64(d) / float64(time.Second)
			bandwidthBytesPerSec := float64(bytesPerObservation) / seconds

			fmt.Printf(
				"  %02dth ptile: %10v (%s/s)\n",
				ptile,
				d,
				formatBytes(bandwidthBytesPerSec))
		}
	}

	reportSlice("Full-file read times", *fFileSize, fullFileRead)
	reportSlice("read(2) latencies", *fReadSize, singleReadCall)

	fmt.Println()

	return
}

func main() {
	log.SetFlags(log.Lmicroseconds | log.Lshortfile)
	flag.Parse()

	err := run()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package cucumberexpressions

import (
	"errors"
	"regexp"
)

var HAS_FLAG_REGEKP = regexp.MustCompile(`\(\?[imsU-]+(\:.*)?\)`)
var UNESCAPE_REGEXP = regexp.MustCompile(`(\\([\[$.|?*+\]]))`)
var ILLEGAL_PARAMETER_NAME_REGEXP = regexp.MustCompile(`([\[\]()$.|?*+])`)

type ParameterType struct {
	name                 string
	regexps              []*regexp.Regexp
	type1                string // Cannot have a field named type as hit a compile error
	transform            func(...*string) interface{}
	useForSnippets       bool
	preferForRegexpMatch bool
}

func CheckParameterTypeName(typeName string) (error) {
	if ILLEGAL_PARAMETER_NAME_REGEXP.MatchString(typeName) {
		return errors.New("Illegal character '[' in parameter name {[string]}")
	}
	return nil
}

func NewParameterType(name string, regexps []*regexp.Regexp, type1 string, transform func(...*string) interface{}, useForSnippets bool, preferForRegexpMatch bool) (*ParameterType, error) {
	if transform == nil {
		transform = func(s ...*string) interface{} {
			return *s[0]
		}
	}
	for _, r := range regexps {
		if HAS_FLAG_REGEKP.MatchString(r.String()) {
			return nil, errors.New("ParameterType Regexps can't use flags")
		}
	}
	err := CheckParameterTypeName(name)
	if err != nil {
		return nil, err
	}
	return &ParameterType{
		name:                 name,
		regexps:              regexps,
		type1:                type1,
		transform:            transform,
		useForSnippets:       useForSnippets,
		preferForRegexpMatch: preferForRegexpMatch,
	}, nil
}

func (p *ParameterType) Name() string {
	return p.name
}

func (p *ParameterType) Regexps() []*regexp.Regexp {
	return p.regexps
}

func (p *ParameterType) Type() string {
	return p.type1
}

func (p *ParameterType) UseForSnippets() bool {
	return p.useForSnippets
}

func (p *ParameterType) PreferForRegexpMatch() bool {
	return p.preferForRegexpMatch
}

func (p *ParameterType) Transform(groupValues []*string) interface{} {
	return p.transform(groupValues...)
}

func CompareParameterTypes(pt1, pt2 *ParameterType) int {
	if pt1.PreferForRegexpMatch() && !pt2.PreferForRegexpMatch() {
		return -1
	}
	if pt2.PreferForRegexpMatch() && !pt1.PreferForRegexpMatch() {
		return 1
	}
	if pt1.Name() < pt2.Name() {
		return -1
	}
	if pt1.Name() > pt2.Name() {
		return 1
	}
	return 0
}
<commit_msg>Remove hard coding<commit_after>package cucumberexpressions

import (
	"errors"
	"regexp"
	"fmt"
)

var HAS_FLAG_REGEKP = regexp.MustCompile(`\(\?[imsU-]+(\:.*)?\)`)
var UNESCAPE_REGEXP = regexp.MustCompile(`(\\([\[$.|?*+\]]))`)
var ILLEGAL_PARAMETER_NAME_REGEXP = regexp.MustCompile(`([\[\]()$.|?*+])`)

type ParameterType struct {
	name                 string
	regexps              []*regexp.Regexp
	type1                string // Cannot have a field named type as hit a compile error
	transform            func(...*string) interface{}
	useForSnippets       bool
	preferForRegexpMatch bool
}

func CheckParameterTypeName(typeName string) (error) {
	unescapedTypeName := UNESCAPE_REGEXP.ReplaceAllString(typeName, "$2");
	if ILLEGAL_PARAMETER_NAME_REGEXP.MatchString(typeName) {
		c := ILLEGAL_PARAMETER_NAME_REGEXP.FindStringSubmatch(typeName)[0]
		return errors.New(fmt.Sprintf("Illegal character '%s' in parameter name {%s}", c, unescapedTypeName))
	}
	return nil
}

func NewParameterType(name string, regexps []*regexp.Regexp, type1 string, transform func(...*string) interface{}, useForSnippets bool, preferForRegexpMatch bool) (*ParameterType, error) {
	if transform == nil {
		transform = func(s ...*string) interface{} {
			return *s[0]
		}
	}
	for _, r := range regexps {
		if HAS_FLAG_REGEKP.MatchString(r.String()) {
			return nil, errors.New("ParameterType Regexps can't use flags")
		}
	}
	err := CheckParameterTypeName(name)
	if err != nil {
		return nil, err
	}
	return &ParameterType{
		name:                 name,
		regexps:              regexps,
		type1:                type1,
		transform:            transform,
		useForSnippets:       useForSnippets,
		preferForRegexpMatch: preferForRegexpMatch,
	}, nil
}

func (p *ParameterType) Name() string {
	return p.name
}

func (p *ParameterType) Regexps() []*regexp.Regexp {
	return p.regexps
}

func (p *ParameterType) Type() string {
	return p.type1
}

func (p *ParameterType) UseForSnippets() bool {
	return p.useForSnippets
}

func (p *ParameterType) PreferForRegexpMatch() bool {
	return p.preferForRegexpMatch
}

func (p *ParameterType) Transform(groupValues []*string) interface{} {
	return p.transform(groupValues...)
}

func CompareParameterTypes(pt1, pt2 *ParameterType) int {
	if pt1.PreferForRegexpMatch() && !pt2.PreferForRegexpMatch() {
		return -1
	}
	if pt2.PreferForRegexpMatch() && !pt1.PreferForRegexpMatch() {
		return 1
	}
	if pt1.Name() < pt2.Name() {
		return -1
	}
	if pt1.Name() > pt2.Name() {
		return 1
	}
	return 0
}
<|endoftext|>


<commit_before><commit_msg>kubernetes: Check for empty container statuses in Launchcontainer (#205)<commit_after><|endoftext|>


<commit_before>// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

import (
	"github.com/vmware/harbor/src/common/dao"
	"github.com/vmware/harbor/src/common/utils/log"
	"github.com/vmware/harbor/src/ui/utils"

	"net/http"
	"strconv"
	"strings"
)

// ScanJobAPI handles request to /api/scanJobs/:id/log
type ScanJobAPI struct {
	BaseController
	jobID       int64
	projectName string
}

// Prepare validates that whether user has read permission to the project of the repo the scan job scanned.
func (this *ScanJobAPI) Prepare() {
	this.BaseController.Prepare()
	if !this.SecurityCtx.IsAuthenticated() {
		this.HandleUnauthorized()
		return
	}
	id, err := this.GetInt64FromPath(":id")
	if err != nil {
		this.CustomAbort(http.StatusBadRequest, "ID is invalid")
	}
	this.jobID = id

	data, err := dao.GetScanJob(id)
	if err != nil {
		log.Errorf("Failed to load job data for job: %d, error: %v", id, err)
		this.CustomAbort(http.StatusInternalServerError, "Failed to get Job data")
	}
	projectName := strings.SplitN(data.Repository, "/", 2)[0]
	if !this.SecurityCtx.HasReadPerm(projectName) {
		log.Errorf("User does not have read permission for project: %s", projectName)
		this.HandleForbidden(this.SecurityCtx.GetUsername())
	}
	this.projectName = projectName
}

//GetLog ...
func (this *ScanJobAPI) GetLog() {
	url := buildJobLogURL(strconv.FormatInt(this.jobID, 10), ScanJobType)
	err := utils.RequestAsUI(http.MethodGet, url, nil, utils.NewJobLogRespHandler(&this.BaseAPI))
	if err != nil {
		this.RenderError(http.StatusInternalServerError, err.Error())
		return
	}
}
<commit_msg>fix issue in golint<commit_after>// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

import (
	"github.com/vmware/harbor/src/common/dao"
	"github.com/vmware/harbor/src/common/utils/log"
	"github.com/vmware/harbor/src/ui/utils"

	"net/http"
	"strconv"
	"strings"
)

// ScanJobAPI handles request to /api/scanJobs/:id/log
type ScanJobAPI struct {
	BaseController
	jobID       int64
	projectName string
}

// Prepare validates that whether user has read permission to the project of the repo the scan job scanned.
func (sj *ScanJobAPI) Prepare() {
	sj.BaseController.Prepare()
	if !sj.SecurityCtx.IsAuthenticated() {
		sj.HandleUnauthorized()
		return
	}
	id, err := sj.GetInt64FromPath(":id")
	if err != nil {
		sj.CustomAbort(http.StatusBadRequest, "ID is invalid")
	}
	sj.jobID = id

	data, err := dao.GetScanJob(id)
	if err != nil {
		log.Errorf("Failed to load job data for job: %d, error: %v", id, err)
		sj.CustomAbort(http.StatusInternalServerError, "Failed to get Job data")
	}
	projectName := strings.SplitN(data.Repository, "/", 2)[0]
	if !sj.SecurityCtx.HasReadPerm(projectName) {
		log.Errorf("User does not have read permission for project: %s", projectName)
		sj.HandleForbidden(sj.SecurityCtx.GetUsername())
	}
	sj.projectName = projectName
}

//GetLog ...
func (sj *ScanJobAPI) GetLog() {
	url := buildJobLogURL(strconv.FormatInt(sj.jobID, 10), ScanJobType)
	err := utils.RequestAsUI(http.MethodGet, url, nil, utils.NewJobLogRespHandler(&sj.BaseAPI))
	if err != nil {
		sj.RenderError(http.StatusInternalServerError, err.Error())
		return
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package lzma

// Maximum and minimum values for individual parameters.
const (
	MinLC       = 0
	MaxLC       = 8
	MinLP       = 0
	MaxLP       = 4
	MinPB       = 0
	MaxPB       = 4
	MinDictSize = 1 << 12
	MaxDictSize = 1<<32 - 1
)

// Properties contains the parameters lc, lp and pb.
type Properties byte

// NewProperties returns a new properties value. It verifies the validity of
// the arguments.
func NewProperties(lc, lp, pb int) (p Properties, err error) {
	if err = verifyProperties(lc, lp, pb); err != nil {
		return
	}
	return Properties((pb*5+lp)*9 + lc), nil
}

// LC returns the number of literal context bits.
func (p Properties) LC() int {
	return int(p) % 9
}

// LP returns the number of literal position bits.
func (p Properties) LP() int {
	return (int(p) / 9) % 5
}

// PB returns the number of position bits.
func (p Properties) PB() int {
	return (int(p) / 45) % 5
}

// verifyProperties checks the argument for any errors.
func verifyProperties(lc, lp, pb int) error {
	if !(MinLC <= lc && lc <= MaxLC) {
		return rangeError{"lc", lc}
	}
	if !(MinLP <= lp && lp <= MaxLC) {
		return rangeError{"lp", lp}
	}
	if !(MinPB <= pb && pb <= MaxPB) {
		return rangeError{"pb", pb}
	}
	return nil
}
<commit_msg>lzma: defined constant MaxProperties<commit_after>// Copyright 2015 Ulrich Kunitz. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package lzma

// Maximum and minimum values for individual parameters.
const (
	MinLC         = 0
	MaxLC         = 8
	MinLP         = 0
	MaxLP         = 4
	MinPB         = 0
	MaxPB         = 4
	MaxProperties = (MaxPB+1)*(MaxLP+1)*(MaxLC+1) - 1
	MinDictSize   = 1 << 12
	MaxDictSize   = 1<<32 - 1
)

// Properties contains the parameters lc, lp and pb.
type Properties byte

// NewProperties returns a new properties value. It verifies the validity of
// the arguments.
func NewProperties(lc, lp, pb int) (p Properties, err error) {
	if err = verifyProperties(lc, lp, pb); err != nil {
		return
	}
	return Properties((pb*5+lp)*9 + lc), nil
}

// LC returns the number of literal context bits.
func (p Properties) LC() int {
	return int(p) % 9
}

// LP returns the number of literal position bits.
func (p Properties) LP() int {
	return (int(p) / 9) % 5
}

// PB returns the number of position bits.
func (p Properties) PB() int {
	return (int(p) / 45) % 5
}

// verifyProperties checks the argument for any errors.
func verifyProperties(lc, lp, pb int) error {
	if !(MinLC <= lc && lc <= MaxLC) {
		return rangeError{"lc", lc}
	}
	if !(MinLP <= lp && lp <= MaxLC) {
		return rangeError{"lp", lp}
	}
	if !(MinPB <= pb && pb <= MaxPB) {
		return rangeError{"pb", pb}
	}
	return nil
}
<|endoftext|>


<commit_before>package macaroons

import (
	"crypto/rand"
	"fmt"
	"io"

	"golang.org/x/net/context"

	"github.com/coreos/bbolt"

	"github.com/roasbeef/btcwallet/snacl"
)

const (
	// RootKeyLen is the length of a root key.
	RootKeyLen = 32
)

var (
	// rootKeyBucketName is the name of the root key store bucket.
	rootKeyBucketName = []byte("macrootkeys")

	// defaultRootKeyID is the ID of the default root key. The first is
	// just 0, to emulate the memory storage that comes with bakery.
	//
	// TODO(aakselrod): Add support for key rotation.
	defaultRootKeyID = []byte("0")

	// encryptedKeyID is the name of the database key that stores the
	// encryption key, encrypted with a salted + hashed password. The
	// format is 32 bytes of salt, and the rest is encrypted key.
	encryptedKeyID = []byte("enckey")

	// ErrAlreadyUnlocked specifies that the store has already been
	// unlocked.
	ErrAlreadyUnlocked = fmt.Errorf("macaroon store already unlocked")

	// ErrStoreLocked specifies that the store needs to be unlocked with
	// a password.
	ErrStoreLocked = fmt.Errorf("macaroon store is locked")

	// ErrPasswordRequired specifies that a nil password has been passed.
	ErrPasswordRequired = fmt.Errorf("a non-nil password is required")
)

// RootKeyStorage implements the bakery.RootKeyStorage interface.
type RootKeyStorage struct {
	*bolt.DB

	encKey *snacl.SecretKey
}

// NewRootKeyStorage creates a RootKeyStorage instance.
// TODO(aakselrod): Add support for encryption of data with passphrase.
func NewRootKeyStorage(db *bolt.DB) (*RootKeyStorage, error) {
	// If the store's bucket doesn't exist, create it.
	err := db.Update(func(tx *bolt.Tx) error {
		_, err := tx.CreateBucketIfNotExists(rootKeyBucketName)
		return err
	})
	if err != nil {
		return nil, err
	}

	// Return the DB wrapped in a RootKeyStorage object.
	return &RootKeyStorage{db, nil}, nil
}

// CreateUnlock sets an encryption key if one is not already set, otherwise it
// checks if the password is correct for the stored encryption key.
func (r *RootKeyStorage) CreateUnlock(password *[]byte) error {
	// Check if we've already unlocked the store; return an error if so.
	if r.encKey != nil {
		return ErrAlreadyUnlocked
	}

	// Check if a nil password has been passed; return an error if so.
	if password == nil {
		return ErrPasswordRequired
	}

	return r.Update(func(tx *bolt.Tx) error {
		bucket := tx.Bucket(rootKeyBucketName)
		dbKey := bucket.Get(encryptedKeyID)
		if len(dbKey) > 0 {
			// We've already stored a key, so try to unlock with
			// the password.
			encKey := &snacl.SecretKey{}
			err := encKey.Unmarshal(dbKey)
			if err != nil {
				return err
			}

			err = encKey.DeriveKey(password)
			if err != nil {
				return err
			}

			r.encKey = encKey
			return nil
		}

		// We haven't yet stored a key, so create a new one.
		encKey, err := snacl.NewSecretKey(password, snacl.DefaultN,
			snacl.DefaultR, snacl.DefaultP)
		if err != nil {
			return err
		}

		err = bucket.Put(encryptedKeyID, encKey.Marshal())
		if err != nil {
			return err
		}

		r.encKey = encKey
		return nil
	})
}

// Get implements the Get method for the bakery.RootKeyStorage interface.
func (r *RootKeyStorage) Get(_ context.Context, id []byte) ([]byte, error) {
	if r.encKey == nil {
		return nil, ErrStoreLocked
	}
	var rootKey []byte
	err := r.View(func(tx *bolt.Tx) error {
		dbKey := tx.Bucket(rootKeyBucketName).Get(id)
		if len(dbKey) == 0 {
			return fmt.Errorf("root key with id %s doesn't exist",
				string(id))
		}

		decKey, err := r.encKey.Decrypt(dbKey)
		if err != nil {
			return err
		}

		rootKey = make([]byte, len(decKey))
		copy(rootKey[:], decKey)
		return nil
	})
	if err != nil {
		return nil, err
	}

	return rootKey, nil
}

// RootKey implements the RootKey method for the bakery.RootKeyStorage
// interface.
// TODO(aakselrod): Add support for key rotation.
func (r *RootKeyStorage) RootKey(_ context.Context) ([]byte, []byte, error) {
	if r.encKey == nil {
		return nil, nil, ErrStoreLocked
	}
	var rootKey []byte
	id := defaultRootKeyID
	err := r.Update(func(tx *bolt.Tx) error {
		ns := tx.Bucket(rootKeyBucketName)
		dbKey := ns.Get(id)

		// If there's a root key stored in the bucket, decrypt it and
		// return it.
		if len(dbKey) != 0 {
			decKey, err := r.encKey.Decrypt(dbKey)
			if err != nil {
				return err
			}

			rootKey = make([]byte, len(decKey))
			copy(rootKey[:], decKey[:])
			return nil
		}

		// Otherwise, create a RootKeyLen-byte root key, encrypt it,
		// and store it in the bucket.
		rootKey = make([]byte, RootKeyLen)
		if _, err := io.ReadFull(rand.Reader, rootKey[:]); err != nil {
			return err
		}

		encKey, err := r.encKey.Encrypt(rootKey)
		if err != nil {
			return err
		}
		return ns.Put(id, encKey)
	})
	if err != nil {
		return nil, nil, err
	}

	return rootKey, id, nil
}

// Close closes the underlying database and zeroes the encryption key stored
// in memory.
func (r *RootKeyStorage) Close() error {
	r.encKey.Zero()
	return r.DB.Close()
}
<commit_msg>macaroons: don't zero mem if never unlocked<commit_after>package macaroons

import (
	"crypto/rand"
	"fmt"
	"io"

	"golang.org/x/net/context"

	"github.com/coreos/bbolt"

	"github.com/roasbeef/btcwallet/snacl"
)

const (
	// RootKeyLen is the length of a root key.
	RootKeyLen = 32
)

var (
	// rootKeyBucketName is the name of the root key store bucket.
	rootKeyBucketName = []byte("macrootkeys")

	// defaultRootKeyID is the ID of the default root key. The first is
	// just 0, to emulate the memory storage that comes with bakery.
	//
	// TODO(aakselrod): Add support for key rotation.
	defaultRootKeyID = []byte("0")

	// encryptedKeyID is the name of the database key that stores the
	// encryption key, encrypted with a salted + hashed password. The
	// format is 32 bytes of salt, and the rest is encrypted key.
	encryptedKeyID = []byte("enckey")

	// ErrAlreadyUnlocked specifies that the store has already been
	// unlocked.
	ErrAlreadyUnlocked = fmt.Errorf("macaroon store already unlocked")

	// ErrStoreLocked specifies that the store needs to be unlocked with
	// a password.
	ErrStoreLocked = fmt.Errorf("macaroon store is locked")

	// ErrPasswordRequired specifies that a nil password has been passed.
	ErrPasswordRequired = fmt.Errorf("a non-nil password is required")
)

// RootKeyStorage implements the bakery.RootKeyStorage interface.
type RootKeyStorage struct {
	*bolt.DB

	encKey *snacl.SecretKey
}

// NewRootKeyStorage creates a RootKeyStorage instance.
// TODO(aakselrod): Add support for encryption of data with passphrase.
func NewRootKeyStorage(db *bolt.DB) (*RootKeyStorage, error) {
	// If the store's bucket doesn't exist, create it.
	err := db.Update(func(tx *bolt.Tx) error {
		_, err := tx.CreateBucketIfNotExists(rootKeyBucketName)
		return err
	})
	if err != nil {
		return nil, err
	}

	// Return the DB wrapped in a RootKeyStorage object.
	return &RootKeyStorage{db, nil}, nil
}

// CreateUnlock sets an encryption key if one is not already set, otherwise it
// checks if the password is correct for the stored encryption key.
func (r *RootKeyStorage) CreateUnlock(password *[]byte) error {
	// Check if we've already unlocked the store; return an error if so.
	if r.encKey != nil {
		return ErrAlreadyUnlocked
	}

	// Check if a nil password has been passed; return an error if so.
	if password == nil {
		return ErrPasswordRequired
	}

	return r.Update(func(tx *bolt.Tx) error {
		bucket := tx.Bucket(rootKeyBucketName)
		dbKey := bucket.Get(encryptedKeyID)
		if len(dbKey) > 0 {
			// We've already stored a key, so try to unlock with
			// the password.
			encKey := &snacl.SecretKey{}
			err := encKey.Unmarshal(dbKey)
			if err != nil {
				return err
			}

			err = encKey.DeriveKey(password)
			if err != nil {
				return err
			}

			r.encKey = encKey
			return nil
		}

		// We haven't yet stored a key, so create a new one.
		encKey, err := snacl.NewSecretKey(password, snacl.DefaultN,
			snacl.DefaultR, snacl.DefaultP)
		if err != nil {
			return err
		}

		err = bucket.Put(encryptedKeyID, encKey.Marshal())
		if err != nil {
			return err
		}

		r.encKey = encKey
		return nil
	})
}

// Get implements the Get method for the bakery.RootKeyStorage interface.
func (r *RootKeyStorage) Get(_ context.Context, id []byte) ([]byte, error) {
	if r.encKey == nil {
		return nil, ErrStoreLocked
	}
	var rootKey []byte
	err := r.View(func(tx *bolt.Tx) error {
		dbKey := tx.Bucket(rootKeyBucketName).Get(id)
		if len(dbKey) == 0 {
			return fmt.Errorf("root key with id %s doesn't exist",
				string(id))
		}

		decKey, err := r.encKey.Decrypt(dbKey)
		if err != nil {
			return err
		}

		rootKey = make([]byte, len(decKey))
		copy(rootKey[:], decKey)
		return nil
	})
	if err != nil {
		return nil, err
	}

	return rootKey, nil
}

// RootKey implements the RootKey method for the bakery.RootKeyStorage
// interface.
// TODO(aakselrod): Add support for key rotation.
func (r *RootKeyStorage) RootKey(_ context.Context) ([]byte, []byte, error) {
	if r.encKey == nil {
		return nil, nil, ErrStoreLocked
	}
	var rootKey []byte
	id := defaultRootKeyID
	err := r.Update(func(tx *bolt.Tx) error {
		ns := tx.Bucket(rootKeyBucketName)
		dbKey := ns.Get(id)

		// If there's a root key stored in the bucket, decrypt it and
		// return it.
		if len(dbKey) != 0 {
			decKey, err := r.encKey.Decrypt(dbKey)
			if err != nil {
				return err
			}

			rootKey = make([]byte, len(decKey))
			copy(rootKey[:], decKey[:])
			return nil
		}

		// Otherwise, create a RootKeyLen-byte root key, encrypt it,
		// and store it in the bucket.
		rootKey = make([]byte, RootKeyLen)
		if _, err := io.ReadFull(rand.Reader, rootKey[:]); err != nil {
			return err
		}

		encKey, err := r.encKey.Encrypt(rootKey)
		if err != nil {
			return err
		}
		return ns.Put(id, encKey)
	})
	if err != nil {
		return nil, nil, err
	}

	return rootKey, id, nil
}

// Close closes the underlying database and zeroes the encryption key stored
// in memory.
func (r *RootKeyStorage) Close() error {
	if r.encKey != nil {
		r.encKey.Zero()
	}
	return r.DB.Close()
}
<|endoftext|>


<commit_before>// Copyright 2015 Martin Hebnes Pedersen (LA5NTA). All rights reserved.
// Use of this source code is governed by the MIT-license that can be
// found in the LICENSE file.

// Package mailbox provides mailbox handlers for a wl2k.Session.
package mailbox

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/user"
	"path"

	"github.com/la5nta/wl2k-go"
)

const (
	DIR_INBOX   = "/in/"
	DIR_OUTBOX  = "/out/"
	DIR_SENT    = "/sent/"
	DIR_ARCHIVE = "/archive/"
)

// NewDirHandler is a file system (directory) oriented mailbox handler.
type DirHandler struct {
	MBoxPath string
	deferred map[string]bool
	sendOnly bool
}

// NewDirHandler wraps the directory given by path as a DirHandler.
//
// If sendOnly is true, all inbound messages will be deferred.
func NewDirHandler(path string, sendOnly bool) *DirHandler {
	return &DirHandler{
		MBoxPath: path,
		sendOnly: sendOnly,
	}
}

func (h *DirHandler) Prepare() (err error) {
	h.deferred = make(map[string]bool)
	return ensureDirStructure(h.MBoxPath)
}

func (h *DirHandler) Inbox() ([]*wl2k.Message, error) {
	return LoadMessageDir(path.Join(h.MBoxPath, DIR_INBOX))
}

func (h *DirHandler) Outbox() ([]*wl2k.Message, error) {
	return LoadMessageDir(path.Join(h.MBoxPath, DIR_OUTBOX))
}

func (h *DirHandler) Sent() ([]*wl2k.Message, error) {
	return LoadMessageDir(path.Join(h.MBoxPath, DIR_SENT))
}

func (h *DirHandler) Archive() ([]*wl2k.Message, error) {
	return LoadMessageDir(path.Join(h.MBoxPath, DIR_ARCHIVE))
}

// InboxCount returns the number of messages in the inbox. -1 on error.
func (h *DirHandler) InboxCount() int   { return countFiles(path.Join(h.MBoxPath, DIR_INBOX)) }
func (h *DirHandler) OutboxCount() int  { return countFiles(path.Join(h.MBoxPath, DIR_OUTBOX)) }
func (h *DirHandler) SentCount() int    { return countFiles(path.Join(h.MBoxPath, DIR_SENT)) }
func (h *DirHandler) ArchiveCount() int { return countFiles(path.Join(h.MBoxPath, DIR_ARCHIVE)) }

func (h *DirHandler) AddOut(msg *wl2k.Message) error {
	data, err := msg.Bytes()
	if err != nil {
		return err
	}

	return ioutil.WriteFile(path.Join(h.MBoxPath, DIR_OUTBOX, msg.MID()), data, 0644)
}

func (h *DirHandler) ProcessInbound(msgs ...*wl2k.Message) (err error) {
	dir := path.Join(h.MBoxPath, DIR_INBOX)
	for _, m := range msgs {
		filename := path.Join(dir, m.MID())

		data, err := m.Bytes()
		if err != nil {
			return err
		}

		if err = ioutil.WriteFile(filename, data, 0664); err != nil {
			err = fmt.Errorf("Unable to write received message (%s): %s", filename, err)
		}
	}
	return
}

func (h *DirHandler) GetInboundAnswer(p wl2k.Proposal) wl2k.ProposalAnswer {
	if h.sendOnly {
		return wl2k.Defer
	}

	// Check if file exists
	f, err := os.Open(path.Join(h.MBoxPath, DIR_INBOX, p.MID()))
	if err == nil {
		f.Close()
		return wl2k.Reject
	} else if os.IsNotExist(err) {
		return wl2k.Accept
	} else if err != nil {
		log.Printf("Unable to determin if %s has been received: %s", p.MID(), err)
	}

	return wl2k.Accept
}

func (h *DirHandler) SetSent(MID string, rejected bool) {
	oldPath := path.Join(h.MBoxPath, DIR_OUTBOX, MID)
	newPath := path.Join(h.MBoxPath, DIR_SENT, MID)

	if err := os.Rename(oldPath, newPath); err != nil {
		log.Fatalf("Unable to move %s to %s: %s", oldPath, newPath, err)
	}
}

func (h *DirHandler) SetDeferred(MID string) {
	h.deferred[MID] = true
}

func (h *DirHandler) GetOutbound(fws ...wl2k.Address) []*wl2k.Message {
	all, err := LoadMessageDir(path.Join(h.MBoxPath, DIR_OUTBOX))
	if err != nil {
		log.Println(err)
	}

	deliver := make([]*wl2k.Message, 0, len(all))
	for _, m := range all {
		if h.deferred[m.MID()] {
			continue
		}

		// Check unsent messages that are addressed to one of the
		// forwarder addresses of the remote.
		if len(fws) > 0 {
			for _, fw := range fws {
				if m.IsOnlyReceiver(fw) {
					deliver = append(deliver, m)
					break
				}
			}
			continue
		}

		if len(fws) == 0 && m.Header.Get("X-P2POnly") == "true" {
			continue // The message is P2POnly and remote is CMS
		}

		deliver = append(deliver, m)
	}
	return deliver
}

func DefaultMailboxPath() (string, error) {
	appdir, err := DefaultAppDir()
	if err != nil {
		return "", fmt.Errorf("Unable to determine application directory: %s", err)
	}
	return path.Join(appdir, "mailbox"), nil
}

func DefaultAppDir() (string, error) {
	usr, err := user.Current()
	if err != nil {
		return "", fmt.Errorf("Unable to determine home directory: %s", err)
	}
	return path.Join(usr.HomeDir, ".wl2k"), nil
}

func ensureDirStructure(mboxPath string) (err error) {
	mode := os.ModeDir | os.ModePerm
	if err = os.MkdirAll(path.Join(mboxPath, DIR_INBOX), mode); err != nil {
		return
	} else if err = os.MkdirAll(path.Join(mboxPath, DIR_OUTBOX), mode); err != nil {
		return
	} else if err = os.MkdirAll(path.Join(mboxPath, DIR_SENT), mode); err != nil {
		return
	} else if err = os.MkdirAll(path.Join(mboxPath, DIR_ARCHIVE), mode); err != nil {
		return
	}
	return
}

func UserPath(root, callsign string) string {
	return path.Join(root, callsign)
}

func countFiles(dirPath string) int {
	files, err := ioutil.ReadDir(dirPath)
	if err != nil {
		return -1
	}

	return len(files)
}

func LoadMessageDir(dirPath string) ([]*wl2k.Message, error) {
	files, err := ioutil.ReadDir(dirPath)
	if err != nil {
		return nil, fmt.Errorf("Unable to read dir (%s): %s", err)
	}

	msgs := make([]*wl2k.Message, 0, len(files))

	for _, file := range files {
		if file.IsDir() || file.Name()[0] == '.' {
			continue
		}

		filePath := path.Join(dirPath, file.Name())

		f, err := os.Open(filePath)
		if err != nil {
			return nil, fmt.Errorf("Unable to open file (%s): %s", filePath, err)
		}

		message := new(wl2k.Message)
		if err := message.ReadFrom(f); err != nil {
			f.Close()
			return nil, fmt.Errorf("Unable to parse message (%s): %s", filePath, err)
		}

		msgs = append(msgs, message)
		f.Close()
	}
	return msgs, nil
}
<commit_msg>mailbox: Keep track of unread messages<commit_after>// Copyright 2015 Martin Hebnes Pedersen (LA5NTA). All rights reserved.
// Use of this source code is governed by the MIT-license that can be
// found in the LICENSE file.

// Package mailbox provides mailbox handlers for a wl2k.Session.
package mailbox

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/user"
	"path"

	"github.com/la5nta/wl2k-go"
)

const (
	DIR_INBOX   = "/in/"
	DIR_OUTBOX  = "/out/"
	DIR_SENT    = "/sent/"
	DIR_ARCHIVE = "/archive/"
)

// NewDirHandler is a file system (directory) oriented mailbox handler.
type DirHandler struct {
	MBoxPath string
	deferred map[string]bool
	sendOnly bool
}

// NewDirHandler wraps the directory given by path as a DirHandler.
//
// If sendOnly is true, all inbound messages will be deferred.
func NewDirHandler(path string, sendOnly bool) *DirHandler {
	return &DirHandler{
		MBoxPath: path,
		sendOnly: sendOnly,
	}
}

func (h *DirHandler) Prepare() (err error) {
	h.deferred = make(map[string]bool)
	return ensureDirStructure(h.MBoxPath)
}

func (h *DirHandler) Inbox() ([]*wl2k.Message, error) {
	return LoadMessageDir(path.Join(h.MBoxPath, DIR_INBOX))
}

func (h *DirHandler) Outbox() ([]*wl2k.Message, error) {
	return LoadMessageDir(path.Join(h.MBoxPath, DIR_OUTBOX))
}

func (h *DirHandler) Sent() ([]*wl2k.Message, error) {
	return LoadMessageDir(path.Join(h.MBoxPath, DIR_SENT))
}

func (h *DirHandler) Archive() ([]*wl2k.Message, error) {
	return LoadMessageDir(path.Join(h.MBoxPath, DIR_ARCHIVE))
}

// InboxCount returns the number of messages in the inbox. -1 on error.
func (h *DirHandler) InboxCount() int   { return countFiles(path.Join(h.MBoxPath, DIR_INBOX)) }
func (h *DirHandler) OutboxCount() int  { return countFiles(path.Join(h.MBoxPath, DIR_OUTBOX)) }
func (h *DirHandler) SentCount() int    { return countFiles(path.Join(h.MBoxPath, DIR_SENT)) }
func (h *DirHandler) ArchiveCount() int { return countFiles(path.Join(h.MBoxPath, DIR_ARCHIVE)) }

func (h *DirHandler) AddOut(msg *wl2k.Message) error {
	data, err := msg.Bytes()
	if err != nil {
		return err
	}

	return ioutil.WriteFile(path.Join(h.MBoxPath, DIR_OUTBOX, msg.MID()), data, 0644)
}

func (h *DirHandler) ProcessInbound(msgs ...*wl2k.Message) (err error) {
	dir := path.Join(h.MBoxPath, DIR_INBOX)
	for _, m := range msgs {
		filename := path.Join(dir, m.MID())

		m.Header.Set("X-Unread", "true")

		data, err := m.Bytes()
		if err != nil {
			return err
		}

		if err = ioutil.WriteFile(filename, data, 0664); err != nil {
			err = fmt.Errorf("Unable to write received message (%s): %s", filename, err)
		}
	}
	return
}

func (h *DirHandler) GetInboundAnswer(p wl2k.Proposal) wl2k.ProposalAnswer {
	if h.sendOnly {
		return wl2k.Defer
	}

	// Check if file exists
	f, err := os.Open(path.Join(h.MBoxPath, DIR_INBOX, p.MID()))
	if err == nil {
		f.Close()
		return wl2k.Reject
	} else if os.IsNotExist(err) {
		return wl2k.Accept
	} else if err != nil {
		log.Printf("Unable to determin if %s has been received: %s", p.MID(), err)
	}

	return wl2k.Accept
}

func (h *DirHandler) SetSent(MID string, rejected bool) {
	oldPath := path.Join(h.MBoxPath, DIR_OUTBOX, MID)
	newPath := path.Join(h.MBoxPath, DIR_SENT, MID)

	if err := os.Rename(oldPath, newPath); err != nil {
		log.Fatalf("Unable to move %s to %s: %s", oldPath, newPath, err)
	}
}

func (h *DirHandler) SetDeferred(MID string) {
	h.deferred[MID] = true
}

func (h *DirHandler) GetOutbound(fws ...wl2k.Address) []*wl2k.Message {
	all, err := LoadMessageDir(path.Join(h.MBoxPath, DIR_OUTBOX))
	if err != nil {
		log.Println(err)
	}

	deliver := make([]*wl2k.Message, 0, len(all))
	for _, m := range all {
		if h.deferred[m.MID()] {
			continue
		}

		// Check unsent messages that are addressed to one of the
		// forwarder addresses of the remote.
		if len(fws) > 0 {
			for _, fw := range fws {
				if m.IsOnlyReceiver(fw) {
					deliver = append(deliver, m)
					break
				}
			}
			continue
		}

		if len(fws) == 0 && m.Header.Get("X-P2POnly") == "true" {
			continue // The message is P2POnly and remote is CMS
		}

		// Remove private headers
		m.Header.Del("X-P2POnly")
		m.Header.Del("X-FilePath")
		m.Header.Del("X-Unread")

		deliver = append(deliver, m)
	}
	return deliver
}

func DefaultMailboxPath() (string, error) {
	appdir, err := DefaultAppDir()
	if err != nil {
		return "", fmt.Errorf("Unable to determine application directory: %s", err)
	}
	return path.Join(appdir, "mailbox"), nil
}

func DefaultAppDir() (string, error) {
	usr, err := user.Current()
	if err != nil {
		return "", fmt.Errorf("Unable to determine home directory: %s", err)
	}
	return path.Join(usr.HomeDir, ".wl2k"), nil
}

func ensureDirStructure(mboxPath string) (err error) {
	mode := os.ModeDir | os.ModePerm
	if err = os.MkdirAll(path.Join(mboxPath, DIR_INBOX), mode); err != nil {
		return
	} else if err = os.MkdirAll(path.Join(mboxPath, DIR_OUTBOX), mode); err != nil {
		return
	} else if err = os.MkdirAll(path.Join(mboxPath, DIR_SENT), mode); err != nil {
		return
	} else if err = os.MkdirAll(path.Join(mboxPath, DIR_ARCHIVE), mode); err != nil {
		return
	}
	return
}

func UserPath(root, callsign string) string {
	return path.Join(root, callsign)
}

func countFiles(dirPath string) int {
	files, err := ioutil.ReadDir(dirPath)
	if err != nil {
		return -1
	}

	return len(files)
}

func LoadMessageDir(dirPath string) ([]*wl2k.Message, error) {
	files, err := ioutil.ReadDir(dirPath)
	if err != nil {
		return nil, fmt.Errorf("Unable to read dir (%s): %s", err)
	}

	msgs := make([]*wl2k.Message, 0, len(files))

	for _, file := range files {
		if file.IsDir() || file.Name()[0] == '.' {
			continue
		}

		msg, err := OpenMessage(path.Join(dirPath, file.Name()))
		if err != nil {
			return nil, err
		}

		msgs = append(msgs, msg)
	}
	return msgs, nil
}

// OpenMessage opens a single a wl2k.Message file.
func OpenMessage(path string) (*wl2k.Message, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, fmt.Errorf("Unable to open file (%s): %s", path, err)
	}
	defer f.Close()

	message := new(wl2k.Message)
	if err := message.ReadFrom(f); err != nil {
		f.Close()
		return nil, fmt.Errorf("Unable to parse message (%s): %s", path, err)
	}

	message.Header.Set("X-FilePath", path)
	return message, nil
}

// IsUnread returns true if the given message is marked as unread.
func IsUnread(msg *wl2k.Message) bool { return msg.Header.Get("X-Unread") == "true" }

// SetUnread marks the given message as read/unread and re-writes the file to disk.
func SetUnread(msg *wl2k.Message, unread bool) error {
	if !unread && msg.Header.Get("X-Unread") == "" {
		return nil
	}

	if unread {
		msg.Header.Set("X-Unread", "true")
	} else {
		msg.Header.Del("X-Unread")
	}

	data, err := msg.Bytes()
	if err != nil {
		return err
	}

	filePath := msg.Header.Get("X-FilePath")
	if filePath == "" {
		return fmt.Errorf("Missing X-FilePath header")
	}
	return ioutil.WriteFile(filePath, data, 0644)
}
<|endoftext|>


<commit_before>package scanner

import (
	"bufio"
	"bytes"
	"github.com/github/git-media/pointer"
	"github.com/rubyist/tracerx"
	"io"
	"os/exec"
	"strconv"
	"strings"
	"time"
)

var (
	blobSizeCutoff = 125
	stdoutBufSize  = 16384
)

type wrappedPointer struct {
	Sha1 string
	*pointer.Pointer
}

// Scan takes a ref and returns a slice of pointer.Pointer objects
// for all git media pointers it finds for that ref.
func Scan(ref string) ([]*wrappedPointer, error) {
	start := time.Now()

	revs, _ := revListShas(ref, ref == "")
	smallShas, _ := catFileBatchCheck(revs)
	pointerc, _ := catFileBatch(smallShas)

	pointers := make([]*wrappedPointer, 0)
	for p := range pointerc {
		pointers = append(pointers, p)
	}

	tracerx.PerformanceSince("scan", start)

	return pointers, nil
}

// revListShas uses git rev-list to return the list of object sha1s
// for the given ref. If all is true, ref is ignored. It returns a
// channel from which sha1 strings can be read.
func revListShas(ref string, all bool) (chan string, error) {
	refArgs := []string{"rev-list", "--objects"}
	if all {
		refArgs = append(refArgs, "--all")
	} else {
		refArgs = append(refArgs, "--no-walk")
		refArgs = append(refArgs, ref)
	}

	cmd, err := startCommand("git", refArgs...)
	if err != nil {
		return nil, err
	}

	cmd.Stdin.Close()

	revs := make(chan string)

	go func() {
		scanner := bufio.NewScanner(cmd.Stdout)
		for scanner.Scan() {
			revs <- scanner.Text()[0:40]
		}
		close(revs)
	}()

	return revs, nil
}

// catFileBatchCheck uses git cat-file --batch-check to get the type
// and size of a git object. Any object that isn't of type blob and
// under the blobSizeCutoff will be ignored. revs is a channel over
// which strings containing git sha1s will be sent. It returns a channel
// from which sha1 strings can be read.
func catFileBatchCheck(revs chan string) (chan string, error) {
	cmd, err := startCommand("git", "cat-file", "--batch-check")
	if err != nil {
		return nil, err
	}

	smallRevs := make(chan string)

	go func() {
		scanner := bufio.NewScanner(cmd.Stdout)
		for scanner.Scan() {
			line := scanner.Text()
			if line[41:45] == "blob" {
				size, err := strconv.Atoi(line[46:len(line)])
				if err != nil {
					continue
				}
				if size < blobSizeCutoff {
					smallRevs <- line[0:40]
				}
			}
		}
		close(smallRevs)
	}()

	go func() {
		for r := range revs {
			cmd.Stdin.Write([]byte(r + "\n"))
		}
		cmd.Stdin.Close()
	}()

	return smallRevs, nil
}

// catFileBatch uses git cat-file --batch to get the object contents
// of a git object, given its sha1. The contents will be decoded into
// a git media pointer. revs is a channel over which strings containing
// git sha1s will be sent. It returns a channel from which point.Pointers
// can be read.
func catFileBatch(revs chan string) (chan *wrappedPointer, error) {
	cmd, err := startCommand("git", "cat-file", "--batch")
	if err != nil {
		return nil, err
	}

	pointers := make(chan *wrappedPointer)

	go func() {
		for {
			l, err := cmd.Stdout.ReadBytes('\n')
			if err != nil { // Probably check for EOF
				break
			}

			// Line is formatted:
			// <sha1> <type> <size>
			fields := bytes.Fields(l)
			s, _ := strconv.Atoi(string(fields[2]))

			nbuf := make([]byte, s)
			_, err = io.ReadFull(cmd.Stdout, nbuf)
			if err != nil {
				break // Legit errors
			}

			p, err := pointer.Decode(bytes.NewBuffer(nbuf))
			if err == nil {
				pointers <- &wrappedPointer{string(fields[0]), p}
			}

			_, err = cmd.Stdout.ReadBytes('\n') // Extra \n inserted by cat-file
			if err != nil {                     // Probably check for EOF
				break
			}
		}
		close(pointers)
	}()

	go func() {
		for r := range revs {
			cmd.Stdin.Write([]byte(r + "\n"))
		}
		cmd.Stdin.Close()
	}()

	return pointers, nil
}

type wrappedCmd struct {
	Stdin  io.WriteCloser
	Stdout *bufio.Reader
	*exec.Cmd
}

// startCommand starts up a command and creates a stdin pipe and a buffered
// stdout pipe, wrapped in a wrappedCmd. The stdout buffer wille be of stdoutBufSize
// bytes.
func startCommand(command string, args ...string) (*wrappedCmd, error) {
	cmd := exec.Command(command, args...)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}

	stdin, err := cmd.StdinPipe()
	if err != nil {
		return nil, err
	}

	tracerx.Printf("run_command: %s %s", command, strings.Join(args, " "))
	if err := cmd.Start(); err != nil {
		return nil, err
	}

	return &wrappedCmd{stdin, bufio.NewReaderSize(stdout, stdoutBufSize), cmd}, nil
}
<commit_msg>ンンンンン ンンンン<commit_after>package scanner

import (
	"bufio"
	"bytes"
	"github.com/github/git-media/pointer"
	"github.com/rubyist/tracerx"
	"io"
	"os/exec"
	"strconv"
	"strings"
	"time"
)

var (
	blobSizeCutoff = 125
	stdoutBufSize  = 16384
	chanBufSize    = 100
)

type wrappedPointer struct {
	Sha1 string
	*pointer.Pointer
}

// Scan takes a ref and returns a slice of pointer.Pointer objects
// for all git media pointers it finds for that ref.
func Scan(ref string) ([]*wrappedPointer, error) {
	start := time.Now()

	revs, err := revListShas(ref, ref == "")
	if err != nil {
		return nil, err
	}

	smallShas, err := catFileBatchCheck(revs)
	if err != nil {
		return nil, err
	}

	pointerc, err := catFileBatch(smallShas)
	if err != nil {
		return nil, err
	}

	pointers := make([]*wrappedPointer, 0)
	for p := range pointerc {
		pointers = append(pointers, p)
	}

	tracerx.PerformanceSince("scan", start)

	return pointers, nil
}

// revListShas uses git rev-list to return the list of object sha1s
// for the given ref. If all is true, ref is ignored. It returns a
// channel from which sha1 strings can be read.
func revListShas(ref string, all bool) (chan string, error) {
	refArgs := []string{"rev-list", "--objects"}
	if all {
		refArgs = append(refArgs, "--all")
	} else {
		refArgs = append(refArgs, "--no-walk")
		refArgs = append(refArgs, ref)
	}

	cmd, err := startCommand("git", refArgs...)
	if err != nil {
		return nil, err
	}

	cmd.Stdin.Close()

	revs := make(chan string, chanBufSize)

	go func() {
		scanner := bufio.NewScanner(cmd.Stdout)
		for scanner.Scan() {
			revs <- scanner.Text()[0:40]
		}
		close(revs)
	}()

	return revs, nil
}

// catFileBatchCheck uses git cat-file --batch-check to get the type
// and size of a git object. Any object that isn't of type blob and
// under the blobSizeCutoff will be ignored. revs is a channel over
// which strings containing git sha1s will be sent. It returns a channel
// from which sha1 strings can be read.
func catFileBatchCheck(revs chan string) (chan string, error) {
	cmd, err := startCommand("git", "cat-file", "--batch-check")
	if err != nil {
		return nil, err
	}

	smallRevs := make(chan string, chanBufSize)

	go func() {
		scanner := bufio.NewScanner(cmd.Stdout)
		for scanner.Scan() {
			line := scanner.Text()
			// Format is:
			// <sha1> <type> <size>
			// type is at a fixed spot, if we see that it's "blob", we can avoid
			// splitting the line just to get the size.
			if line[41:45] == "blob" {
				size, err := strconv.Atoi(line[46:len(line)])
				if err != nil {
					continue
				}
				if size < blobSizeCutoff {
					smallRevs <- line[0:40]
				}
			}
		}
		close(smallRevs)
	}()

	go func() {
		for r := range revs {
			cmd.Stdin.Write([]byte(r + "\n"))
		}
		cmd.Stdin.Close()
	}()

	return smallRevs, nil
}

// catFileBatch uses git cat-file --batch to get the object contents
// of a git object, given its sha1. The contents will be decoded into
// a git media pointer. revs is a channel over which strings containing
// git sha1s will be sent. It returns a channel from which point.Pointers
// can be read.
func catFileBatch(revs chan string) (chan *wrappedPointer, error) {
	cmd, err := startCommand("git", "cat-file", "--batch")
	if err != nil {
		return nil, err
	}

	pointers := make(chan *wrappedPointer, chanBufSize)

	go func() {
		for {
			l, err := cmd.Stdout.ReadBytes('\n')
			if err != nil {
				break
			}

			// Line is formatted:
			// <sha1> <type> <size>
			fields := bytes.Fields(l)
			s, _ := strconv.Atoi(string(fields[2]))

			nbuf := make([]byte, s)
			_, err = io.ReadFull(cmd.Stdout, nbuf)
			if err != nil {
				break // Legit errors
			}

			p, err := pointer.Decode(bytes.NewBuffer(nbuf))
			if err == nil {
				pointers <- &wrappedPointer{string(fields[0]), p}
			}

			_, err = cmd.Stdout.ReadBytes('\n') // Extra \n inserted by cat-file
			if err != nil {
				break
			}
		}
		close(pointers)
	}()

	go func() {
		for r := range revs {
			cmd.Stdin.Write([]byte(r + "\n"))
		}
		cmd.Stdin.Close()
	}()

	return pointers, nil
}

type wrappedCmd struct {
	Stdin  io.WriteCloser
	Stdout *bufio.Reader
	*exec.Cmd
}

// startCommand starts up a command and creates a stdin pipe and a buffered
// stdout pipe, wrapped in a wrappedCmd. The stdout buffer wille be of stdoutBufSize
// bytes.
func startCommand(command string, args ...string) (*wrappedCmd, error) {
	cmd := exec.Command(command, args...)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}

	stdin, err := cmd.StdinPipe()
	if err != nil {
		return nil, err
	}

	tracerx.Printf("run_command: %s %s", command, strings.Join(args, " "))
	if err := cmd.Start(); err != nil {
		return nil, err
	}

	return &wrappedCmd{stdin, bufio.NewReaderSize(stdout, stdoutBufSize), cmd}, nil
}
<|endoftext|>


<commit_before>package tlv_test

import (
	"bytes"
	"io"
	"io/ioutil"
	"testing"

	"github.com/lightningnetwork/lnd/lnwallet/chainfee"
	"github.com/lightningnetwork/lnd/tlv"
	"github.com/lightningnetwork/lnd/watchtower/blob"
	"github.com/lightningnetwork/lnd/watchtower/wtwire"
	"github.com/stretchr/testify/require"
)

// CreateSessionTLV mirrors the wtwire.CreateSession message, but uses TLV for
// encoding/decoding.
type CreateSessionTLV struct {
	BlobType     blob.Type
	MaxUpdates   uint16
	RewardBase   uint32
	RewardRate   uint32
	SweepFeeRate chainfee.SatPerKWeight

	tlvStream *tlv.Stream
}

// EBlobType is an encoder for blob.Type.
func EBlobType(w io.Writer, val interface{}, buf *[8]byte) error {
	if t, ok := val.(*blob.Type); ok {
		return tlv.EUint16T(w, uint16(*t), buf)
	}
	return tlv.NewTypeForEncodingErr(val, "blob.Type")
}

// EBlobType is an decoder for blob.Type.
func DBlobType(r io.Reader, val interface{}, buf *[8]byte, l uint64) error {
	if typ, ok := val.(*blob.Type); ok {
		var t uint16
		err := tlv.DUint16(r, &t, buf, l)
		if err != nil {
			return err
		}
		*typ = blob.Type(t)
		return nil
	}
	return tlv.NewTypeForDecodingErr(val, "blob.Type", l, 2)
}

// ESatPerKW is an encoder for lnwallet.SatPerKWeight.
func ESatPerKW(w io.Writer, val interface{}, buf *[8]byte) error {
	if v, ok := val.(*chainfee.SatPerKWeight); ok {
		return tlv.EUint64(w, uint64(*v), buf)
	}
	return tlv.NewTypeForEncodingErr(val, "chainfee.SatPerKWeight")
}

// DSatPerKW is an decoder for lnwallet.SatPerKWeight.
func DSatPerKW(r io.Reader, val interface{}, buf *[8]byte, l uint64) error {
	if v, ok := val.(*chainfee.SatPerKWeight); ok {
		var sat uint64
		err := tlv.DUint64(r, &sat, buf, l)
		if err != nil {
			return err
		}
		*v = chainfee.SatPerKWeight(sat)
		return nil
	}
	return tlv.NewTypeForDecodingErr(val, "chainfee.SatPerKWeight", l, 8)
}

// NewCreateSessionTLV initializes a new CreateSessionTLV message.
func NewCreateSessionTLV() *CreateSessionTLV {
	m := &CreateSessionTLV{}
	m.tlvStream = tlv.MustNewStream(
		tlv.MakeStaticRecord(0, &m.BlobType, 2, EBlobType, DBlobType),
		tlv.MakePrimitiveRecord(1, &m.MaxUpdates),
		tlv.MakePrimitiveRecord(2, &m.RewardBase),
		tlv.MakePrimitiveRecord(3, &m.RewardRate),
		tlv.MakeStaticRecord(4, &m.SweepFeeRate, 8, ESatPerKW, DSatPerKW),
	)

	return m
}

// Encode writes the CreateSessionTLV to the passed io.Writer.
func (c *CreateSessionTLV) Encode(w io.Writer) error {
	return c.tlvStream.Encode(w)
}

// Decode reads the CreateSessionTLV from the passed io.Reader.
func (c *CreateSessionTLV) Decode(r io.Reader) error {
	return c.tlvStream.Decode(r)
}

// BenchmarkEncodeCreateSession benchmarks encoding of the non-TLV
// CreateSession.
func BenchmarkEncodeCreateSession(t *testing.B) {
	m := &wtwire.CreateSession{}

	t.ReportAllocs()
	t.ResetTimer()

	var err error
	for i := 0; i < t.N; i++ {
		err = m.Encode(ioutil.Discard, 0)
	}
	require.NoError(t, err)
}

// BenchmarkEncodeCreateSessionTLV benchmarks encoding of the TLV CreateSession.
func BenchmarkEncodeCreateSessionTLV(t *testing.B) {
	m := NewCreateSessionTLV()

	t.ReportAllocs()
	t.ResetTimer()

	var err error
	for i := 0; i < t.N; i++ {
		err = m.Encode(ioutil.Discard)
	}
	require.NoError(t, err)
}

// BenchmarkDecodeCreateSession benchmarks encoding of the non-TLV
// CreateSession.
func BenchmarkDecodeCreateSession(t *testing.B) {
	m := &wtwire.CreateSession{}

	var b bytes.Buffer
	err := m.Encode(&b, 0)
	require.NoError(t, err)

	r := bytes.NewReader(b.Bytes())

	t.ReportAllocs()
	t.ResetTimer()

	for i := 0; i < t.N; i++ {
		r.Seek(0, 0)
		err = m.Decode(r, 0)
	}
	require.NoError(t, err)
}

// BenchmarkDecodeCreateSessionTLV benchmarks decoding of the TLV CreateSession.
func BenchmarkDecodeCreateSessionTLV(t *testing.B) {
	m := NewCreateSessionTLV()

	var b bytes.Buffer
	err := m.Encode(&b)
	require.NoError(t, err)

	r := bytes.NewReader(b.Bytes())

	t.ReportAllocs()
	t.ResetTimer()

	for i := 0; i < t.N; i++ {
		r.Seek(0, 0)
		err = m.Decode(r)
	}
	require.NoError(t, err)
}
<commit_msg>tlv/bench_test: fix ESatPerKw, pass reference to uint64<commit_after>package tlv_test

import (
	"bytes"
	"io"
	"io/ioutil"
	"testing"

	"github.com/lightningnetwork/lnd/lnwallet/chainfee"
	"github.com/lightningnetwork/lnd/tlv"
	"github.com/lightningnetwork/lnd/watchtower/blob"
	"github.com/lightningnetwork/lnd/watchtower/wtwire"
	"github.com/stretchr/testify/require"
)

// CreateSessionTLV mirrors the wtwire.CreateSession message, but uses TLV for
// encoding/decoding.
type CreateSessionTLV struct {
	BlobType     blob.Type
	MaxUpdates   uint16
	RewardBase   uint32
	RewardRate   uint32
	SweepFeeRate chainfee.SatPerKWeight

	tlvStream *tlv.Stream
}

// EBlobType is an encoder for blob.Type.
func EBlobType(w io.Writer, val interface{}, buf *[8]byte) error {
	if t, ok := val.(*blob.Type); ok {
		return tlv.EUint16T(w, uint16(*t), buf)
	}
	return tlv.NewTypeForEncodingErr(val, "blob.Type")
}

// EBlobType is an decoder for blob.Type.
func DBlobType(r io.Reader, val interface{}, buf *[8]byte, l uint64) error {
	if typ, ok := val.(*blob.Type); ok {
		var t uint16
		err := tlv.DUint16(r, &t, buf, l)
		if err != nil {
			return err
		}
		*typ = blob.Type(t)
		return nil
	}
	return tlv.NewTypeForDecodingErr(val, "blob.Type", l, 2)
}

// ESatPerKW is an encoder for lnwallet.SatPerKWeight.
func ESatPerKW(w io.Writer, val interface{}, buf *[8]byte) error {
	if v, ok := val.(*chainfee.SatPerKWeight); ok {
		v64 := uint64(*v)
		return tlv.EUint64(w, &v64, buf)
	}
	return tlv.NewTypeForEncodingErr(val, "chainfee.SatPerKWeight")
}

// DSatPerKW is an decoder for lnwallet.SatPerKWeight.
func DSatPerKW(r io.Reader, val interface{}, buf *[8]byte, l uint64) error {
	if v, ok := val.(*chainfee.SatPerKWeight); ok {
		var sat uint64
		err := tlv.DUint64(r, &sat, buf, l)
		if err != nil {
			return err
		}
		*v = chainfee.SatPerKWeight(sat)
		return nil
	}
	return tlv.NewTypeForDecodingErr(val, "chainfee.SatPerKWeight", l, 8)
}

// NewCreateSessionTLV initializes a new CreateSessionTLV message.
func NewCreateSessionTLV() *CreateSessionTLV {
	m := &CreateSessionTLV{}
	m.tlvStream = tlv.MustNewStream(
		tlv.MakeStaticRecord(0, &m.BlobType, 2, EBlobType, DBlobType),
		tlv.MakePrimitiveRecord(1, &m.MaxUpdates),
		tlv.MakePrimitiveRecord(2, &m.RewardBase),
		tlv.MakePrimitiveRecord(3, &m.RewardRate),
		tlv.MakeStaticRecord(4, &m.SweepFeeRate, 8, ESatPerKW, DSatPerKW),
	)

	return m
}

// Encode writes the CreateSessionTLV to the passed io.Writer.
func (c *CreateSessionTLV) Encode(w io.Writer) error {
	return c.tlvStream.Encode(w)
}

// Decode reads the CreateSessionTLV from the passed io.Reader.
func (c *CreateSessionTLV) Decode(r io.Reader) error {
	return c.tlvStream.Decode(r)
}

// BenchmarkEncodeCreateSession benchmarks encoding of the non-TLV
// CreateSession.
func BenchmarkEncodeCreateSession(t *testing.B) {
	m := &wtwire.CreateSession{}

	t.ReportAllocs()
	t.ResetTimer()

	var err error
	for i := 0; i < t.N; i++ {
		err = m.Encode(ioutil.Discard, 0)
	}
	require.NoError(t, err)
}

// BenchmarkEncodeCreateSessionTLV benchmarks encoding of the TLV CreateSession.
func BenchmarkEncodeCreateSessionTLV(t *testing.B) {
	m := NewCreateSessionTLV()

	t.ReportAllocs()
	t.ResetTimer()

	var err error
	for i := 0; i < t.N; i++ {
		err = m.Encode(ioutil.Discard)
	}
	require.NoError(t, err)
}

// BenchmarkDecodeCreateSession benchmarks encoding of the non-TLV
// CreateSession.
func BenchmarkDecodeCreateSession(t *testing.B) {
	m := &wtwire.CreateSession{}

	var b bytes.Buffer
	err := m.Encode(&b, 0)
	require.NoError(t, err)

	r := bytes.NewReader(b.Bytes())

	t.ReportAllocs()
	t.ResetTimer()

	for i := 0; i < t.N; i++ {
		r.Seek(0, 0)
		err = m.Decode(r, 0)
	}
	require.NoError(t, err)
}

// BenchmarkDecodeCreateSessionTLV benchmarks decoding of the TLV CreateSession.
func BenchmarkDecodeCreateSessionTLV(t *testing.B) {
	m := NewCreateSessionTLV()

	var b bytes.Buffer
	err := m.Encode(&b)
	require.NoError(t, err)

	r := bytes.NewReader(b.Bytes())

	t.ReportAllocs()
	t.ResetTimer()

	for i := 0; i < t.N; i++ {
		r.Seek(0, 0)
		err = m.Decode(r)
	}
	require.NoError(t, err)
}
<|endoftext|>


<commit_before>// +build e2e

/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package ingress

import (
	"net/http"
	"testing"

	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/apimachinery/pkg/util/sets"
	"knative.dev/serving/pkg/apis/networking"
	"knative.dev/serving/pkg/apis/networking/v1alpha1"
	"knative.dev/serving/test"
)

func TestVisibility(t *testing.T) {
	t.Parallel()
	clients := test.Setup(t)

	// Create the private backend
	name, port, cancel := CreateRuntimeService(t, clients, networking.ServicePortNameHTTP1)
	defer cancel()

	privateHostName := "private." + test.ServingNamespace + ".svc.cluster.local"
	ingress, client, cancel := CreateIngressReady(t, clients, v1alpha1.IngressSpec{
		Rules: []v1alpha1.IngressRule{{
			Hosts:      []string{privateHostName},
			Visibility: v1alpha1.IngressVisibilityClusterLocal,
			HTTP: &v1alpha1.HTTPIngressRuleValue{
				Paths: []v1alpha1.HTTPIngressPath{{
					Splits: []v1alpha1.IngressBackendSplit{{
						IngressBackend: v1alpha1.IngressBackend{
							ServiceName:      name,
							ServiceNamespace: test.ServingNamespace,
							ServicePort:      intstr.FromInt(port),
						},
					}},
				}},
			},
		}},
	})
	defer cancel()

	// Ensure the service is not publicly accessible
	RuntimeRequestWithStatus(t, client, "http://"+privateHostName, sets.NewInt(http.StatusNotFound))

	loadbalancerAddress := ingress.Status.PrivateLoadBalancer.Ingress[0].DomainInternal
	proxyName, proxyPort, cancel := CreateProxyService(t, clients, privateHostName, loadbalancerAddress)
	defer cancel()

	publicHostName := "publicproxy.example.com"
	_, client, cancel = CreateIngressReady(t, clients, v1alpha1.IngressSpec{
		Rules: []v1alpha1.IngressRule{{
			Hosts:      []string{publicHostName},
			Visibility: v1alpha1.IngressVisibilityExternalIP,
			HTTP: &v1alpha1.HTTPIngressRuleValue{
				Paths: []v1alpha1.HTTPIngressPath{{
					Splits: []v1alpha1.IngressBackendSplit{{
						IngressBackend: v1alpha1.IngressBackend{
							ServiceName:      proxyName,
							ServiceNamespace: test.ServingNamespace,
							ServicePort:      intstr.FromInt(proxyPort),
						},
					}},
				}},
			},
		}},
	})
	defer cancel()

	// Ensure the service is accessible from within the cluster.
	RuntimeRequest(t, client, "http://"+publicHostName)
}
<commit_msg>Vary the name of the private service. (#6764)<commit_after>// +build e2e

/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package ingress

import (
	"net/http"
	"testing"

	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/apimachinery/pkg/util/sets"
	"knative.dev/serving/pkg/apis/networking"
	"knative.dev/serving/pkg/apis/networking/v1alpha1"
	"knative.dev/serving/test"
)

func TestVisibility(t *testing.T) {
	t.Parallel()
	clients := test.Setup(t)

	// Create the private backend
	name, port, cancel := CreateRuntimeService(t, clients, networking.ServicePortNameHTTP1)
	defer cancel()

	privateServiceName := test.ObjectNameForTest(t)
	privateHostName := privateServiceName + "." + test.ServingNamespace + ".svc.cluster.local"
	ingress, client, cancel := CreateIngressReady(t, clients, v1alpha1.IngressSpec{
		Rules: []v1alpha1.IngressRule{{
			Hosts:      []string{privateHostName},
			Visibility: v1alpha1.IngressVisibilityClusterLocal,
			HTTP: &v1alpha1.HTTPIngressRuleValue{
				Paths: []v1alpha1.HTTPIngressPath{{
					Splits: []v1alpha1.IngressBackendSplit{{
						IngressBackend: v1alpha1.IngressBackend{
							ServiceName:      name,
							ServiceNamespace: test.ServingNamespace,
							ServicePort:      intstr.FromInt(port),
						},
					}},
				}},
			},
		}},
	})
	defer cancel()

	// Ensure the service is not publicly accessible
	RuntimeRequestWithStatus(t, client, "http://"+privateHostName, sets.NewInt(http.StatusNotFound))

	loadbalancerAddress := ingress.Status.PrivateLoadBalancer.Ingress[0].DomainInternal
	proxyName, proxyPort, cancel := CreateProxyService(t, clients, privateHostName, loadbalancerAddress)
	defer cancel()

	publicHostName := "publicproxy.example.com"
	_, client, cancel = CreateIngressReady(t, clients, v1alpha1.IngressSpec{
		Rules: []v1alpha1.IngressRule{{
			Hosts:      []string{publicHostName},
			Visibility: v1alpha1.IngressVisibilityExternalIP,
			HTTP: &v1alpha1.HTTPIngressRuleValue{
				Paths: []v1alpha1.HTTPIngressPath{{
					Splits: []v1alpha1.IngressBackendSplit{{
						IngressBackend: v1alpha1.IngressBackend{
							ServiceName:      proxyName,
							ServiceNamespace: test.ServingNamespace,
							ServicePort:      intstr.FromInt(proxyPort),
						},
					}},
				}},
			},
		}},
	})
	defer cancel()

	// Ensure the service is accessible from within the cluster.
	RuntimeRequest(t, client, "http://"+publicHostName)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"testing"
	"time"

	"github.com/globalsign/mgo"
	"github.com/globalsign/mgo/bson"
	"github.com/stretchr/testify/require"

	"github.com/feliixx/mgodatagen/config"
)

const (
	URL          = "mongodb://"
	connectError = 1
	configError  = 2
	dateFormat   = "2006-Jan-02"
)

var (
	collections []config.Collection
	d           *datagen
)

type expectedDoc struct {
	ID         bson.ObjectId `bson:"_id"`
	Name       string        `bson:"name"`
	C32        int32         `bson:"c32"`
	C64        int64         `bson:"c64"`
	Float      float64       `bson:"float"`
	Verified   bool          `bson:"verified"`
	Position   []float64     `bson:"position"`
	Dt         string        `bson:"dt"`
	Fake       string        `bson:"faker"`
	Cst        int32         `bson:"cst"`
	Nb         int64         `bson:"nb"`
	Nnb        int32         `bson:"nnb"`
	Date       time.Time     `bson:"date"`
	BinaryData []byte        `bson:"binaryData"`
	List       []int32       `bson:"list"`
	Object     struct {
		K1    string `bson:"k1"`
		K2    int32  `bson:"k2"`
		Subob struct {
			Sk int32 `bson:"s-k"`
		} `bson:"sub-ob"`
	} `bson:"object"`
}

type logger struct{}

// redirect progress bar and non useful info to avoid polluting stderr
func (l logger) Write(b []byte) (int, error) {
	return 0, nil
}

func TestMain(m *testing.M) {
	s, err := mgo.Dial(URL)
	if err != nil {
		fmt.Printf("couldn't connect to db: %v\n", err)
		os.Exit(connectError)
	}
	defer s.Close()
	datagen := &datagen{
		session: s,
		Options: Options{},
		out:     logger{},
	}
	d = datagen

	c, err := config.CollectionList("samples/bson_test.json")
	if err != nil {
		fmt.Printf("error in config file: %v\n", err)
		os.Exit(configError)
	}
	collections = c
	retCode := m.Run()

	err = s.DB(collections[0].DB).DropDatabase()
	if err != nil {
		fmt.Printf("couldn't drop db: %v\n", err)
		os.Exit(connectError)
	}
	os.Exit(retCode)
}

func TestCollectionContent(t *testing.T) {
	assert := require.New(t)

	err := d.createCollection(&collections[0])
	assert.Nil(err)

	err = d.fillCollection(&collections[0])
	assert.Nil(err)

	c := d.session.DB(collections[0].DB).C(collections[0].Name)
	docCount, err := c.Count()
	assert.Nil(err)
	assert.Equal(docCount, int(collections[0].Count))

	var results []expectedDoc
	err = c.Find(nil).All(&results)
	assert.Nil(err)
	count := 0

	fromArr := []string{
		"2012-10-10",
		"2012-12-12",
		"2014-01-01",
		"2016-05-05",
	}

	expectedDate, _ := time.Parse(dateFormat, "2014-Jan-01")

	for i, r := range results {
		// int32
		assert.InDelta(6, len(r.Name), 2)
		// int64
		assert.InDelta(15, r.C32, 5)
		if r.C64 == 0 {
			count++
		} else {
			assert.InDelta(150, r.C64, 50)
		}
		// float
		assert.InDelta(5, r.Float, 5)
		// position testing
		assert.InDelta(0, r.Position[0], 90)
		assert.InDelta(0, r.Position[1], 180)
		// fromArray
		assert.Contains(fromArr, r.Dt)
		// cst
		assert.Equal(int32(2), r.Cst)
		// autoincrement
		assert.Equal(int64(i), r.Nb)
		// Date
		assert.WithinDuration(expectedDate, r.Date, time.Second*60*60*24*365*4)
		// binary data
		assert.Equal(24, len(r.BinaryData))
		// array
		assert.Equal(3, len(r.List))
		// object
		assert.Equal(3, len(r.Object.K1))
		assert.InDelta(-7, r.Object.K2, 3)
		assert.InDelta(5, r.Object.Subob.Sk, 5)
	}
	// null precentage test, allow 2.5% error
	assert.InDelta(100, count, 25)

	var resultStr struct {
		Values []string `bson:"values"`
	}
	// test maxDistinctValues option
	err = d.session.DB(collections[0].DB).Run(bson.D{
		{Name: "distinct", Value: collections[0].Name},
		{Name: "key", Value: "name"},
	}, &resultStr)
	assert.Nil(err)
	assert.Equal(collections[0].Content["name"].MaxDistinctValue, len(resultStr.Values))
	// test unique option
	err = d.session.DB(collections[0].DB).Run(bson.D{
		{Name: "distinct", Value: collections[0].Name},
		{Name: "key", Value: "object.k1"},
	}, &resultStr)
	assert.Nil(err)
	assert.Equal(1000, len(resultStr.Values))

	// test value distribution
	err = d.session.DB(collections[0].DB).Run(bson.D{
		{Name: "distinct", Value: collections[0].Name},
		{Name: "key", Value: "dt"},
	}, &resultStr)
	assert.Nil(err)
	assert.Equal(4, len(resultStr.Values))

	var resultObjectID struct {
		Values []bson.ObjectId `bson:"values"`
	}
	err = d.session.DB(collections[0].DB).Run(bson.D{
		{Name: "distinct", Value: collections[0].Name},
		{Name: "key", Value: "_id"},
	}, &resultObjectID)
	assert.Nil(err)
	assert.Equal(1000, len(resultObjectID.Values))

	var resultInt32 struct {
		Values []int32 `bson:"values"`
	}
	err = d.session.DB(collections[0].DB).Run(bson.D{
		{Name: "distinct", Value: collections[0].Name},
		{Name: "key", Value: "c32"},
	}, &resultInt32)
	assert.Nil(err)
	assert.Equal(11, len(resultInt32.Values))

	err = d.session.DB(collections[0].DB).Run(bson.D{
		{Name: "distinct", Value: collections[0].Name},
		{Name: "key", Value: "list"},
	}, &resultInt32)
	assert.Nil(err)
	assert.Equal(11, len(resultInt32.Values))

	err = d.session.DB(collections[0].DB).Run(bson.D{
		{Name: "distinct", Value: collections[0].Name},
		{Name: "key", Value: "nnb"},
	}, &resultInt32)
	assert.Nil(err)
	assert.Equal(1000, len(resultInt32.Values))

	var resultInt64 struct {
		Values []int64 `bson:"values"`
	}
	err = d.session.DB(collections[0].DB).Run(bson.D{
		{Name: "distinct", Value: collections[0].Name},
		{Name: "key", Value: "c64"},
	}, &resultInt64)
	assert.Nil(err)
	assert.True(len(resultInt64.Values) > 80)

	err = d.session.DB(collections[0].DB).Run(bson.D{
		{Name: "distinct", Value: collections[0].Name},
		{Name: "key", Value: "nb"},
	}, &resultInt64)
	assert.Nil(err)
	assert.Equal(1000, len(resultInt64.Values))

	var resultFloat64 struct {
		Values []float64 `bson:"values"`
	}
	err = d.session.DB(collections[0].DB).Run(bson.D{
		{Name: "distinct", Value: collections[0].Name},
		{Name: "key", Value: "float"},
	}, &resultFloat64)
	assert.Nil(err)
	assert.True(len(resultFloat64.Values) > 800)

	err = d.session.DB(collections[0].DB).Run(bson.D{
		{Name: "distinct", Value: collections[0].Name},
		{Name: "key", Value: "position"},
	}, &resultFloat64)
	assert.Nil(err)
	assert.True(len(resultFloat64.Values) > 1800)

	var resultBool struct {
		Values []bool `bson:"values"`
	}
	err = d.session.DB(collections[0].DB).Run(bson.D{
		{Name: "distinct", Value: collections[0].Name},
		{Name: "key", Value: "verified"},
	}, &resultBool)
	assert.Nil(err)
	assert.Equal(2, len(resultBool.Values))
}

func TestCollectionWithRef(t *testing.T) {
	assert := require.New(t)

	refColl, err := config.CollectionList("samples/config.json")
	assert.Nil(err)

	// TODO : for some reason, the test fails if first collection has more documents
	// than the second collection
	refColl[0].Count = 1000
	refColl[1].Count = 1000

	err = d.createCollection(&refColl[0])
	assert.Nil(err)

	err = d.fillCollection(&refColl[0])
	assert.Nil(err)

	err = d.createCollection(&refColl[1])
	assert.Nil(err)

	err = d.fillCollection(&refColl[1])
	assert.Nil(err)

	var result struct {
		Values []bson.ObjectId `bson:"values"`
		Ok     bool            `bson:"ok"`
	}
	err = d.session.DB(refColl[0].DB).Run(bson.D{
		{Name: "distinct", Value: refColl[0].Name},
		{Name: "key", Value: "_id"},
	}, &result)
	assert.Nil(err)

	c := d.session.DB(refColl[1].DB).C(refColl[1].Name)
	var results []struct {
		ID  bson.ObjectId `bson:"_id"`
		Ref bson.ObjectId `bson:"ref"`
	}
	err = c.Find(nil).All(&results)
	assert.Nil(err)

	for _, doc := range results {
		assert.Contains(result.Values, doc.Ref)
	}
}

func TestCollectionContentWithAggregation(t *testing.T) {
	assert := require.New(t)

	aggColl, err := config.CollectionList("samples/agg.json")
	assert.Nil(err)

	err = d.createCollection(&aggColl[0])
	assert.Nil(err)

	err = d.fillCollection(&aggColl[0])
	assert.Nil(err)

	err = d.createCollection(&aggColl[1])
	assert.Nil(err)

	err = d.fillCollection(&aggColl[1])
	assert.Nil(err)

	c := d.session.DB(aggColl[1].DB).C(aggColl[1].Name)
	var results []bson.M
	err = c.Find(nil).All(&results)
	assert.Nil(err)

	possibleValues := []string{"a", "b", "e", "d", "c", "h", "f", "g", "i"}

	for _, r := range results {
		b := r["AG-CI"].(bson.M)
		assert.InDelta(50, b["m"], 50)
		assert.InDelta(9950, b["M"], 50)

		assert.InDelta(1650, r["AG-FI"], 200)

		a := r["AG-VA"].([]interface{})
		assert.True(len(a) > 0)

		for _, v := range a {
			assert.Contains(possibleValues, v)
		}
	}
}

func TestCreateCollection(t *testing.T) {
	assert := require.New(t)

	collConfig := &config.Collection{
		DB:               collections[0].DB,
		Name:             collections[0].Name,
		Count:            1,
		CompressionLevel: "zlib",
	}

	err := d.createCollection(collConfig)
	assert.Nil(err)

	var result struct {
		WiredTiger struct {
			CreationString string `bson:"creationString"`
		} `bson:"wiredTiger"`
	}
	err = d.session.DB(collections[0].DB).Run(bson.D{{Name: "collStats", Value: collections[0].Name}}, &result)
	assert.Nil(err)
	assert.Contains(result.WiredTiger.CreationString, "block_compressor=zlib")
}

func TestCollectionWithIndexes(t *testing.T) {
	assert := require.New(t)

	err := d.createCollection(&collections[0])
	assert.Nil(err)

	err = d.fillCollection(&collections[0])
	assert.Nil(err)

	indexes := []config.Index{
		{
			Name: "idx_1",
			Key:  bson.M{"c32": 1},
		},
		{
			Name: "idx_2",
			Key:  bson.M{"c64": -1},
		},
	}

	collections[0].Indexes = indexes
	err = d.ensureIndex(&collections[0])
	assert.Nil(err)

	c := d.session.DB(collections[0].DB).C(collections[0].Name)
	idx, err := c.Indexes()
	assert.Nil(err)

	assert.Equal(len(idx), len(indexes)+1)
	assert.Equal(indexes[0].Name, idx[1].Name)
	assert.Equal(indexes[1].Name, idx[2].Name)
}
<commit_msg>fix spelling issue<commit_after>package main

import (
	"fmt"
	"os"
	"testing"
	"time"

	"github.com/globalsign/mgo"
	"github.com/globalsign/mgo/bson"
	"github.com/stretchr/testify/require"

	"github.com/feliixx/mgodatagen/config"
)

const (
	URL          = "mongodb://"
	connectError = 1
	configError  = 2
	dateFormat   = "2006-Jan-02"
)

var (
	collections []config.Collection
	d           *datagen
)

type expectedDoc struct {
	ID         bson.ObjectId `bson:"_id"`
	Name       string        `bson:"name"`
	C32        int32         `bson:"c32"`
	C64        int64         `bson:"c64"`
	Float      float64       `bson:"float"`
	Verified   bool          `bson:"verified"`
	Position   []float64     `bson:"position"`
	Dt         string        `bson:"dt"`
	Fake       string        `bson:"faker"`
	Cst        int32         `bson:"cst"`
	Nb         int64         `bson:"nb"`
	Nnb        int32         `bson:"nnb"`
	Date       time.Time     `bson:"date"`
	BinaryData []byte        `bson:"binaryData"`
	List       []int32       `bson:"list"`
	Object     struct {
		K1    string `bson:"k1"`
		K2    int32  `bson:"k2"`
		Subob struct {
			Sk int32 `bson:"s-k"`
		} `bson:"sub-ob"`
	} `bson:"object"`
}

type logger struct{}

// redirect progress bar and non useful info to avoid polluting stderr
func (l logger) Write(b []byte) (int, error) {
	return 0, nil
}

func TestMain(m *testing.M) {
	s, err := mgo.Dial(URL)
	if err != nil {
		fmt.Printf("couldn't connect to db: %v\n", err)
		os.Exit(connectError)
	}
	defer s.Close()
	datagen := &datagen{
		session: s,
		Options: Options{},
		out:     logger{},
	}
	d = datagen

	c, err := config.CollectionList("samples/bson_test.json")
	if err != nil {
		fmt.Printf("error in config file: %v\n", err)
		os.Exit(configError)
	}
	collections = c
	retCode := m.Run()

	err = s.DB(collections[0].DB).DropDatabase()
	if err != nil {
		fmt.Printf("couldn't drop db: %v\n", err)
		os.Exit(connectError)
	}
	os.Exit(retCode)
}

func TestCollectionContent(t *testing.T) {
	assert := require.New(t)

	err := d.createCollection(&collections[0])
	assert.Nil(err)

	err = d.fillCollection(&collections[0])
	assert.Nil(err)

	c := d.session.DB(collections[0].DB).C(collections[0].Name)
	docCount, err := c.Count()
	assert.Nil(err)
	assert.Equal(docCount, int(collections[0].Count))

	var results []expectedDoc
	err = c.Find(nil).All(&results)
	assert.Nil(err)
	count := 0

	fromArr := []string{
		"2012-10-10",
		"2012-12-12",
		"2014-01-01",
		"2016-05-05",
	}

	expectedDate, _ := time.Parse(dateFormat, "2014-Jan-01")

	for i, r := range results {
		// int32
		assert.InDelta(6, len(r.Name), 2)
		// int64
		assert.InDelta(15, r.C32, 5)
		if r.C64 == 0 {
			count++
		} else {
			assert.InDelta(150, r.C64, 50)
		}
		// float
		assert.InDelta(5, r.Float, 5)
		// position testing
		assert.InDelta(0, r.Position[0], 90)
		assert.InDelta(0, r.Position[1], 180)
		// fromArray
		assert.Contains(fromArr, r.Dt)
		// cst
		assert.Equal(int32(2), r.Cst)
		// autoincrement
		assert.Equal(int64(i), r.Nb)
		// Date
		assert.WithinDuration(expectedDate, r.Date, time.Second*60*60*24*365*4)
		// binary data
		assert.Equal(24, len(r.BinaryData))
		// array
		assert.Equal(3, len(r.List))
		// object
		assert.Equal(3, len(r.Object.K1))
		assert.InDelta(-7, r.Object.K2, 3)
		assert.InDelta(5, r.Object.Subob.Sk, 5)
	}
	// null percentage test, allow 2.5% error
	assert.InDelta(100, count, 25)

	var resultStr struct {
		Values []string `bson:"values"`
	}
	// test maxDistinctValues option
	err = d.session.DB(collections[0].DB).Run(bson.D{
		{Name: "distinct", Value: collections[0].Name},
		{Name: "key", Value: "name"},
	}, &resultStr)
	assert.Nil(err)
	assert.Equal(collections[0].Content["name"].MaxDistinctValue, len(resultStr.Values))
	// test unique option
	err = d.session.DB(collections[0].DB).Run(bson.D{
		{Name: "distinct", Value: collections[0].Name},
		{Name: "key", Value: "object.k1"},
	}, &resultStr)
	assert.Nil(err)
	assert.Equal(1000, len(resultStr.Values))

	// test value distribution
	err = d.session.DB(collections[0].DB).Run(bson.D{
		{Name: "distinct", Value: collections[0].Name},
		{Name: "key", Value: "dt"},
	}, &resultStr)
	assert.Nil(err)
	assert.Equal(4, len(resultStr.Values))

	var resultObjectID struct {
		Values []bson.ObjectId `bson:"values"`
	}
	err = d.session.DB(collections[0].DB).Run(bson.D{
		{Name: "distinct", Value: collections[0].Name},
		{Name: "key", Value: "_id"},
	}, &resultObjectID)
	assert.Nil(err)
	assert.Equal(1000, len(resultObjectID.Values))

	var resultInt32 struct {
		Values []int32 `bson:"values"`
	}
	err = d.session.DB(collections[0].DB).Run(bson.D{
		{Name: "distinct", Value: collections[0].Name},
		{Name: "key", Value: "c32"},
	}, &resultInt32)
	assert.Nil(err)
	assert.Equal(11, len(resultInt32.Values))

	err = d.session.DB(collections[0].DB).Run(bson.D{
		{Name: "distinct", Value: collections[0].Name},
		{Name: "key", Value: "list"},
	}, &resultInt32)
	assert.Nil(err)
	assert.Equal(11, len(resultInt32.Values))

	err = d.session.DB(collections[0].DB).Run(bson.D{
		{Name: "distinct", Value: collections[0].Name},
		{Name: "key", Value: "nnb"},
	}, &resultInt32)
	assert.Nil(err)
	assert.Equal(1000, len(resultInt32.Values))

	var resultInt64 struct {
		Values []int64 `bson:"values"`
	}
	err = d.session.DB(collections[0].DB).Run(bson.D{
		{Name: "distinct", Value: collections[0].Name},
		{Name: "key", Value: "c64"},
	}, &resultInt64)
	assert.Nil(err)
	assert.True(len(resultInt64.Values) > 80)

	err = d.session.DB(collections[0].DB).Run(bson.D{
		{Name: "distinct", Value: collections[0].Name},
		{Name: "key", Value: "nb"},
	}, &resultInt64)
	assert.Nil(err)
	assert.Equal(1000, len(resultInt64.Values))

	var resultFloat64 struct {
		Values []float64 `bson:"values"`
	}
	err = d.session.DB(collections[0].DB).Run(bson.D{
		{Name: "distinct", Value: collections[0].Name},
		{Name: "key", Value: "float"},
	}, &resultFloat64)
	assert.Nil(err)
	assert.True(len(resultFloat64.Values) > 800)

	err = d.session.DB(collections[0].DB).Run(bson.D{
		{Name: "distinct", Value: collections[0].Name},
		{Name: "key", Value: "position"},
	}, &resultFloat64)
	assert.Nil(err)
	assert.True(len(resultFloat64.Values) > 1800)

	var resultBool struct {
		Values []bool `bson:"values"`
	}
	err = d.session.DB(collections[0].DB).Run(bson.D{
		{Name: "distinct", Value: collections[0].Name},
		{Name: "key", Value: "verified"},
	}, &resultBool)
	assert.Nil(err)
	assert.Equal(2, len(resultBool.Values))
}

func TestCollectionWithRef(t *testing.T) {
	assert := require.New(t)

	refColl, err := config.CollectionList("samples/config.json")
	assert.Nil(err)

	// TODO : for some reason, the test fails if first collection has more documents
	// than the second collection
	refColl[0].Count = 1000
	refColl[1].Count = 1000

	err = d.createCollection(&refColl[0])
	assert.Nil(err)

	err = d.fillCollection(&refColl[0])
	assert.Nil(err)

	err = d.createCollection(&refColl[1])
	assert.Nil(err)

	err = d.fillCollection(&refColl[1])
	assert.Nil(err)

	var result struct {
		Values []bson.ObjectId `bson:"values"`
		Ok     bool            `bson:"ok"`
	}
	err = d.session.DB(refColl[0].DB).Run(bson.D{
		{Name: "distinct", Value: refColl[0].Name},
		{Name: "key", Value: "_id"},
	}, &result)
	assert.Nil(err)

	c := d.session.DB(refColl[1].DB).C(refColl[1].Name)
	var results []struct {
		ID  bson.ObjectId `bson:"_id"`
		Ref bson.ObjectId `bson:"ref"`
	}
	err = c.Find(nil).All(&results)
	assert.Nil(err)

	for _, doc := range results {
		assert.Contains(result.Values, doc.Ref)
	}
}

func TestCollectionContentWithAggregation(t *testing.T) {
	assert := require.New(t)

	aggColl, err := config.CollectionList("samples/agg.json")
	assert.Nil(err)

	err = d.createCollection(&aggColl[0])
	assert.Nil(err)

	err = d.fillCollection(&aggColl[0])
	assert.Nil(err)

	err = d.createCollection(&aggColl[1])
	assert.Nil(err)

	err = d.fillCollection(&aggColl[1])
	assert.Nil(err)

	c := d.session.DB(aggColl[1].DB).C(aggColl[1].Name)
	var results []bson.M
	err = c.Find(nil).All(&results)
	assert.Nil(err)

	possibleValues := []string{"a", "b", "e", "d", "c", "h", "f", "g", "i"}

	for _, r := range results {
		b := r["AG-CI"].(bson.M)
		assert.InDelta(50, b["m"], 50)
		assert.InDelta(9950, b["M"], 50)

		assert.InDelta(1650, r["AG-FI"], 200)

		a := r["AG-VA"].([]interface{})
		assert.True(len(a) > 0)

		for _, v := range a {
			assert.Contains(possibleValues, v)
		}
	}
}

func TestCreateCollection(t *testing.T) {
	assert := require.New(t)

	collConfig := &config.Collection{
		DB:               collections[0].DB,
		Name:             collections[0].Name,
		Count:            1,
		CompressionLevel: "zlib",
	}

	err := d.createCollection(collConfig)
	assert.Nil(err)

	var result struct {
		WiredTiger struct {
			CreationString string `bson:"creationString"`
		} `bson:"wiredTiger"`
	}
	err = d.session.DB(collections[0].DB).Run(bson.D{{Name: "collStats", Value: collections[0].Name}}, &result)
	assert.Nil(err)
	assert.Contains(result.WiredTiger.CreationString, "block_compressor=zlib")
}

func TestCollectionWithIndexes(t *testing.T) {
	assert := require.New(t)

	err := d.createCollection(&collections[0])
	assert.Nil(err)

	err = d.fillCollection(&collections[0])
	assert.Nil(err)

	indexes := []config.Index{
		{
			Name: "idx_1",
			Key:  bson.M{"c32": 1},
		},
		{
			Name: "idx_2",
			Key:  bson.M{"c64": -1},
		},
	}

	collections[0].Indexes = indexes
	err = d.ensureIndex(&collections[0])
	assert.Nil(err)

	c := d.session.DB(collections[0].DB).C(collections[0].Name)
	idx, err := c.Indexes()
	assert.Nil(err)

	assert.Equal(len(idx), len(indexes)+1)
	assert.Equal(indexes[0].Name, idx[1].Name)
	assert.Equal(indexes[1].Name, idx[2].Name)
}
<|endoftext|>


<commit_before>package caspercloud

import (
	"encoding/json"
	"fmt"
	"github.com/BigTong/gocounter"
	"github.com/xlvector/dlog"
	"net/http"
	"net/url"
	"runtime/debug"
)

const (
	kInternalErrorResut = "server get internal result"
)

type CasperServer struct {
	cmdCache   *CommandCache
	ct         *gocounter.Counter
	cmdFactory CommandFactory
}

func NewCasperServer(cf CommandFactory) *CasperServer {
	return &CasperServer{
		cmdCache:   NewCommandCache(),
		ct:         gocounter.NewCounter(),
		cmdFactory: cf,
	}
}

func (self *CasperServer) setArgs(cmd Command, params url.Values) *Output {
	args := self.getArgs(params)
	dlog.Println("setArgs:", args)
	cmd.SetInputArgs(args)

	if message := cmd.GetMessage(); message != nil {
		return message
	}
	return nil
}

func (self *CasperServer) getArgs(params url.Values) map[string]string {
	args := make(map[string]string)
	for k, v := range params {
		args[k] = v[0]
	}
	return args
}

func (self *CasperServer) Process(params url.Values) *Output {
	dlog.Info("%s", params.Encode())
	id := params.Get("id")
	if len(id) == 0 {
		c := self.cmdFactory.CreateCommand(params)
		if c == nil {
			return &Output{Status: FAIL, Data: "no create command"}
		}
		self.cmdCache.SetCommand(c)
		params.Set("id", c.GetId())
		return self.setArgs(c, params)
	}

	dlog.Info("get id:%s", id)
	c := self.cmdCache.GetCommand(id)
	if c == nil {
		dlog.Warn("get nil command id:%s", id)
		return &Output{Status: FAIL, Data: "not get command"}
	}

	dlog.Info("get cmd:%s", id)
	ret := self.setArgs(c, params)

	if c.Finished() || ret.Status == FAIL || ret.Status == FINISH_FETCH_DATA || ret.Status == FINISH_ALL {
		c.Successed()
		self.cmdCache.Delete(id)
	}

	return ret
}

func (self *CasperServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	defer func() {
		if r := recover(); r != nil {
			dlog.Println("ERROR: http submit", r)
			debug.PrintStack()
		}
	}()
	self.ct.Incr("request", 1)
	params := req.URL.Query()
	ret := self.Process(params)
	output, _ := json.Marshal(ret)
	fmt.Fprint(w, string(output))
	return
}
<commit_msg>support post<commit_after>package caspercloud

import (
	"encoding/json"
	"fmt"
	"github.com/BigTong/gocounter"
	"github.com/xlvector/dlog"
	"net/http"
	"net/url"
	"runtime/debug"
)

const (
	kInternalErrorResut = "server get internal result"
)

type CasperServer struct {
	cmdCache   *CommandCache
	ct         *gocounter.Counter
	cmdFactory CommandFactory
}

func NewCasperServer(cf CommandFactory) *CasperServer {
	return &CasperServer{
		cmdCache:   NewCommandCache(),
		ct:         gocounter.NewCounter(),
		cmdFactory: cf,
	}
}

func (self *CasperServer) setArgs(cmd Command, params url.Values) *Output {
	args := self.getArgs(params)
	dlog.Println("setArgs:", args)
	cmd.SetInputArgs(args)

	if message := cmd.GetMessage(); message != nil {
		return message
	}
	return nil
}

func (self *CasperServer) getArgs(params url.Values) map[string]string {
	args := make(map[string]string)
	for k, v := range params {
		args[k] = v[0]
	}
	return args
}

func (self *CasperServer) Process(params url.Values) *Output {
	dlog.Info("%s", params.Encode())
	id := params.Get("id")
	if len(id) == 0 {
		c := self.cmdFactory.CreateCommand(params)
		if c == nil {
			return &Output{Status: FAIL, Data: "no create command"}
		}
		self.cmdCache.SetCommand(c)
		params.Set("id", c.GetId())
		return self.setArgs(c, params)
	}

	dlog.Info("get id:%s", id)
	c := self.cmdCache.GetCommand(id)
	if c == nil {
		dlog.Warn("get nil command id:%s", id)
		return &Output{Status: FAIL, Data: "not get command"}
	}

	dlog.Info("get cmd:%s", id)
	ret := self.setArgs(c, params)

	if c.Finished() || ret.Status == FAIL || ret.Status == FINISH_FETCH_DATA || ret.Status == FINISH_ALL {
		c.Successed()
		self.cmdCache.Delete(id)
	}

	return ret
}

func (self *CasperServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	defer func() {
		if r := recover(); r != nil {
			dlog.Println("ERROR: http submit", r)
			debug.PrintStack()
		}
	}()
	self.ct.Incr("request", 1)
	req.ParseForm()
	params := req.Form
	ret := self.Process(params)
	output, _ := json.Marshal(ret)
	fmt.Fprint(w, string(output))
	return
}
<|endoftext|>


<commit_before><commit_msg>providers: improve test coverage<commit_after><|endoftext|>


<commit_before>package main

import (
	"github.com/BellerophonMobile/logberry"
)

func main() {

	logberry.Main.Info("Demo is functional")

	logberry.Main.Info("Computed data", logberry.D{"X": 23, "Label": "priority"})

	logberry.Main.Failure("Arbitrary failure")

	// Wait for all log messages to be output
	logberry.Std.Stop()

}
<commit_msg>No comment.<commit_after>package main

import (
	"github.com/BellerophonMobile/logberry"
)

func main() {

	logberry.Main.Info("Demo is functional")

	logberry.Main.Info("Computed data", logberry.D{"X": 23, "Label": "priority"})

	logberry.Main.Failure("Arbitrary failure")

	logberry.Std.Stop()

}
<|endoftext|>