<commit_before>package integration_test

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/sclevine/agouti/dsl"

	"github.com/t3hmrman/casgo/cas"
	"log"
	"net/http/httptest"
	"testing"
)

// Testing globals for HTTP tests
var testHTTPServer *httptest.Server
var testCASConfig map[string]string
var testCASServer *cas.CAS

func TestCasgoEndToEnd(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "CasGo Integration Suite")
}

var _ = BeforeSuite(func() {
	// Start PhantomJS for integration tests
	StartPhantomJS()

	// Setup CAS server & DB
	testCASConfig, err := cas.NewCASServerConfig("")
	testCASConfig["companyName"] = "Casgo Testing Company"
	testCASConfig["dbName"] = "casgo_test"
	testCASConfig["templatesDirectory"] = "../templates"
	if err != nil {
		log.Fatalf("Failed to generate cas server config, err: %v", err)
	}

	testCASServer, err = cas.NewCASServer(testCASConfig)
	if err != nil {
		log.Fatalf("Failed to generate setup cas server, err: %v", err)
	}
	testCASServer.SetupDb()

	// Setup http test server
	testHTTPServer = httptest.NewServer(testCASServer.ServeMux)

	// Load database fixtures
	testCASServer.Db.LoadJSONFixture(
		testCASServer.Db.GetDbName(),
		testCASServer.Db.GetServicesTableName(),
		"../../fixtures/services.json",
	)
	testCASServer.Db.LoadJSONFixture(
		testCASServer.Db.GetDbName(),
		testCASServer.Db.GetUsersTableName(),
		"../../fixtures/users.json",
	)

})

var _ = AfterSuite(func() {
	testHTTPServer.Close()
	testCASServer.TeardownDb()
	StopWebdriver()
})
<commit_msg>Fix incorrect templates directory<commit_after>package integration_test

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/sclevine/agouti/dsl"

	"github.com/t3hmrman/casgo/cas"
	"log"
	"net/http/httptest"
	"testing"
)

// Testing globals for HTTP tests
var testHTTPServer *httptest.Server
var testCASConfig map[string]string
var testCASServer *cas.CAS

func TestCasgoEndToEnd(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "CasGo Integration Suite")
}

var _ = BeforeSuite(func() {
	// Start PhantomJS for integration tests
	StartPhantomJS()

	// Setup CAS server & DB
	testCASConfig, err := cas.NewCASServerConfig("")
	testCASConfig["companyName"] = "Casgo Testing Company"
	testCASConfig["dbName"] = "casgo_test"
	testCASConfig["templatesDirectory"] = "../../templates"
	if err != nil {
		log.Fatalf("Failed to generate cas server config, err: %v", err)
	}

	testCASServer, err = cas.NewCASServer(testCASConfig)
	if err != nil {
		log.Fatalf("Failed to generate setup cas server, err: %v", err)
	}
	testCASServer.SetupDb()

	// Setup http test server
	testHTTPServer = httptest.NewServer(testCASServer.ServeMux)

	// Load database fixtures
	testCASServer.Db.LoadJSONFixture(
		testCASServer.Db.GetDbName(),
		testCASServer.Db.GetServicesTableName(),
		"../../fixtures/services.json",
	)
	testCASServer.Db.LoadJSONFixture(
		testCASServer.Db.GetDbName(),
		testCASServer.Db.GetUsersTableName(),
		"../../fixtures/users.json",
	)

})

var _ = AfterSuite(func() {
	testHTTPServer.Close()
	testCASServer.TeardownDb()
	StopWebdriver()
})
<|endoftext|>


<commit_before>package mpawsses

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestPrepare_Region(t *testing.T) {
	var p SESPlugin
	p.Region = "us-west-2"
	p.prepare()
	assert.Equal(t, "us-west-2", *p.Svc.Config.Region, "Specified region is used")
}

func TestPrepare_Endpoint(t *testing.T) {
	var p SESPlugin
	p.Endpoint = "https://email.us-west-2.amazonaws.com"
	p.prepare()
	assert.Equal(t, "us-west-2", *p.Svc.Config.Region, "Convert from Endpoint to Specified Region")
}

func TestPrepare_InvalidEndpoint(t *testing.T) {
	var p SESPlugin
	p.Endpoint = "https://email.us-west-2.foobar.com"
	actual := p.prepare()
	expected := "--endpoint is invalid"

	if actual.Error() != expected {
		t.Errorf("Error actual = %v, and Expected = %v.", actual, expected)
	}
}
<commit_msg>added a invalid argument test<commit_after>package mpawsses

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestPrepare_Region(t *testing.T) {
	var p SESPlugin
	p.Region = "us-west-2"
	p.prepare()
	assert.Equal(t, "us-west-2", *p.Svc.Config.Region, "Specified region is used")
}

func TestPrepare_Endpoint(t *testing.T) {
	var p SESPlugin
	p.Endpoint = "https://email.us-west-2.amazonaws.com"
	p.prepare()
	assert.Equal(t, "us-west-2", *p.Svc.Config.Region, "Convert from Endpoint to Specified Region")
}

func TestPrepare_InvalidEndpoint(t *testing.T) {
	var p SESPlugin
	p.Endpoint = "https://email.us-west-2.foobar.com"
	actual := p.prepare()
	expected := "--endpoint is invalid"

	if actual.Error() != expected {
		t.Errorf("Error actual = %v, and Expected = %v.", actual, expected)
	}
}

func TestPrepare_InvalidEndpoint2(t *testing.T) {
	var p SESPlugin
	p.Endpoint = "https://email.us-west-2.foobar.bazqux.com"
	actual := p.prepare()
	expected := "--endpoint is invalid"

	if actual.Error() != expected {
		t.Errorf("Error actual = %v, and Expected = %v.", actual, expected)
	}
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"io/ioutil"
	"os"

	"github.com/concourse/atc"
	"github.com/concourse/atc/config"
	"github.com/concourse/fly/commands/internal/displayhelpers"
	"github.com/concourse/fly/template"
	"github.com/mitchellh/mapstructure"
	"gopkg.in/yaml.v2"
)

type ValidatePipelineCommand struct {
	Config atc.PathFlag `short:"c" long:"config" required:"true"        description:"Pipeline configuration file"`
	Strict bool         `short:"s" long:"strict"                        description:"Fail on warnings"`
}

func (command *ValidatePipelineCommand) Execute(args []string) error {
	configPath := command.Config
	strict := command.Strict

	configFile, err := ioutil.ReadFile(string(configPath))
	if err != nil {
		displayhelpers.FailWithErrorf("could not read config file", err)
	}

	configFile = template.EvaluateEmpty(configFile)

	var configStructure interface{}
	err = yaml.Unmarshal(configFile, &configStructure)
	if err != nil {
		displayhelpers.FailWithErrorf("failed to unmarshal configStructure", err)
	}

	var newConfig atc.Config
	msConfig := &mapstructure.DecoderConfig{
		Result:           &newConfig,
		WeaklyTypedInput: true,
		DecodeHook: mapstructure.ComposeDecodeHookFunc(
			atc.SanitizeDecodeHook,
			atc.VersionConfigDecodeHook,
		),
	}

	decoder, err := mapstructure.NewDecoder(msConfig)
	if err != nil {
		displayhelpers.FailWithErrorf("failed to construct decoder", err)
	}

	if err := decoder.Decode(configStructure); err != nil {
		displayhelpers.FailWithErrorf("failed to decode config", err)
	}

	warnings, errorMessages := config.ValidateConfig(newConfig)

	if len(warnings) > 0 {
		fmt.Fprintln(os.Stderr, "")
		displayhelpers.PrintDeprecationWarningHeader()

		for _, warning := range warnings {
			fmt.Fprintf(os.Stderr, "  - %s\n", warning.Message)
		}

		fmt.Fprintln(os.Stderr, "")
	}

	if len(errorMessages) > 0 {
		fmt.Fprintln(os.Stderr, "")
		displayhelpers.PrintWarningHeader()

		for _, errorMessage := range errorMessages {
			fmt.Fprintf(os.Stderr, "  - %s\n", errorMessage)
		}

		fmt.Fprintln(os.Stderr, "")
	}

	if len(errorMessages) > 0 || (strict && len(warnings) > 0) {
		displayhelpers.Failf("configuration invalid")
	}

	fmt.Println("looks good")
	return nil
}
<commit_msg>Use refactored validate method.<commit_after>package commands

import (
	"fmt"
	"io/ioutil"
	"os"

	"github.com/concourse/atc"
	"github.com/concourse/fly/commands/internal/displayhelpers"
	"github.com/concourse/fly/template"
	"github.com/mitchellh/mapstructure"
	"gopkg.in/yaml.v2"
)

type ValidatePipelineCommand struct {
	Config atc.PathFlag `short:"c" long:"config" required:"true"        description:"Pipeline configuration file"`
	Strict bool         `short:"s" long:"strict"                        description:"Fail on warnings"`
}

func (command *ValidatePipelineCommand) Execute(args []string) error {
	configPath := command.Config
	strict := command.Strict

	configFile, err := ioutil.ReadFile(string(configPath))
	if err != nil {
		displayhelpers.FailWithErrorf("could not read config file", err)
	}

	configFile = template.EvaluateEmpty(configFile)

	var configStructure interface{}
	err = yaml.Unmarshal(configFile, &configStructure)
	if err != nil {
		displayhelpers.FailWithErrorf("failed to unmarshal configStructure", err)
	}

	var newConfig atc.Config
	msConfig := &mapstructure.DecoderConfig{
		Result:           &newConfig,
		WeaklyTypedInput: true,
		DecodeHook: mapstructure.ComposeDecodeHookFunc(
			atc.SanitizeDecodeHook,
			atc.VersionConfigDecodeHook,
		),
	}

	decoder, err := mapstructure.NewDecoder(msConfig)
	if err != nil {
		displayhelpers.FailWithErrorf("failed to construct decoder", err)
	}

	if err := decoder.Decode(configStructure); err != nil {
		displayhelpers.FailWithErrorf("failed to decode config", err)
	}

	warnings, errorMessages := newConfig.Validate()

	if len(warnings) > 0 {
		fmt.Fprintln(os.Stderr, "")
		displayhelpers.PrintDeprecationWarningHeader()

		for _, warning := range warnings {
			fmt.Fprintf(os.Stderr, "  - %s\n", warning.Message)
		}

		fmt.Fprintln(os.Stderr, "")
	}

	if len(errorMessages) > 0 {
		fmt.Fprintln(os.Stderr, "")
		displayhelpers.PrintWarningHeader()

		for _, errorMessage := range errorMessages {
			fmt.Fprintf(os.Stderr, "  - %s\n", errorMessage)
		}

		fmt.Fprintln(os.Stderr, "")
	}

	if len(errorMessages) > 0 || (strict && len(warnings) > 0) {
		displayhelpers.Failf("configuration invalid")
	}

	fmt.Println("looks good")
	return nil
}
<|endoftext|>


<commit_before>package fire

import (
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/256dpi/xo"
	"github.com/stretchr/testify/assert"
)

func panicReporter(err error) {
	panic(err)
}

func TestGroupAdd(t *testing.T) {
	withTester(t, func(t *testing.T, tester *Tester) {
		group := NewGroup(panicReporter)

		group.Add(&Controller{
			Model: &postModel{},
			Store: tester.Store,
			Authorizers: L{
				C("Panic", All(), func(*Context) error {
					xo.Abort(xo.F("foo"))
					return nil
				}),
			},
		})

		assert.PanicsWithValue(t, `fire: controller with name "posts" already exists`, func() {
			group.Add(&Controller{
				Model: &postModel{},
			})
		})
	})
}

func TestGroupEndpointMissingResource(t *testing.T) {
	withTester(t, func(t *testing.T, tester *Tester) {
		tester.Handler = NewGroup(panicReporter).Endpoint("api")

		tester.Request("GET", "api/", "", func(r *httptest.ResponseRecorder, rq *http.Request) {
			assert.Equal(t, http.StatusNotFound, r.Result().StatusCode)
		})

		tester.Request("GET", "api/foo", "", func(r *httptest.ResponseRecorder, rq *http.Request) {
			assert.Equal(t, http.StatusNotFound, r.Result().StatusCode)
		})
	})
}

func TestGroupAbort(t *testing.T) {
	withTester(t, func(t *testing.T, tester *Tester) {
		var lastErr error

		group := NewGroup(func(err error) {
			assert.Equal(t, "foo", err.Error())
			lastErr = err
		})

		group.Add(&Controller{
			Model: &postModel{},
			Store: tester.Store,
			Authorizers: L{
				C("Panic", All(), func(*Context) error {
					xo.Abort(xo.F("foo"))
					return nil
				}),
			},
		})

		tester.Handler = group.Endpoint("")

		tester.Request("GET", "posts", "", func(r *httptest.ResponseRecorder, rq *http.Request) {
			assert.Equal(t, http.StatusInternalServerError, r.Result().StatusCode)
			assert.JSONEq(t, `{
				"errors": [{
					"status": "500",
					"title": "internal server error"
				}]
			}`, r.Body.String())
		})

		assert.Error(t, lastErr)
	})
}

func TestGroupPanic(t *testing.T) {
	withTester(t, func(t *testing.T, tester *Tester) {
		var lastErr error

		group := NewGroup(func(err error) {
			assert.Equal(t, "PANIC: foo", err.Error())
			lastErr = err
		})

		group.Add(&Controller{
			Model: &postModel{},
			Store: tester.Store,
			Authorizers: L{
				C("Panic", All(), func(*Context) error {
					panic("foo")
					return nil
				}),
			},
		})

		tester.Handler = group.Endpoint("")

		tester.Request("GET", "posts", "", func(r *httptest.ResponseRecorder, rq *http.Request) {
			assert.Equal(t, http.StatusInternalServerError, r.Result().StatusCode)
			assert.JSONEq(t, `{
				"errors": [{
					"status": "500",
					"title": "internal server error"
				}]
			}`, r.Body.String())
		})

		assert.Error(t, lastErr)
	})
}

func TestGroupAction(t *testing.T) {
	withTester(t, func(t *testing.T, tester *Tester) {
		group := NewGroup(panicReporter)

		group.Handle("foo", &GroupAction{
			Authorizers: L{
				C("TestGroupAction", All(), func(ctx *Context) error {
					if ctx.HTTPRequest.Method == "DELETE" {
						return xo.SF("error")
					}
					return nil
				}),
			},
			Action: A("TestGroupAction", []string{"GET", "PUT", "DELETE"}, 0, func(ctx *Context) error {
				ctx.ResponseWriter.WriteHeader(http.StatusFound)
				return nil
			}),
		})

		assert.PanicsWithValue(t, `fire: invalid group action ""`, func() {
			group.Handle("", &GroupAction{
				Action: &Action{},
			})
		})

		assert.PanicsWithValue(t, `fire: action with name "foo" already exists`, func() {
			group.Handle("foo", &GroupAction{
				Action: &Action{},
			})
		})

		tester.Handler = group.Endpoint("")

		tester.Request("GET", "foo", "", func(r *httptest.ResponseRecorder, rq *http.Request) {
			assert.Equal(t, http.StatusFound, r.Result().StatusCode)
		})

		tester.Request("POST", "foo", "", func(r *httptest.ResponseRecorder, rq *http.Request) {
			assert.Equal(t, http.StatusNotFound, r.Result().StatusCode)
		})

		tester.Request("PUT", "bar", "", func(r *httptest.ResponseRecorder, rq *http.Request) {
			assert.Equal(t, http.StatusNotFound, r.Result().StatusCode)
		})

		tester.Request("DELETE", "foo", "", func(r *httptest.ResponseRecorder, rq *http.Request) {
			assert.Equal(t, http.StatusUnauthorized, r.Result().StatusCode)
		})
	})
}
<commit_msg>fix vet<commit_after>package fire

import (
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/256dpi/xo"
	"github.com/stretchr/testify/assert"
)

func panicReporter(err error) {
	panic(err)
}

func TestGroupAdd(t *testing.T) {
	withTester(t, func(t *testing.T, tester *Tester) {
		group := NewGroup(panicReporter)

		group.Add(&Controller{
			Model: &postModel{},
			Store: tester.Store,
			Authorizers: L{
				C("Panic", All(), func(*Context) error {
					xo.Abort(xo.F("foo"))
					return nil
				}),
			},
		})

		assert.PanicsWithValue(t, `fire: controller with name "posts" already exists`, func() {
			group.Add(&Controller{
				Model: &postModel{},
			})
		})
	})
}

func TestGroupEndpointMissingResource(t *testing.T) {
	withTester(t, func(t *testing.T, tester *Tester) {
		tester.Handler = NewGroup(panicReporter).Endpoint("api")

		tester.Request("GET", "api/", "", func(r *httptest.ResponseRecorder, rq *http.Request) {
			assert.Equal(t, http.StatusNotFound, r.Result().StatusCode)
		})

		tester.Request("GET", "api/foo", "", func(r *httptest.ResponseRecorder, rq *http.Request) {
			assert.Equal(t, http.StatusNotFound, r.Result().StatusCode)
		})
	})
}

func TestGroupAbort(t *testing.T) {
	withTester(t, func(t *testing.T, tester *Tester) {
		var lastErr error

		group := NewGroup(func(err error) {
			assert.Equal(t, "foo", err.Error())
			lastErr = err
		})

		group.Add(&Controller{
			Model: &postModel{},
			Store: tester.Store,
			Authorizers: L{
				C("Panic", All(), func(*Context) error {
					xo.Abort(xo.F("foo"))
					return nil
				}),
			},
		})

		tester.Handler = group.Endpoint("")

		tester.Request("GET", "posts", "", func(r *httptest.ResponseRecorder, rq *http.Request) {
			assert.Equal(t, http.StatusInternalServerError, r.Result().StatusCode)
			assert.JSONEq(t, `{
				"errors": [{
					"status": "500",
					"title": "internal server error"
				}]
			}`, r.Body.String())
		})

		assert.Error(t, lastErr)
	})
}

func TestGroupPanic(t *testing.T) {
	withTester(t, func(t *testing.T, tester *Tester) {
		var lastErr error

		group := NewGroup(func(err error) {
			assert.Equal(t, "PANIC: foo", err.Error())
			lastErr = err
		})

		group.Add(&Controller{
			Model: &postModel{},
			Store: tester.Store,
			Authorizers: L{
				C("Panic", All(), func(*Context) error {
					panic("foo")
				}),
			},
		})

		tester.Handler = group.Endpoint("")

		tester.Request("GET", "posts", "", func(r *httptest.ResponseRecorder, rq *http.Request) {
			assert.Equal(t, http.StatusInternalServerError, r.Result().StatusCode)
			assert.JSONEq(t, `{
				"errors": [{
					"status": "500",
					"title": "internal server error"
				}]
			}`, r.Body.String())
		})

		assert.Error(t, lastErr)
	})
}

func TestGroupAction(t *testing.T) {
	withTester(t, func(t *testing.T, tester *Tester) {
		group := NewGroup(panicReporter)

		group.Handle("foo", &GroupAction{
			Authorizers: L{
				C("TestGroupAction", All(), func(ctx *Context) error {
					if ctx.HTTPRequest.Method == "DELETE" {
						return xo.SF("error")
					}
					return nil
				}),
			},
			Action: A("TestGroupAction", []string{"GET", "PUT", "DELETE"}, 0, func(ctx *Context) error {
				ctx.ResponseWriter.WriteHeader(http.StatusFound)
				return nil
			}),
		})

		assert.PanicsWithValue(t, `fire: invalid group action ""`, func() {
			group.Handle("", &GroupAction{
				Action: &Action{},
			})
		})

		assert.PanicsWithValue(t, `fire: action with name "foo" already exists`, func() {
			group.Handle("foo", &GroupAction{
				Action: &Action{},
			})
		})

		tester.Handler = group.Endpoint("")

		tester.Request("GET", "foo", "", func(r *httptest.ResponseRecorder, rq *http.Request) {
			assert.Equal(t, http.StatusFound, r.Result().StatusCode)
		})

		tester.Request("POST", "foo", "", func(r *httptest.ResponseRecorder, rq *http.Request) {
			assert.Equal(t, http.StatusNotFound, r.Result().StatusCode)
		})

		tester.Request("PUT", "bar", "", func(r *httptest.ResponseRecorder, rq *http.Request) {
			assert.Equal(t, http.StatusNotFound, r.Result().StatusCode)
		})

		tester.Request("DELETE", "foo", "", func(r *httptest.ResponseRecorder, rq *http.Request) {
			assert.Equal(t, http.StatusUnauthorized, r.Result().StatusCode)
		})
	})
}
<|endoftext|>


<commit_before>package vizzini_test

import (
	"flag"
	"fmt"
	"log"
	"os"

	"github.com/nu7hatch/gouuid"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"testing"
	"time"

	"github.com/cloudfoundry-incubator/receptor"
)

var client receptor.Client
var domain string
var otherDomain string
var stack string
var guid string
var startTime time.Time

var receptorAddress string
var routableDomainSuffix string
var hostAddress string

func init() {
	flag.StringVar(&receptorAddress, "receptor-address", "http://receptor.10.244.0.34.xip.io", "http address for the receptor (required)")
	flag.StringVar(&routableDomainSuffix, "routable-domain-suffix", "10.244.0.34.xip.io", "suffix to use when constructing FQDN")
	flag.StringVar(&hostAddress, "host-address", "10.0.2.2", "address that a process running in a container on Diego can use to reach the machine running this test.  Typically the gateway on the vagrant VM.")
	flag.Parse()

	if receptorAddress == "" {
		log.Fatal("i need a receptor-address to talk to Diego...")
	}
}

func TestReceptorSuite(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "ReceptorSuite Suite")
}

func NewGuid() string {
	u, err := uuid.NewV4()
	Ω(err).ShouldNot(HaveOccurred())
	return domain + "-" + u.String()[:8]
}

var _ = BeforeSuite(func() {
	timeout := os.Getenv("DEFAULT_EVENTUALLY_TIMEOUT")
	if timeout == "" {
		SetDefaultEventuallyTimeout(10 * time.Second)
	} else {
		duration, err := time.ParseDuration(timeout)
		Ω(err).ShouldNot(HaveOccurred(), "invalid timeout")
		fmt.Printf("Setting Default Eventually Timeout to %s\n", duration)
		SetDefaultEventuallyTimeout(duration)
	}
	SetDefaultEventuallyPollingInterval(500 * time.Millisecond)
	SetDefaultConsistentlyPollingInterval(200 * time.Millisecond)
	domain = fmt.Sprintf("vizzini-%d", GinkgoParallelNode())
	otherDomain = fmt.Sprintf("vizzini-other-%d", GinkgoParallelNode())
	stack = "lucid64"

	client = receptor.NewClient(receptorAddress)
})

var _ = BeforeEach(func() {
	startTime = time.Now()
	guid = NewGuid()
})

var _ = AfterEach(func() {
	for _, domain := range []string{domain, otherDomain} {
		ClearOutTasksInDomain(domain)
		ClearOutDesiredLRPsInDomain(domain)
	}

	endTime := time.Now()
	fmt.Fprintf(GinkgoWriter, "%s\nThis test referenced GUID %s\nStart time: %s (%d)\nEnd time: %s (%d)\n", CurrentGinkgoTestDescription().FullTestText, guid, startTime, startTime.Unix(), endTime, endTime.Unix())
})

var _ = AfterSuite(func() {
	for _, domain := range []string{domain, otherDomain} {
		client.UpsertDomain(domain, 5*time.Minute) //leave the domain around forever so that Diego cleans up if need be
	}

	for _, domain := range []string{domain, otherDomain} {
		ClearOutDesiredLRPsInDomain(domain)
		ClearOutTasksInDomain(domain)
	}
})
<commit_msg>cyan makes everything better<commit_after>package vizzini_test

import (
	"flag"
	"fmt"
	"log"
	"os"

	"github.com/onsi/say"

	"github.com/nu7hatch/gouuid"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"testing"
	"time"

	"github.com/cloudfoundry-incubator/receptor"
)

var client receptor.Client
var domain string
var otherDomain string
var stack string
var guid string
var startTime time.Time

var receptorAddress string
var routableDomainSuffix string
var hostAddress string

func init() {
	flag.StringVar(&receptorAddress, "receptor-address", "http://receptor.10.244.0.34.xip.io", "http address for the receptor (required)")
	flag.StringVar(&routableDomainSuffix, "routable-domain-suffix", "10.244.0.34.xip.io", "suffix to use when constructing FQDN")
	flag.StringVar(&hostAddress, "host-address", "10.0.2.2", "address that a process running in a container on Diego can use to reach the machine running this test.  Typically the gateway on the vagrant VM.")
	flag.Parse()

	if receptorAddress == "" {
		log.Fatal("i need a receptor-address to talk to Diego...")
	}
}

func TestReceptorSuite(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "ReceptorSuite Suite")
}

func NewGuid() string {
	u, err := uuid.NewV4()
	Ω(err).ShouldNot(HaveOccurred())
	return domain + "-" + u.String()[:8]
}

var _ = BeforeSuite(func() {
	timeout := os.Getenv("DEFAULT_EVENTUALLY_TIMEOUT")
	if timeout == "" {
		SetDefaultEventuallyTimeout(10 * time.Second)
	} else {
		duration, err := time.ParseDuration(timeout)
		Ω(err).ShouldNot(HaveOccurred(), "invalid timeout")
		fmt.Printf("Setting Default Eventually Timeout to %s\n", duration)
		SetDefaultEventuallyTimeout(duration)
	}
	SetDefaultEventuallyPollingInterval(500 * time.Millisecond)
	SetDefaultConsistentlyPollingInterval(200 * time.Millisecond)
	domain = fmt.Sprintf("vizzini-%d", GinkgoParallelNode())
	otherDomain = fmt.Sprintf("vizzini-other-%d", GinkgoParallelNode())
	stack = "lucid64"

	client = receptor.NewClient(receptorAddress)
})

var _ = BeforeEach(func() {
	startTime = time.Now()
	guid = NewGuid()
})

var _ = AfterEach(func() {
	for _, domain := range []string{domain, otherDomain} {
		ClearOutTasksInDomain(domain)
		ClearOutDesiredLRPsInDomain(domain)
	}

	endTime := time.Now()
	fmt.Fprint(GinkgoWriter, say.Cyan("\n%s\nThis test referenced GUID %s\nStart time: %s (%d)\nEnd time: %s (%d)\n", CurrentGinkgoTestDescription().FullTestText, guid, startTime, startTime.Unix(), endTime, endTime.Unix()))
})

var _ = AfterSuite(func() {
	for _, domain := range []string{domain, otherDomain} {
		client.UpsertDomain(domain, 5*time.Minute) //leave the domain around forever so that Diego cleans up if need be
	}

	for _, domain := range []string{domain, otherDomain} {
		ClearOutDesiredLRPsInDomain(domain)
		ClearOutTasksInDomain(domain)
	}
})
<|endoftext|>


<commit_before>package restic_test

import (
	"bytes"
	"flag"
	"io"
	"math/rand"
	"testing"

	"github.com/restic/restic"
	"github.com/restic/restic/backend"
	"github.com/restic/restic/chunker"
)

var benchArchiveDirectory = flag.String("test.benchdir", ".", "benchmark archiving a real directory (default: .)")

func get_random(seed, count int) []byte {
	buf := make([]byte, count)

	rnd := rand.New(rand.NewSource(int64(seed)))
	for i := 0; i < count; i++ {
		buf[i] = byte(rnd.Uint32())
	}

	return buf
}

func randomReader(seed, size int) *bytes.Reader {
	return bytes.NewReader(get_random(seed, size))
}

const bufSize = chunker.MiB

type Rdr interface {
	io.ReadSeeker
	io.ReaderAt
}

func benchmarkChunkEncrypt(b testing.TB, buf []byte, rd Rdr, key *restic.Key) {
	ch := restic.GetChunker("BenchmarkChunkEncrypt")
	rd.Seek(0, 0)
	ch.Reset(rd)

	for {
		chunk, err := ch.Next()

		if err == io.EOF {
			break
		}

		ok(b, err)

		// reduce length of buf
		buf = buf[:chunk.Length]
		n, err := io.ReadFull(chunk.Reader(rd), buf)
		ok(b, err)
		assert(b, uint(n) == chunk.Length, "invalid length: got %d, expected %d", n, chunk.Length)

		_, err = key.Encrypt(buf, buf)
		ok(b, err)
	}

	restic.FreeChunker("BenchmarkChunkEncrypt", ch)
}

func BenchmarkChunkEncrypt(b *testing.B) {
	data := get_random(23, 10<<20) // 10MiB
	rd := bytes.NewReader(data)

	be := setupBackend(b)
	defer teardownBackend(b, be)
	key := setupKey(b, be, "geheim")

	buf := restic.GetChunkBuf("BenchmarkChunkEncrypt")

	b.ResetTimer()
	b.SetBytes(int64(len(data)))

	for i := 0; i < b.N; i++ {
		benchmarkChunkEncrypt(b, buf, rd, key)
	}

	restic.FreeChunkBuf("BenchmarkChunkEncrypt", buf)
}

func benchmarkChunkEncryptP(b *testing.PB, buf []byte, rd Rdr, key *restic.Key) {
	ch := restic.GetChunker("BenchmarkChunkEncryptP")
	rd.Seek(0, 0)
	ch.Reset(rd)

	for {
		chunk, err := ch.Next()
		if err == io.EOF {
			break
		}

		// reduce length of chunkBuf
		buf = buf[:chunk.Length]
		io.ReadFull(chunk.Reader(rd), buf)
		key.Encrypt(buf, buf)
	}

	restic.FreeChunker("BenchmarkChunkEncryptP", ch)
}

func BenchmarkChunkEncryptParallel(b *testing.B) {
	be := setupBackend(b)
	defer teardownBackend(b, be)
	key := setupKey(b, be, "geheim")

	data := get_random(23, 10<<20) // 10MiB

	buf := restic.GetChunkBuf("BenchmarkChunkEncryptParallel")

	b.ResetTimer()
	b.SetBytes(int64(len(data)))

	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			rd := bytes.NewReader(data)
			benchmarkChunkEncryptP(pb, buf, rd, key)
		}
	})

	restic.FreeChunkBuf("BenchmarkChunkEncryptParallel", buf)
}

func BenchmarkArchiveDirectory(b *testing.B) {
	if *benchArchiveDirectory == "" {
		b.Skip("benchdir not set, skipping BenchmarkArchiveDirectory")
	}

	be := setupBackend(b)
	defer teardownBackend(b, be)
	key := setupKey(b, be, "geheim")
	server := restic.NewServerWithKey(be, key)

	arch, err := restic.NewArchiver(server, nil)
	ok(b, err)

	_, id, err := arch.Snapshot(*benchArchiveDirectory, nil)

	b.Logf("snapshot archived as %v", id)
}

func snapshot(t *testing.T, server restic.Server, path string) *restic.Snapshot {
	arch, err := restic.NewArchiver(server, nil)
	ok(t, err)
	ok(t, arch.Preload())
	sn, _, err := arch.Snapshot(path, nil)
	ok(t, err)
	return sn
}

func countBlobs(t *testing.T, server restic.Server) int {
	blobs := 0
	err := server.EachID(backend.Tree, func(id backend.ID) {
		tree, err := restic.LoadTree(server, id)
		ok(t, err)

		blobs += tree.Map.Len()
	})
	ok(t, err)

	return blobs
}

func TestArchiverPreload(t *testing.T) {
	be := setupBackend(t)
	defer teardownBackend(t, be)
	key := setupKey(t, be, "geheim")
	server := restic.NewServerWithKey(be, key)

	// archive a few files
	sn := snapshot(t, server, *benchArchiveDirectory)
	t.Logf("archived snapshot %v", sn.ID)

	// get archive stats
	blobsBefore := countBlobs(t, server)
	t.Logf("found %v blobs", blobsBefore)

	// archive the same files again
	sn2 := snapshot(t, server, *benchArchiveDirectory)
	t.Logf("archived snapshot %v", sn2.ID)

	// get archive stats
	blobsAfter := countBlobs(t, server)
	t.Logf("found %v blobs", blobsAfter)

	// if there are more than 10% more blobs, something is wrong
	if blobsAfter > (blobsBefore + blobsBefore/10) {
		t.Fatalf("TestArchiverPreload: too many blobs in repository: before %d, after %d, threshhold %d",
			blobsBefore, blobsAfter, (blobsBefore + blobsBefore/10))
	}
}
<commit_msg>Do not run archiver tests by default<commit_after>package restic_test

import (
	"bytes"
	"flag"
	"io"
	"math/rand"
	"testing"

	"github.com/restic/restic"
	"github.com/restic/restic/backend"
	"github.com/restic/restic/chunker"
)

var benchArchiveDirectory = flag.String("test.benchdir", "", "benchmark archiving a real directory")

func get_random(seed, count int) []byte {
	buf := make([]byte, count)

	rnd := rand.New(rand.NewSource(int64(seed)))
	for i := 0; i < count; i++ {
		buf[i] = byte(rnd.Uint32())
	}

	return buf
}

func randomReader(seed, size int) *bytes.Reader {
	return bytes.NewReader(get_random(seed, size))
}

const bufSize = chunker.MiB

type Rdr interface {
	io.ReadSeeker
	io.ReaderAt
}

func benchmarkChunkEncrypt(b testing.TB, buf []byte, rd Rdr, key *restic.Key) {
	ch := restic.GetChunker("BenchmarkChunkEncrypt")
	rd.Seek(0, 0)
	ch.Reset(rd)

	for {
		chunk, err := ch.Next()

		if err == io.EOF {
			break
		}

		ok(b, err)

		// reduce length of buf
		buf = buf[:chunk.Length]
		n, err := io.ReadFull(chunk.Reader(rd), buf)
		ok(b, err)
		assert(b, uint(n) == chunk.Length, "invalid length: got %d, expected %d", n, chunk.Length)

		_, err = key.Encrypt(buf, buf)
		ok(b, err)
	}

	restic.FreeChunker("BenchmarkChunkEncrypt", ch)
}

func BenchmarkChunkEncrypt(b *testing.B) {
	data := get_random(23, 10<<20) // 10MiB
	rd := bytes.NewReader(data)

	be := setupBackend(b)
	defer teardownBackend(b, be)
	key := setupKey(b, be, "geheim")

	buf := restic.GetChunkBuf("BenchmarkChunkEncrypt")

	b.ResetTimer()
	b.SetBytes(int64(len(data)))

	for i := 0; i < b.N; i++ {
		benchmarkChunkEncrypt(b, buf, rd, key)
	}

	restic.FreeChunkBuf("BenchmarkChunkEncrypt", buf)
}

func benchmarkChunkEncryptP(b *testing.PB, buf []byte, rd Rdr, key *restic.Key) {
	ch := restic.GetChunker("BenchmarkChunkEncryptP")
	rd.Seek(0, 0)
	ch.Reset(rd)

	for {
		chunk, err := ch.Next()
		if err == io.EOF {
			break
		}

		// reduce length of chunkBuf
		buf = buf[:chunk.Length]
		io.ReadFull(chunk.Reader(rd), buf)
		key.Encrypt(buf, buf)
	}

	restic.FreeChunker("BenchmarkChunkEncryptP", ch)
}

func BenchmarkChunkEncryptParallel(b *testing.B) {
	be := setupBackend(b)
	defer teardownBackend(b, be)
	key := setupKey(b, be, "geheim")

	data := get_random(23, 10<<20) // 10MiB

	buf := restic.GetChunkBuf("BenchmarkChunkEncryptParallel")

	b.ResetTimer()
	b.SetBytes(int64(len(data)))

	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			rd := bytes.NewReader(data)
			benchmarkChunkEncryptP(pb, buf, rd, key)
		}
	})

	restic.FreeChunkBuf("BenchmarkChunkEncryptParallel", buf)
}

func BenchmarkArchiveDirectory(b *testing.B) {
	if *benchArchiveDirectory == "" {
		b.Skip("benchdir not set, skipping BenchmarkArchiveDirectory")
	}

	be := setupBackend(b)
	defer teardownBackend(b, be)
	key := setupKey(b, be, "geheim")
	server := restic.NewServerWithKey(be, key)

	arch, err := restic.NewArchiver(server, nil)
	ok(b, err)

	_, id, err := arch.Snapshot(*benchArchiveDirectory, nil)

	b.Logf("snapshot archived as %v", id)
}

func snapshot(t *testing.T, server restic.Server, path string) *restic.Snapshot {
	arch, err := restic.NewArchiver(server, nil)
	ok(t, err)
	ok(t, arch.Preload())
	sn, _, err := arch.Snapshot(path, nil)
	ok(t, err)
	return sn
}

func countBlobs(t *testing.T, server restic.Server) int {
	blobs := 0
	err := server.EachID(backend.Tree, func(id backend.ID) {
		tree, err := restic.LoadTree(server, id)
		ok(t, err)

		blobs += tree.Map.Len()
	})
	ok(t, err)

	return blobs
}

func TestArchiverPreload(t *testing.T) {
	be := setupBackend(t)
	defer teardownBackend(t, be)
	key := setupKey(t, be, "geheim")
	server := restic.NewServerWithKey(be, key)

	// archive a few files
	sn := snapshot(t, server, *benchArchiveDirectory)
	t.Logf("archived snapshot %v", sn.ID)

	// get archive stats
	blobsBefore := countBlobs(t, server)
	t.Logf("found %v blobs", blobsBefore)

	// archive the same files again
	sn2 := snapshot(t, server, *benchArchiveDirectory)
	t.Logf("archived snapshot %v", sn2.ID)

	// get archive stats
	blobsAfter := countBlobs(t, server)
	t.Logf("found %v blobs", blobsAfter)

	// if there are more than 10% more blobs, something is wrong
	if blobsAfter > (blobsBefore + blobsBefore/10) {
		t.Fatalf("TestArchiverPreload: too many blobs in repository: before %d, after %d, threshhold %d",
			blobsBefore, blobsAfter, (blobsBefore + blobsBefore/10))
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Dorival de Moraes Pedroso. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package goga

import (
	"bytes"
	"math"

	"github.com/cpmech/gosl/chk"
	"github.com/cpmech/gosl/io"
	"github.com/cpmech/gosl/la"
	"github.com/cpmech/gosl/plt"
	"github.com/cpmech/gosl/rnd"
	"github.com/cpmech/gosl/utl"
)

// ObjFunc_t defines the template for the objective function
type ObjFunc_t func(ind *Individual, idIsland, time int, report *bytes.Buffer) (ova, oor float64)

// Island holds one population and performs the reproduction operation
type Island struct {

	// input
	Id        int         // index of this island
	C         *ConfParams // configuration parameters
	Pop       Population  // pointer to current population
	BkpPop    Population  // backup population
	OvOorFunc ObjFunc_t   // function to compute objective and out-of-range values
	BingoGrid *Bingo      // bingo for regeneration with initial values from grid
	BingoBest *Bingo      // bingo for regeneration with values recomputed based on best individual

	// results
	Report bytes.Buffer // buffer to report results
	OVS    []float64    // best objective values collected from multiple calls to SelectAndReprod

	// auxiliary internal data
	foundov bool      // found first feasible individual
	ovas    []float64 // all ova values
	oors    []float64 // all oor values
	sovas   []float64 // scaled ova values
	soors   []float64 // scaled oor values
	fitness []float64 // all fitness values
	prob    []float64 // probabilities
	cumprob []float64 // cumulated probabilities
	selinds []int     // indices of selected individuals
	A, B    []int     // indices of selected parents

	// for statistics
	maxabsgene []float64   // [ngenes] max absolute values of genes
	fltbases   [][]float64 // [ngenes*nbases][ninds] all bases
	devbases   []float64   // [ngenes*nbases] deviations of bases
}

// NewIsland allocates a new island but with a give population already allocated
// Input:
//  id     -- index of this island
//  pop    -- the population
//  ovfunc -- objective function
//  bingo  -- structure needed for regeneration of individuals
func NewIsland(id int, C *ConfParams, pop Population, ovfunc ObjFunc_t, bingo *Bingo) (o *Island) {

	// check
	ninds := len(pop)
	chk.IntAssert(C.Ninds, ninds)
	if ninds%2 != 0 {
		chk.Panic("size of population must be even")
	}

	// allocate
	o = new(Island)
	o.Id = id
	o.C = C
	o.Pop = pop
	o.BkpPop = pop.GetCopy()
	o.OvOorFunc = ovfunc
	o.BingoGrid = bingo
	o.BingoBest = bingo.GetCopy()

	// auxiliary data
	o.ovas = make([]float64, ninds)
	o.oors = make([]float64, ninds)
	o.sovas = make([]float64, ninds)
	o.soors = make([]float64, ninds)
	o.fitness = make([]float64, ninds)
	o.prob = make([]float64, ninds)
	o.cumprob = make([]float64, ninds)
	o.selinds = make([]int, ninds)
	o.A = make([]int, ninds/2)
	o.B = make([]int, ninds/2)

	// compute objective values, demerits, and sort population
	o.CalcOvs(o.Pop, 0)
	o.CalcDemeritsAndSort(o.Pop)

	// results
	o.OVS = []float64{o.Pop[0].Ova}

	// for statistics
	nfltgenes := o.Pop[0].Nfltgenes
	if nfltgenes > 0 {
		nbases := o.Pop[0].Nbases
		o.maxabsgene = make([]float64, nfltgenes)
		o.fltbases = la.MatAlloc(nfltgenes*nbases, ninds)
		o.devbases = make([]float64, nfltgenes*nbases)
	}
	return
}

// CalcOvs computes objective and out-of-range values
func (o *Island) CalcOvs(pop Population, time int) {
	for _, ind := range pop {
		ova, oor := o.OvOorFunc(ind, o.Id, time, &o.Report)
		if oor < 0 {
			chk.Panic("out-of-range values must be positive (or zero) indicating the positive distance to constraints. oor=%g is invalid", oor)
		}
		ind.Ova = ova
		ind.Oor = oor
	}
}

// CalcDemeritsAndSort computes demerits and sort population
func (o *Island) CalcDemeritsAndSort(pop Population) {

	// ovs and oors
	var iova, ioor int // indices of individuals with ova and with oor, respectively
	for _, ind := range pop {
		if ind.Oor > 0 { // infeasible solutions (out-of-range)
			o.oors[ioor] = ind.Oor
			ioor++
		} else { // feasible solutions
			o.foundov = true
			o.ovas[iova] = ind.Ova
			iova++
		}
	}

	// scaled ovs and oors
	utl.Scaling(o.sovas[:iova], o.ovas[:iova], 0, 1e-16, false, true)
	utl.Scaling(o.soors[:ioor], o.oors[:ioor], 2, 1e-16, false, true)

	// set demerits in individuals (loop with the same comparisons as before)
	ioor, iova = 0, 0
	for _, ind := range pop {
		if ind.Oor > 0 { // infeasible solutions (out-of-range)
			ind.Demerit = o.soors[ioor]
			ioor++
		} else { // feasible solutions
			ind.Demerit = o.sovas[iova]
			iova++
		}
	}

	// sort population
	pop.Sort()
}

// SelectAndReprod performs the selection and reproduction processes
//  Note: this function considers a SORTED population already
func (o *Island) SelectAndReprod(time int) (averho float64) {

	// fitness
	ninds := len(o.Pop)
	var sumfit float64
	if o.C.Rnk { // ranking
		sp := o.C.RnkSp
		for i := 0; i < ninds; i++ {
			o.fitness[i] = 2.0 - sp + 2.0*(sp-1.0)*float64(ninds-i-1)/float64(ninds-1)
			sumfit += o.fitness[i]
		}
	} else {
		mindem := o.Pop[0].Demerit
		maxdem := mindem
		for i := 0; i < ninds; i++ {
			mindem = min(mindem, o.Pop[i].Demerit)
			maxdem = max(maxdem, o.Pop[i].Demerit)
		}
		for i, ind := range o.Pop {
			o.fitness[i] = (maxdem - ind.Demerit) / (maxdem - mindem)
			sumfit += o.fitness[i]
		}
	}

	// probabilities
	for i := 0; i < ninds; i++ {
		o.prob[i] = o.fitness[i] / sumfit
		if i == 0 {
			o.cumprob[i] = o.prob[i]
		} else {
			o.cumprob[i] = o.cumprob[i-1] + o.prob[i]
		}
	}

	// selection
	if o.C.Rws {
		RouletteSelect(o.selinds, o.cumprob, nil)
	} else {
		SUSselect(o.selinds, o.cumprob, -1)
	}
	FilterPairs(o.A, o.B, o.selinds)

	// reproduction
	h := ninds / 2
	for i := 0; i < ninds/2; i++ {
		Crossover(o.BkpPop[i], o.BkpPop[h+i], o.Pop[o.A[i]], o.Pop[o.B[i]], o.C.CxNcuts, o.C.CxCuts, o.C.CxProbs, o.C.CxIntFunc, o.C.CxFltFunc, o.C.CxStrFunc, o.C.CxKeyFunc, o.C.CxBytFunc, o.C.CxFunFunc)
		Mutation(o.BkpPop[i], o.C.MtNchanges, o.C.MtProbs, o.C.MtExtra, o.C.MtIntFunc, o.C.MtFltFunc, o.C.MtStrFunc, o.C.MtKeyFunc, o.C.MtBytFunc, o.C.MtFunFunc)
		Mutation(o.BkpPop[h+i], o.C.MtNchanges, o.C.MtProbs, o.C.MtExtra, o.C.MtIntFunc, o.C.MtFltFunc, o.C.MtStrFunc, o.C.MtKeyFunc, o.C.MtBytFunc, o.C.MtFunFunc)
	}

	// compute objective values, demerits, and sort population
	o.CalcOvs(o.BkpPop, time+1) // +1 => this is an updated generation
	o.CalcDemeritsAndSort(o.BkpPop)

	// elitism
	if o.C.Elite {
		if o.foundov {
			if o.Pop[0].Ova < o.BkpPop[0].Ova {
				o.Pop[0].CopyInto(o.BkpPop[ninds-1])
				o.CalcDemeritsAndSort(o.BkpPop)
			}
		} else {
			if o.Pop[0].Oor < o.BkpPop[0].Oor {
				o.Pop[0].CopyInto(o.BkpPop[ninds-1])
				o.CalcDemeritsAndSort(o.BkpPop)
			}
		}
	}

	// swap populations (Pop will always point to current one)
	o.Pop, o.BkpPop = o.BkpPop, o.Pop

	// statistics
	_, averho, _, _ = o.Stat()

	// results
	o.OVS = append(o.OVS, o.Pop[0].Ova)
	return
}

// Regenerate regenerates population with basis on best individual(s)
//  Output:
//   regtype -- 1=best, 2=lims
func (o *Island) Regenerate(time int, basedOnBest bool) (regtype int) {
	bingo := o.BingoGrid
	regtype = 2
	if basedOnBest || o.C.RegBest {
		regtype = 1
		o.BingoBest.ResetBasedOnRef(time, o.Pop[0], o.C.RegMmin, o.C.RegMmax)
		bingo = o.BingoBest
	}
	ninds := len(o.Pop)
	start := ninds - int(o.C.RegPct*float64(ninds))
	for i := start; i < ninds; i++ {
		for j := 0; j < o.Pop[i].Nfltgenes; j++ {
			o.Pop[i].SetFloat(j, bingo.DrawFloat(i, j, ninds))
		}
	}
	o.CalcOvs(o.Pop, time)
	o.CalcDemeritsAndSort(o.Pop)
	return
}

// Stat computes some statistic information
//  rho (ρ) is a normalised quantity measuring the deviation of bases of each gene
//  Note: OoR individuals are excluded
func (o *Island) Stat() (minrho, averho, maxrho, devrho float64) {
	ngenes := o.Pop[0].Nfltgenes
	if ngenes < 1 {
		return
	}
	nbases := o.Pop[0].Nbases
	iova := 0
	for _, ind := range o.Pop {
		if ind.Oor > 0 && o.C.StatOorSkip { // skip oor individuals
			continue
		}
		for i := 0; i < ngenes; i++ {
			x := math.Abs(ind.GetFloat(i))
			if iova == 0 {
				o.maxabsgene[i] = x
			} else {
				o.maxabsgene[i] = max(o.maxabsgene[i], x)
			}
			for j := 0; j < nbases; j++ {
				o.fltbases[i*nbases+j][iova] = ind.Floats[i*nbases+j]
			}
		}
		iova++
	}
	if iova < 2 {
		return
	}
	for i := 0; i < ngenes; i++ {
		x := 1.0 + o.maxabsgene[i]
		for j := 0; j < nbases; j++ {
			o.devbases[i*nbases+j] = rnd.StatDev(o.fltbases[i*nbases+j][:iova], o.C.UseStdDev) / x
		}
	}
	minrho, averho, maxrho, devrho = rnd.StatBasic(o.devbases, o.C.UseStdDev)
	return
}

// Write writes results to buffer
func (o Island) Write(buf *bytes.Buffer, t int, json bool) {
	if json {
		return
	}
	buf.Write(o.Pop.Output(nil, o.C.ShowBases).Bytes())
}

// PlotOvs plots objective values versus time
func (o Island) PlotOvs(ext, args string, t0, tf int, withtxt bool, numfmt string, first, last bool) {
	if o.C.DoPlot == false || o.C.FnKey == "" {
		return
	}
	if first {
		plt.SetForEps(0.75, 250)
	}
	n := len(o.OVS) - t0
	T := utl.LinSpace(float64(t0), float64(tf), n)
	plt.Plot(T, o.OVS[:n], args)
	if withtxt {
		plt.Text(T[0], o.OVS[0], io.Sf(numfmt, o.OVS[0]), "ha='left'")
		plt.Text(T[n-1], o.OVS[n-1], io.Sf(numfmt, o.OVS[n-1]), "ha='right'")
	}
	if last {
		plt.Gll("time", "objective value", "")
		plt.SaveD(o.C.DirOut, o.C.FnKey+ext)
	}
}

// SaveReport saves report to file
func (o Island) SaveReport(dirout, fnkey string, verbose bool) {
	if dirout == "" {
		dirout = "/tmp/goga"
	}
	if verbose {
		io.WriteFileVD(dirout, fnkey+".rpt", &o.Report)
		return
	}
	io.WriteFileD(dirout, fnkey+".rpt", &o.Report)
}
<commit_msg>plot of ovs fixed<commit_after>// Copyright 2015 Dorival de Moraes Pedroso. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package goga

import (
	"bytes"
	"math"

	"github.com/cpmech/gosl/chk"
	"github.com/cpmech/gosl/io"
	"github.com/cpmech/gosl/la"
	"github.com/cpmech/gosl/plt"
	"github.com/cpmech/gosl/rnd"
	"github.com/cpmech/gosl/utl"
)

// ObjFunc_t defines the template for the objective function
type ObjFunc_t func(ind *Individual, idIsland, time int, report *bytes.Buffer) (ova, oor float64)

// Island holds one population and performs the reproduction operation
type Island struct {

	// input
	Id        int         // index of this island
	C         *ConfParams // configuration parameters
	Pop       Population  // pointer to current population
	BkpPop    Population  // backup population
	OvOorFunc ObjFunc_t   // function to compute objective and out-of-range values
	BingoGrid *Bingo      // bingo for regeneration with initial values from grid
	BingoBest *Bingo      // bingo for regeneration with values recomputed based on best individual

	// results
	Report bytes.Buffer // buffer to report results
	OVS    []float64    // best objective values collected from multiple calls to SelectAndReprod

	// auxiliary internal data
	foundov bool      // found first feasible individual
	ovas    []float64 // all ova values
	oors    []float64 // all oor values
	sovas   []float64 // scaled ova values
	soors   []float64 // scaled oor values
	fitness []float64 // all fitness values
	prob    []float64 // probabilities
	cumprob []float64 // cumulated probabilities
	selinds []int     // indices of selected individuals
	A, B    []int     // indices of selected parents

	// for statistics
	maxabsgene []float64   // [ngenes] max absolute values of genes
	fltbases   [][]float64 // [ngenes*nbases][ninds] all bases
	devbases   []float64   // [ngenes*nbases] deviations of bases
}

// NewIsland allocates a new island but with a give population already allocated
// Input:
//  id     -- index of this island
//  pop    -- the population
//  ovfunc -- objective function
//  bingo  -- structure needed for regeneration of individuals
func NewIsland(id int, C *ConfParams, pop Population, ovfunc ObjFunc_t, bingo *Bingo) (o *Island) {

	// check
	ninds := len(pop)
	chk.IntAssert(C.Ninds, ninds)
	if ninds%2 != 0 {
		chk.Panic("size of population must be even")
	}

	// allocate
	o = new(Island)
	o.Id = id
	o.C = C
	o.Pop = pop
	o.BkpPop = pop.GetCopy()
	o.OvOorFunc = ovfunc
	o.BingoGrid = bingo
	o.BingoBest = bingo.GetCopy()

	// auxiliary data
	o.ovas = make([]float64, ninds)
	o.oors = make([]float64, ninds)
	o.sovas = make([]float64, ninds)
	o.soors = make([]float64, ninds)
	o.fitness = make([]float64, ninds)
	o.prob = make([]float64, ninds)
	o.cumprob = make([]float64, ninds)
	o.selinds = make([]int, ninds)
	o.A = make([]int, ninds/2)
	o.B = make([]int, ninds/2)

	// compute objective values, demerits, and sort population
	o.CalcOvs(o.Pop, 0)
	o.CalcDemeritsAndSort(o.Pop)

	// results
	o.OVS = []float64{o.Pop[0].Ova}

	// for statistics
	nfltgenes := o.Pop[0].Nfltgenes
	if nfltgenes > 0 {
		nbases := o.Pop[0].Nbases
		o.maxabsgene = make([]float64, nfltgenes)
		o.fltbases = la.MatAlloc(nfltgenes*nbases, ninds)
		o.devbases = make([]float64, nfltgenes*nbases)
	}
	return
}

// CalcOvs computes objective and out-of-range values
func (o *Island) CalcOvs(pop Population, time int) {
	for _, ind := range pop {
		ova, oor := o.OvOorFunc(ind, o.Id, time, &o.Report)
		if oor < 0 {
			chk.Panic("out-of-range values must be positive (or zero) indicating the positive distance to constraints. oor=%g is invalid", oor)
		}
		ind.Ova = ova
		ind.Oor = oor
	}
}

// CalcDemeritsAndSort computes demerits and sort population
func (o *Island) CalcDemeritsAndSort(pop Population) {

	// ovs and oors
	var iova, ioor int // indices of individuals with ova and with oor, respectively
	for _, ind := range pop {
		if ind.Oor > 0 { // infeasible solutions (out-of-range)
			o.oors[ioor] = ind.Oor
			ioor++
		} else { // feasible solutions
			o.foundov = true
			o.ovas[iova] = ind.Ova
			iova++
		}
	}

	// scaled ovs and oors
	utl.Scaling(o.sovas[:iova], o.ovas[:iova], 0, 1e-16, false, true)
	utl.Scaling(o.soors[:ioor], o.oors[:ioor], 2, 1e-16, false, true)

	// set demerits in individuals (loop with the same comparisons as before)
	ioor, iova = 0, 0
	for _, ind := range pop {
		if ind.Oor > 0 { // infeasible solutions (out-of-range)
			ind.Demerit = o.soors[ioor]
			ioor++
		} else { // feasible solutions
			ind.Demerit = o.sovas[iova]
			iova++
		}
	}

	// sort population
	pop.Sort()
}

// SelectAndReprod performs the selection and reproduction processes
//  Note: this function considers a SORTED population already
func (o *Island) SelectAndReprod(time int) (averho float64) {

	// fitness
	ninds := len(o.Pop)
	var sumfit float64
	if o.C.Rnk { // ranking
		sp := o.C.RnkSp
		for i := 0; i < ninds; i++ {
			o.fitness[i] = 2.0 - sp + 2.0*(sp-1.0)*float64(ninds-i-1)/float64(ninds-1)
			sumfit += o.fitness[i]
		}
	} else {
		mindem := o.Pop[0].Demerit
		maxdem := mindem
		for i := 0; i < ninds; i++ {
			mindem = min(mindem, o.Pop[i].Demerit)
			maxdem = max(maxdem, o.Pop[i].Demerit)
		}
		for i, ind := range o.Pop {
			o.fitness[i] = (maxdem - ind.Demerit) / (maxdem - mindem)
			sumfit += o.fitness[i]
		}
	}

	// probabilities
	for i := 0; i < ninds; i++ {
		o.prob[i] = o.fitness[i] / sumfit
		if i == 0 {
			o.cumprob[i] = o.prob[i]
		} else {
			o.cumprob[i] = o.cumprob[i-1] + o.prob[i]
		}
	}

	// selection
	if o.C.Rws {
		RouletteSelect(o.selinds, o.cumprob, nil)
	} else {
		SUSselect(o.selinds, o.cumprob, -1)
	}
	FilterPairs(o.A, o.B, o.selinds)

	// reproduction
	h := ninds / 2
	for i := 0; i < ninds/2; i++ {
		Crossover(o.BkpPop[i], o.BkpPop[h+i], o.Pop[o.A[i]], o.Pop[o.B[i]], o.C.CxNcuts, o.C.CxCuts, o.C.CxProbs, o.C.CxIntFunc, o.C.CxFltFunc, o.C.CxStrFunc, o.C.CxKeyFunc, o.C.CxBytFunc, o.C.CxFunFunc)
		Mutation(o.BkpPop[i], o.C.MtNchanges, o.C.MtProbs, o.C.MtExtra, o.C.MtIntFunc, o.C.MtFltFunc, o.C.MtStrFunc, o.C.MtKeyFunc, o.C.MtBytFunc, o.C.MtFunFunc)
		Mutation(o.BkpPop[h+i], o.C.MtNchanges, o.C.MtProbs, o.C.MtExtra, o.C.MtIntFunc, o.C.MtFltFunc, o.C.MtStrFunc, o.C.MtKeyFunc, o.C.MtBytFunc, o.C.MtFunFunc)
	}

	// compute objective values, demerits, and sort population
	o.CalcOvs(o.BkpPop, time+1) // +1 => this is an updated generation
	o.CalcDemeritsAndSort(o.BkpPop)

	// elitism
	if o.C.Elite {
		if o.foundov {
			if o.Pop[0].Ova < o.BkpPop[0].Ova {
				o.Pop[0].CopyInto(o.BkpPop[ninds-1])
				o.CalcDemeritsAndSort(o.BkpPop)
			}
		} else {
			if o.Pop[0].Oor < o.BkpPop[0].Oor {
				o.Pop[0].CopyInto(o.BkpPop[ninds-1])
				o.CalcDemeritsAndSort(o.BkpPop)
			}
		}
	}

	// swap populations (Pop will always point to current one)
	o.Pop, o.BkpPop = o.BkpPop, o.Pop

	// statistics
	_, averho, _, _ = o.Stat()

	// results
	o.OVS = append(o.OVS, o.Pop[0].Ova)
	return
}

// Regenerate regenerates population with basis on best individual(s)
//  Output:
//   regtype -- 1=best, 2=lims
func (o *Island) Regenerate(time int, basedOnBest bool) (regtype int) {
	bingo := o.BingoGrid
	regtype = 2
	if basedOnBest || o.C.RegBest {
		regtype = 1
		o.BingoBest.ResetBasedOnRef(time, o.Pop[0], o.C.RegMmin, o.C.RegMmax)
		bingo = o.BingoBest
	}
	ninds := len(o.Pop)
	start := ninds - int(o.C.RegPct*float64(ninds))
	for i := start; i < ninds; i++ {
		for j := 0; j < o.Pop[i].Nfltgenes; j++ {
			o.Pop[i].SetFloat(j, bingo.DrawFloat(i, j, ninds))
		}
	}
	o.CalcOvs(o.Pop, time)
	o.CalcDemeritsAndSort(o.Pop)
	return
}

// Stat computes some statistic information
//  rho (ρ) is a normalised quantity measuring the deviation of bases of each gene
//  Note: OoR individuals are excluded
func (o *Island) Stat() (minrho, averho, maxrho, devrho float64) {
	ngenes := o.Pop[0].Nfltgenes
	if ngenes < 1 {
		return
	}
	nbases := o.Pop[0].Nbases
	iova := 0
	for _, ind := range o.Pop {
		if ind.Oor > 0 && o.C.StatOorSkip { // skip oor individuals
			continue
		}
		for i := 0; i < ngenes; i++ {
			x := math.Abs(ind.GetFloat(i))
			if iova == 0 {
				o.maxabsgene[i] = x
			} else {
				o.maxabsgene[i] = max(o.maxabsgene[i], x)
			}
			for j := 0; j < nbases; j++ {
				o.fltbases[i*nbases+j][iova] = ind.Floats[i*nbases+j]
			}
		}
		iova++
	}
	if iova < 2 {
		return
	}
	for i := 0; i < ngenes; i++ {
		x := 1.0 + o.maxabsgene[i]
		for j := 0; j < nbases; j++ {
			o.devbases[i*nbases+j] = rnd.StatDev(o.fltbases[i*nbases+j][:iova], o.C.UseStdDev) / x
		}
	}
	minrho, averho, maxrho, devrho = rnd.StatBasic(o.devbases, o.C.UseStdDev)
	return
}

// Write writes results to buffer
func (o Island) Write(buf *bytes.Buffer, t int, json bool) {
	if json {
		return
	}
	buf.Write(o.Pop.Output(nil, o.C.ShowBases).Bytes())
}

// PlotOvs plots objective values versus time
func (o Island) PlotOvs(ext, args string, t0, tf int, withtxt bool, numfmt string, first, last bool) {
	if o.C.DoPlot == false || o.C.FnKey == "" {
		return
	}
	if first {
		plt.SetForEps(0.75, 250)
	}
	var y []float64
	if tf == -1 {
		y = o.OVS[t0:]
		tf = len(o.OVS)
	} else {
		y = o.OVS[t0:tf]
	}
	n := len(y)
	T := utl.LinSpace(float64(t0), float64(tf), n)
	plt.Plot(T, y, args)
	if withtxt {
		plt.Text(T[0], y[0], io.Sf(numfmt, y[0]), "ha='left'")
		plt.Text(T[n-1], y[n-1], io.Sf(numfmt, y[n-1]), "ha='right'")
	}
	if last {
		plt.Gll("time", "objective value", "")
		plt.SaveD(o.C.DirOut, o.C.FnKey+ext)
	}
}

// SaveReport saves report to file
func (o Island) SaveReport(dirout, fnkey string, verbose bool) {
	if dirout == "" {
		dirout = "/tmp/goga"
	}
	if verbose {
		io.WriteFileVD(dirout, fnkey+".rpt", &o.Report)
		return
	}
	io.WriteFileD(dirout, fnkey+".rpt", &o.Report)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
)

/* This script exists because godep deprecated -copy=false, and I really
 * don't agree that importing the actual source code for peco is the
 * correct choice
 *
 * It's used by my own build script to release files to GitHub.
 * You (the contributor) do not need to use it, but if you need a
 * particular revision of the dependent packages, make sure to update
 * the SHA1 below.
 */

var pwd string
var deps = map[string]string{
	"github.com/jessevdk/go-flags":  "8ec9564882e7923e632f012761c81c46dcf5bec1",
	"github.com/mattn/go-runewidth": "58a0da4ed7b321c9b5dfeffb7e03ee188fae1c60",
	"github.com/nsf/termbox-go":     "10f14d7408b64a659b7c694a771f5006952d336c",
	"github.com/google/btree":       "0c05920fc3d98100a5e3f7fd339865a6e2aaa671",
}

func symlink(src, dest string) error {
	if runtime.GOOS == "windows" {
		if _, err := exec.Command("cmd", "/c", "mklink", "/J", dest, src).CombinedOutput(); err != nil {
			return err
		}
		return nil
	}
	return os.Symlink(src, dest)
}

func init() {
	var err error
	if pwd, err = os.Getwd(); err != nil {
		panic(err)
	}
}

func main() {
	action := ""
	if len(os.Args) == 2 {
		action = os.Args[1]
	}

	switch action {
	case "gopath":
		fmt.Printf("%s\n", getBuildDir())
	case "deps":
		setupDeps()
	case "build":
		setupDeps()
		buildBinaries()
	default:
		panic("Unknown action: " + action)
	}
}

func getBuildDir() string {
	buildDir := ".build"
	if dir := os.Getenv("PECO_BUILD_DIR"); dir != "" {
		buildDir = dir
	}

	d, err := filepath.Abs(buildDir)
	if err != nil {
		panic(err)
	}
	buildDir = d

	for _, subdir := range []string{"bin", "pkg", "src", "artifacts"} {
		dir := filepath.Join(buildDir, subdir)

		// Make sure this directory exists to avoid errors...
		if _, err := os.Stat(dir); err != nil {
			if err := os.MkdirAll(dir, 0777); err != nil {
				panic(err)
			}
		}
	}

	return buildDir
}

func setupDeps() {
	var err error

	buildDir := getBuildDir()
	for dir, hash := range deps {
		repo := repoURL(dir)
		dir = filepath.Join(buildDir, "src", dir)
		if _, err = os.Stat(dir); err != nil {
			if err = run("git", "clone", repo, dir); err != nil {
				panic(err)
			}
		}

		if err = os.Chdir(dir); err != nil {
			panic(err)
		}

		if err = run("git", "reset", "--hard"); err != nil {
			panic(err)
		}

		if err = run("git", "checkout", "master"); err != nil {
			panic(err)
		}

		if err = run("git", "pull"); err != nil {
			panic(err)
		}

		if err = run("git", "checkout", hash); err != nil {
			panic(err)
		}

		if err = os.Chdir(pwd); err != nil {
			panic(err)
		}
	}

	setupPecoInGopath()

	log.Println("dependencies have been checked out to under %s", buildDir)
}

func setupPecoInGopath() {
	// Make a symlink to current directory so that it's in GOPATH
	buildDir := getBuildDir()
	linkDest := filepath.Join(buildDir, "src", "github.com", "peco", "peco")
	linkDestDir := filepath.Dir(linkDest)
	if _, err := os.Stat(linkDestDir); err != nil {
		if err := os.MkdirAll(linkDestDir, 0777); err != nil {
			panic(err)
		}
	}

	if _, err := os.Stat(linkDest); err != nil {
		log.Printf("Creating symlink from '%s' to '%s'", pwd, linkDest)
		if err := symlink(pwd, linkDest); err != nil {
			panic(err)
		}
	}
}

func buildBinaries() {
	buildDir := getBuildDir()
	os.Setenv("GOPATH", buildDir)

	for _, osname := range []string{"darwin", "linux", "windows"} {
		for _, arch := range []string{"amd64", "386"} {
			buildBinaryFor(osname, arch)
		}
	}

	buildBinaryFor("linux", "arm")
}

func buildBinaryFor(osname, arch string) {
	os.Setenv("GOOS", osname)
	os.Setenv("GOARCH", arch)

	log.Printf("Building for %s/%s", osname, arch)

	name := fmt.Sprintf("peco_%s_%s", osname, arch)

	run("go", "build", "-o",
		filepath.Join(name, "peco"),
		filepath.Join("cmd", "peco", "peco.go"),
	)

	run("cp", "Changes", filepath.Join(name, "Changes"))
	run("cp", "README.md", filepath.Join(name, "README.md"))

	var file string
	if osname == "linux" {
		file = fmt.Sprintf("%s.tar.gz", name)
		run("tar", "czvf", file, name)
	} else {
		file = fmt.Sprintf("%s.zip", name)
		run("zip", "-r", file, name)
	}

	os.RemoveAll(name)
	run("mv", file, filepath.Join(getBuildDir(), "artifacts"))
}

func run(name string, args ...string) error {
	splat := []string{name}
	splat = append(splat, args...)
	log.Printf("---> Running %v...\n", splat)

	cmd := exec.Command(name, args...)
	out, err := cmd.CombinedOutput()
	for _, line := range strings.Split(string(out), "\n") {
		log.Print(line)
	}
	log.Println("")
	log.Println("<---DONE")
	return err
}

func repoURL(spec string) string {
	return "https://" + spec + ".git"
}
<commit_msg>Update deps<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
)

/* This script exists because godep deprecated -copy=false, and I really
 * don't agree that importing the actual source code for peco is the
 * correct choice
 *
 * It's used by my own build script to release files to GitHub.
 * You (the contributor) do not need to use it, but if you need a
 * particular revision of the dependent packages, make sure to update
 * the SHA1 below.
 */

var pwd string
var deps = map[string]string{
	"github.com/jessevdk/go-flags":  "fc93116606d0a71d7e9de0ad5734fdb4b8eae834",
	"github.com/mattn/go-runewidth": "12e0ff74603c9a3209d8bf84f8ab349fe1ad9477",
	"github.com/nsf/termbox-go":     "62033d80b58736ea31beaf43348f5147913af30e",
	"github.com/google/btree":       "0c05920fc3d98100a5e3f7fd339865a6e2aaa671",
}

func symlink(src, dest string) error {
	if runtime.GOOS == "windows" {
		if _, err := exec.Command("cmd", "/c", "mklink", "/J", dest, src).CombinedOutput(); err != nil {
			return err
		}
		return nil
	}
	return os.Symlink(src, dest)
}

func init() {
	var err error
	if pwd, err = os.Getwd(); err != nil {
		panic(err)
	}
}

func main() {
	action := ""
	if len(os.Args) == 2 {
		action = os.Args[1]
	}

	switch action {
	case "gopath":
		fmt.Printf("%s\n", getBuildDir())
	case "deps":
		setupDeps()
	case "build":
		setupDeps()
		buildBinaries()
	default:
		panic("Unknown action: " + action)
	}
}

func getBuildDir() string {
	buildDir := ".build"
	if dir := os.Getenv("PECO_BUILD_DIR"); dir != "" {
		buildDir = dir
	}

	d, err := filepath.Abs(buildDir)
	if err != nil {
		panic(err)
	}
	buildDir = d

	for _, subdir := range []string{"bin", "pkg", "src", "artifacts"} {
		dir := filepath.Join(buildDir, subdir)

		// Make sure this directory exists to avoid errors...
		if _, err := os.Stat(dir); err != nil {
			if err := os.MkdirAll(dir, 0777); err != nil {
				panic(err)
			}
		}
	}

	return buildDir
}

func setupDeps() {
	var err error

	buildDir := getBuildDir()
	for dir, hash := range deps {
		repo := repoURL(dir)
		dir = filepath.Join(buildDir, "src", dir)
		if _, err = os.Stat(dir); err != nil {
			if err = run("git", "clone", repo, dir); err != nil {
				panic(err)
			}
		}

		if err = os.Chdir(dir); err != nil {
			panic(err)
		}

		if err = run("git", "reset", "--hard"); err != nil {
			panic(err)
		}

		if err = run("git", "checkout", "master"); err != nil {
			panic(err)
		}

		if err = run("git", "pull"); err != nil {
			panic(err)
		}

		if err = run("git", "checkout", hash); err != nil {
			panic(err)
		}

		if err = os.Chdir(pwd); err != nil {
			panic(err)
		}
	}

	setupPecoInGopath()

	log.Println("dependencies have been checked out to under %s", buildDir)
}

func setupPecoInGopath() {
	// Make a symlink to current directory so that it's in GOPATH
	buildDir := getBuildDir()
	linkDest := filepath.Join(buildDir, "src", "github.com", "peco", "peco")
	linkDestDir := filepath.Dir(linkDest)
	if _, err := os.Stat(linkDestDir); err != nil {
		if err := os.MkdirAll(linkDestDir, 0777); err != nil {
			panic(err)
		}
	}

	if _, err := os.Stat(linkDest); err != nil {
		log.Printf("Creating symlink from '%s' to '%s'", pwd, linkDest)
		if err := symlink(pwd, linkDest); err != nil {
			panic(err)
		}
	}
}

func buildBinaries() {
	buildDir := getBuildDir()
	os.Setenv("GOPATH", buildDir)

	for _, osname := range []string{"darwin", "linux", "windows"} {
		for _, arch := range []string{"amd64", "386"} {
			buildBinaryFor(osname, arch)
		}
	}

	buildBinaryFor("linux", "arm")
}

func buildBinaryFor(osname, arch string) {
	os.Setenv("GOOS", osname)
	os.Setenv("GOARCH", arch)

	log.Printf("Building for %s/%s", osname, arch)

	name := fmt.Sprintf("peco_%s_%s", osname, arch)

	run("go", "build", "-o",
		filepath.Join(name, "peco"),
		filepath.Join("cmd", "peco", "peco.go"),
	)

	run("cp", "Changes", filepath.Join(name, "Changes"))
	run("cp", "README.md", filepath.Join(name, "README.md"))

	var file string
	if osname == "linux" {
		file = fmt.Sprintf("%s.tar.gz", name)
		run("tar", "czvf", file, name)
	} else {
		file = fmt.Sprintf("%s.zip", name)
		run("zip", "-r", file, name)
	}

	os.RemoveAll(name)
	run("mv", file, filepath.Join(getBuildDir(), "artifacts"))
}

func run(name string, args ...string) error {
	splat := []string{name}
	splat = append(splat, args...)
	log.Printf("---> Running %v...\n", splat)

	cmd := exec.Command(name, args...)
	out, err := cmd.CombinedOutput()
	for _, line := range strings.Split(string(out), "\n") {
		log.Print(line)
	}
	log.Println("")
	log.Println("<---DONE")
	return err
}

func repoURL(spec string) string {
	return "https://" + spec + ".git"
}
<|endoftext|>


<commit_before>package manssh

import (
	"io/ioutil"
	"os"
	"strings"

	"github.com/kevinburke/ssh_config"
)

const (
	// User is a ssh config key
	User = "user"
	// Hostname is a ssh config key
	Hostname = "hostname"
	// Port is a ssh config key
	Port = "port"
)

// ParseConfig parse configs from ssh config file, return config object and alias map
func ParseConfig(path string) (*ssh_config.Config, map[string]*ssh_config.Host) {
	f, _ := os.OpenFile(path, os.O_APPEND|os.O_CREATE, 0600)
	cfg, _ := ssh_config.Decode(f)
	aliasMap := map[string]*ssh_config.Host{}
	for _, host := range cfg.Hosts {
		for _, pattern := range host.Patterns {
			aliasMap[pattern.String()] = host
		}
	}
	return cfg, aliasMap
}

// List ssh alias, filter by optional keyword
func List(path string, keywords ...string) []*HostConfig {
	cfg, _ := ParseConfig(path)
	hosts := []*HostConfig{}

	// Convert to HostConfig
	for _, host := range cfg.Hosts {
		aliases := []string{}
		for _, pattern := range host.Patterns {
			aliases = append(aliases, pattern.String())
		}
		h := &HostConfig{
			Aliases: strings.Join(aliases, " "),
			Config:  map[string]string{},
		}
		isGlobal := h.Aliases == "*"
		connectMap := map[string]string{}
		if !isGlobal {
			connectMap[User] = ""
			connectMap[Hostname] = ""
			connectMap[Port] = ""
		}
		values := []string{}
		values = append(values, aliases...)

		for _, node := range host.Nodes {
			switch t := node.(type) {
			case *ssh_config.KV:
				t.Key = strings.ToLower(t.Key)
				values = append(values, t.Value)
				if _, ok := connectMap[t.Key]; ok {
					connectMap[t.Key] = t.Value
				} else {
					h.Config[t.Key] = t.Value
				}
				t.SetLeadingSpace(4)
			}
		}
		if isGlobal && len(h.Config) == 0 {
			continue
		}
		if len(keywords) > 0 && !Query(values, keywords) {
			continue
		}
		if !isGlobal {
			h.Connect = FormatConnect(connectMap[User], connectMap[Hostname], connectMap[Port])
		}
		hosts = append(hosts, h)
	}

	// Format
	if len(cfg.Hosts) > 0 {
		ioutil.WriteFile(path, []byte(cfg.String()), 0644)
	}
	return hosts
}

// Add ssh host config to ssh config file
func Add(path string, host *HostConfig) error {
	cfg, aliasMap := ParseConfig(path)
	isGlobal := host.Aliases == "*"
	// Alias should not exist. except "*" because it always existing
	if !isGlobal {
		if err := CheckAlias(aliasMap, false, host.Aliases); err != nil {
			return err
		}
	}
	if host.Config == nil {
		host.Config = map[string]string{}
	}
	nodes := []ssh_config.Node{}
	checkKeyRepeat := map[string]bool{}

	// Parse connect string
	if host.Connect != "" {
		user, hostname, port := ParseConnct(host.Connect)
		host.Connect = FormatConnect(user, hostname, port)

		userKV := &ssh_config.KV{Key: User, Value: user}
		nodes = append(nodes, userKV.SetLeadingSpace(4))

		hostnameKV := &ssh_config.KV{Key: Hostname, Value: hostname}
		nodes = append(nodes, hostnameKV.SetLeadingSpace(4))

		portKV := &ssh_config.KV{Key: Port, Value: port}
		nodes = append(nodes, portKV.SetLeadingSpace(4))

		checkKeyRepeat[User] = true
		checkKeyRepeat[Hostname] = true
		checkKeyRepeat[Port] = true
	}

	// Get nodes and delete repeat config
	deleteKeys := []string{}
	for k, v := range host.Config {
		lk := strings.ToLower(k)
		if !checkKeyRepeat[lk] {
			node := &ssh_config.KV{Key: lk, Value: v}
			nodes = append(nodes, node.SetLeadingSpace(4))
			checkKeyRepeat[lk] = true
		} else {
			deleteKeys = append(deleteKeys, k)
		}
	}
	for _, deleteKey := range deleteKeys {
		delete(host.Config, deleteKey)
	}

	pattern, err := ssh_config.NewPattern(host.Aliases)
	if err != nil {
		return nil
	}
	newHost := &ssh_config.Host{
		Patterns: []*ssh_config.Pattern{pattern},
		Nodes:    nodes,
	}
	cfg.Hosts = append(cfg.Hosts, newHost)
	return ioutil.WriteFile(path, []byte(cfg.String()), 0644)
}

// Update existing record
func Update(path string, h *HostConfig, newAlias string) error {
	cfg, aliasMap := ParseConfig(path)
	if err := CheckAlias(aliasMap, true, h.Aliases); err != nil {
		return err
	}

	updateHost := aliasMap[h.Aliases]
	if newAlias != "" {
		// alias rename
		for _, pattern := range updateHost.Patterns {
			if pattern.String() == h.Aliases {
				pattern.SetStr(newAlias)
				h.Aliases = newAlias
			}
		}
	}

	updateKV := map[string]string{}
	if h.Connect != "" {
		user, hostname, port := ParseConnct(h.Connect)
		updateKV[User] = user
		updateKV[Hostname] = hostname
		updateKV[Port] = port
	}

	if h.Config != nil {
		for k, v := range h.Config {
			updateKV[strings.ToLower(k)] = v
		}
	}
	h.Config = map[string]string{}
	connectMap := map[string]string{}
	if h.Aliases != "*" {
		connectMap[User] = ""
		connectMap[Hostname] = ""
		connectMap[Port] = ""
	}
	// update node
	for _, node := range updateHost.Nodes {
		switch t := node.(type) {
		case *ssh_config.KV:
			t.Key = strings.ToLower(t.Key)
			if value, ok := updateKV[t.Key]; ok {
				t.SetLeadingSpace(4)
				t.Value = value
				delete(updateKV, t.Key)
			}
			if _, ok := connectMap[t.Key]; ok {
				connectMap[t.Key] = t.Value
			} else {
				h.Config[t.Key] = t.Value
			}
		}
	}
	// append new node
	for k, v := range updateKV {
		kv := &ssh_config.KV{Key: k, Value: v}
		updateHost.Nodes = append(updateHost.Nodes, kv.SetLeadingSpace(4))
		if _, ok := connectMap[k]; ok {
			connectMap[k] = v
		} else {
			h.Config[k] = v
		}
	}
	h.Connect = FormatConnect(connectMap[User], connectMap[Hostname], connectMap[Port])
	return ioutil.WriteFile(path, []byte(cfg.String()), 0644)
}

// Delete existing alias record
func Delete(path string, aliases ...string) error {
	cfg, aliasMap := ParseConfig(path)
	if err := CheckAlias(aliasMap, true, aliases...); err != nil {
		return err
	}
	deleteAliasMap := map[string]bool{}
	for _, alias := range aliases {
		deleteAliasMap[alias] = true
	}
	newHosts := []*ssh_config.Host{}
	for _, host := range cfg.Hosts {
		newPattern := []*ssh_config.Pattern{}
		for _, pattern := range host.Patterns {
			if !deleteAliasMap[pattern.String()] {
				newPattern = append(newPattern, pattern)
			}
		}
		if len(newPattern) > 0 {
			host.Patterns = newPattern
			newHosts = append(newHosts, host)
		}
	}
	cfg.Hosts = newHosts
	return ioutil.WriteFile(path, []byte(cfg.String()), 0644)
}
<commit_msg>Bug fix<commit_after>package manssh

import (
	"io/ioutil"
	"os"
	"strings"

	"github.com/kevinburke/ssh_config"
)

const (
	// User is a ssh config key
	User = "user"
	// Hostname is a ssh config key
	Hostname = "hostname"
	// Port is a ssh config key
	Port = "port"
)

// ParseConfig parse configs from ssh config file, return config object and alias map
func ParseConfig(path string) (*ssh_config.Config, map[string]*ssh_config.Host) {
	f, _ := os.OpenFile(path, os.O_APPEND|os.O_CREATE, 0600)
	cfg, _ := ssh_config.Decode(f)
	aliasMap := map[string]*ssh_config.Host{}
	for _, host := range cfg.Hosts {
		for _, pattern := range host.Patterns {
			aliasMap[pattern.String()] = host
		}
	}
	return cfg, aliasMap
}

// List ssh alias, filter by optional keyword
func List(path string, keywords ...string) []*HostConfig {
	cfg, _ := ParseConfig(path)
	hosts := []*HostConfig{}

	// Convert to HostConfig
	for _, host := range cfg.Hosts {
		aliases := []string{}
		for _, pattern := range host.Patterns {
			aliases = append(aliases, pattern.String())
		}
		h := &HostConfig{
			Aliases: strings.Join(aliases, " "),
			Config:  map[string]string{},
		}
		isGlobal := h.Aliases == "*"
		connectMap := map[string]string{}
		if !isGlobal {
			connectMap[User] = ""
			connectMap[Hostname] = ""
			connectMap[Port] = ""
		}
		values := []string{}
		values = append(values, aliases...)

		for _, node := range host.Nodes {
			switch t := node.(type) {
			case *ssh_config.KV:
				t.Key = strings.ToLower(t.Key)
				values = append(values, t.Value)
				if _, ok := connectMap[t.Key]; ok {
					connectMap[t.Key] = t.Value
				} else {
					h.Config[t.Key] = t.Value
				}
				t.SetLeadingSpace(4)
			}
		}
		if isGlobal && len(h.Config) == 0 {
			continue
		}
		if len(keywords) > 0 && !Query(values, keywords) {
			continue
		}
		if !isGlobal {
			h.Connect = FormatConnect(connectMap[User], connectMap[Hostname], connectMap[Port])
		}
		hosts = append(hosts, h)
	}

	// Format
	if len(cfg.Hosts) > 0 {
		ioutil.WriteFile(path, []byte(cfg.String()), 0644)
	}
	return hosts
}

// Add ssh host config to ssh config file
func Add(path string, host *HostConfig) error {
	cfg, aliasMap := ParseConfig(path)
	isGlobal := host.Aliases == "*"
	// Alias should not exist. except "*" because it always existing
	if !isGlobal || (isGlobal && len(aliasMap["*"].Nodes) > 0) {
		if err := CheckAlias(aliasMap, false, host.Aliases); err != nil {
			return err
		}
	}
	if host.Config == nil {
		host.Config = map[string]string{}
	}
	nodes := []ssh_config.Node{}
	checkKeyRepeat := map[string]bool{}

	// Parse connect string
	if host.Connect != "" {
		user, hostname, port := ParseConnct(host.Connect)
		host.Connect = FormatConnect(user, hostname, port)

		userKV := &ssh_config.KV{Key: User, Value: user}
		nodes = append(nodes, userKV.SetLeadingSpace(4))

		hostnameKV := &ssh_config.KV{Key: Hostname, Value: hostname}
		nodes = append(nodes, hostnameKV.SetLeadingSpace(4))

		portKV := &ssh_config.KV{Key: Port, Value: port}
		nodes = append(nodes, portKV.SetLeadingSpace(4))

		checkKeyRepeat[User] = true
		checkKeyRepeat[Hostname] = true
		checkKeyRepeat[Port] = true
	}

	// Get nodes and delete repeat config
	deleteKeys := []string{}
	for k, v := range host.Config {
		lk := strings.ToLower(k)
		if !checkKeyRepeat[lk] {
			node := &ssh_config.KV{Key: lk, Value: v}
			nodes = append(nodes, node.SetLeadingSpace(4))
			checkKeyRepeat[lk] = true
		} else {
			deleteKeys = append(deleteKeys, k)
		}
	}
	for _, deleteKey := range deleteKeys {
		delete(host.Config, deleteKey)
	}

	pattern, err := ssh_config.NewPattern(host.Aliases)
	if err != nil {
		return nil
	}
	patterns := []*ssh_config.Pattern{pattern}
	newHost := &ssh_config.Host{
		Patterns: patterns,
		Nodes:    nodes,
	}
	if !isGlobal {
		cfg.Hosts = append(cfg.Hosts, newHost)
	} else {
		*aliasMap["*"] = *newHost
	}
	return ioutil.WriteFile(path, []byte(cfg.String()), 0644)
}

// Update existing record
func Update(path string, h *HostConfig, newAlias string) error {
	cfg, aliasMap := ParseConfig(path)
	if err := CheckAlias(aliasMap, true, h.Aliases); err != nil {
		return err
	}

	updateHost := aliasMap[h.Aliases]
	if newAlias != "" {
		// alias rename
		for _, pattern := range updateHost.Patterns {
			if pattern.String() == h.Aliases {
				pattern.SetStr(newAlias)
				h.Aliases = newAlias
			}
		}
	}

	updateKV := map[string]string{}
	if h.Connect != "" {
		user, hostname, port := ParseConnct(h.Connect)
		updateKV[User] = user
		updateKV[Hostname] = hostname
		updateKV[Port] = port
	}

	if h.Config != nil {
		for k, v := range h.Config {
			updateKV[strings.ToLower(k)] = v
		}
	}
	h.Config = map[string]string{}
	connectMap := map[string]string{}
	if h.Aliases != "*" {
		connectMap[User] = ""
		connectMap[Hostname] = ""
		connectMap[Port] = ""
	}
	// update node
	for _, node := range updateHost.Nodes {
		switch t := node.(type) {
		case *ssh_config.KV:
			t.Key = strings.ToLower(t.Key)
			if value, ok := updateKV[t.Key]; ok {
				t.SetLeadingSpace(4)
				t.Value = value
				delete(updateKV, t.Key)
			}
			if _, ok := connectMap[t.Key]; ok {
				connectMap[t.Key] = t.Value
			} else {
				h.Config[t.Key] = t.Value
			}
		}
	}
	// append new node
	for k, v := range updateKV {
		kv := &ssh_config.KV{Key: k, Value: v}
		updateHost.Nodes = append(updateHost.Nodes, kv.SetLeadingSpace(4))
		if _, ok := connectMap[k]; ok {
			connectMap[k] = v
		} else {
			h.Config[k] = v
		}
	}
	h.Connect = FormatConnect(connectMap[User], connectMap[Hostname], connectMap[Port])
	return ioutil.WriteFile(path, []byte(cfg.String()), 0644)
}

// Delete existing alias record
func Delete(path string, aliases ...string) error {
	cfg, aliasMap := ParseConfig(path)
	if err := CheckAlias(aliasMap, true, aliases...); err != nil {
		return err
	}
	deleteAliasMap := map[string]bool{}
	for _, alias := range aliases {
		deleteAliasMap[alias] = true
	}
	newHosts := []*ssh_config.Host{}
	for _, host := range cfg.Hosts {
		newPattern := []*ssh_config.Pattern{}
		for _, pattern := range host.Patterns {
			if !deleteAliasMap[pattern.String()] {
				newPattern = append(newPattern, pattern)
			}
		}
		if len(newPattern) > 0 {
			host.Patterns = newPattern
			newHosts = append(newHosts, host)
		}
	}
	cfg.Hosts = newHosts
	return ioutil.WriteFile(path, []byte(cfg.String()), 0644)
}
<|endoftext|>


<commit_before>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package clientv3

import (
	"context"
	"crypto/tls"
	"time"

	"go.uber.org/zap"
	"google.golang.org/grpc"
)

type Config struct {
	// Endpoints is a list of URLs.
	Endpoints []string `json:"endpoints"`

	// AutoSyncInterval is the interval to update endpoints with its latest members.
	// 0 disables auto-sync. By default auto-sync is disabled.
	AutoSyncInterval time.Duration `json:"auto-sync-interval"`

	// DialTimeout is the timeout for failing to establish a connection.
	DialTimeout time.Duration `json:"dial-timeout"`

	// DialKeepAliveTime is the time after which client pings the server to see if
	// transport is alive.
	DialKeepAliveTime time.Duration `json:"dial-keep-alive-time"`

	// DialKeepAliveTimeout is the time that the client waits for a response for the
	// keep-alive probe. If the response is not received in this time, the connection is closed.
	DialKeepAliveTimeout time.Duration `json:"dial-keep-alive-timeout"`

	// MaxCallSendMsgSize is the client-side request send limit in bytes.
	// If 0, it defaults to 2.0 MiB (2 * 1024 * 1024).
	// Make sure that "MaxCallSendMsgSize" < server-side default send/recv limit.
	// ("--max-request-bytes" flag to etcd or "embed.Config.MaxRequestBytes").
	MaxCallSendMsgSize int

	// MaxCallRecvMsgSize is the client-side response receive limit.
	// If 0, it defaults to "math.MaxInt32", because range response can
	// easily exceed request send limits.
	// Make sure that "MaxCallRecvMsgSize" >= server-side default send/recv limit.
	// ("--max-request-bytes" flag to etcd or "embed.Config.MaxRequestBytes").
	MaxCallRecvMsgSize int

	// TLS holds the client secure credentials, if any.
	TLS *tls.Config

	// Username is a user name for authentication.
	Username string `json:"username"`

	// Password is a password for authentication.
	Password string `json:"password"`

	// RejectOldCluster when set will refuse to create a client against an outdated cluster.
	RejectOldCluster bool `json:"reject-old-cluster"`

	// DialOptions is a list of dial options for the grpc client (e.g., for interceptors).
	DialOptions []grpc.DialOption

	// Context is the default client context; it can be used to cancel grpc dial out and
	// other operations that do not have an explicit context.
	Context context.Context

	// LogConfig configures client-side logger.
	// If nil, use the default logger.
	// TODO: configure gRPC logger
	LogConfig *zap.Config

	// PermitWithoutStream when set will allow client to send keepalive pings to server without any active streams(RPCs).
	PermitWithoutStream bool `json:"permit-without-stream"`

	// TODO: support custom balancer picker
}
<commit_msg>clientv3: document "WithBlock" dial option<commit_after>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package clientv3

import (
	"context"
	"crypto/tls"
	"time"

	"go.uber.org/zap"
	"google.golang.org/grpc"
)

type Config struct {
	// Endpoints is a list of URLs.
	Endpoints []string `json:"endpoints"`

	// AutoSyncInterval is the interval to update endpoints with its latest members.
	// 0 disables auto-sync. By default auto-sync is disabled.
	AutoSyncInterval time.Duration `json:"auto-sync-interval"`

	// DialTimeout is the timeout for failing to establish a connection.
	DialTimeout time.Duration `json:"dial-timeout"`

	// DialKeepAliveTime is the time after which client pings the server to see if
	// transport is alive.
	DialKeepAliveTime time.Duration `json:"dial-keep-alive-time"`

	// DialKeepAliveTimeout is the time that the client waits for a response for the
	// keep-alive probe. If the response is not received in this time, the connection is closed.
	DialKeepAliveTimeout time.Duration `json:"dial-keep-alive-timeout"`

	// MaxCallSendMsgSize is the client-side request send limit in bytes.
	// If 0, it defaults to 2.0 MiB (2 * 1024 * 1024).
	// Make sure that "MaxCallSendMsgSize" < server-side default send/recv limit.
	// ("--max-request-bytes" flag to etcd or "embed.Config.MaxRequestBytes").
	MaxCallSendMsgSize int

	// MaxCallRecvMsgSize is the client-side response receive limit.
	// If 0, it defaults to "math.MaxInt32", because range response can
	// easily exceed request send limits.
	// Make sure that "MaxCallRecvMsgSize" >= server-side default send/recv limit.
	// ("--max-request-bytes" flag to etcd or "embed.Config.MaxRequestBytes").
	MaxCallRecvMsgSize int

	// TLS holds the client secure credentials, if any.
	TLS *tls.Config

	// Username is a user name for authentication.
	Username string `json:"username"`

	// Password is a password for authentication.
	Password string `json:"password"`

	// RejectOldCluster when set will refuse to create a client against an outdated cluster.
	RejectOldCluster bool `json:"reject-old-cluster"`

	// DialOptions is a list of dial options for the grpc client (e.g., for interceptors).
	// For example, pass "grpc.WithBlock()" to block until the underlying connection is up.
	// Without this, Dial returns immediately and connecting the server happens in background.
	DialOptions []grpc.DialOption

	// Context is the default client context; it can be used to cancel grpc dial out and
	// other operations that do not have an explicit context.
	Context context.Context

	// LogConfig configures client-side logger.
	// If nil, use the default logger.
	// TODO: configure gRPC logger
	LogConfig *zap.Config

	// PermitWithoutStream when set will allow client to send keepalive pings to server without any active streams(RPCs).
	PermitWithoutStream bool `json:"permit-without-stream"`

	// TODO: support custom balancer picker
}
<|endoftext|>


<commit_before>package bloom

import (
	"github.com/pmylund/go-bitset"

	"encoding/binary"
	"hash"
	"hash/fnv"
	"math"
)

type filter struct {
	n uint32
	k uint32
	h hash.Hash64
}

func (f *filter) indices(data []byte) []uint32 {
	f.h.Reset()
	f.h.Write(data)
	d := f.h.Sum(nil)
	a := binary.BigEndian.Uint32(d[0:4])
	b := binary.BigEndian.Uint32(d[4:8])
	is := make([]uint32, f.k)
	for i := uint32(0); i < f.k; i++ {
		is[i] = (a + b*i) % f.n
	}
	return is
}

func new(n, k uint32) *filter {
	return &filter{
		n: n,
		k: k,
		h: fnv.New64a(),
	}
}

func estimates(num int, fpRate float64) (uint32, uint32) {
	n := uint32(-1 * float64(num) * math.Log(fpRate) / math.Pow(math.Log(2), 2))
	k := uint32(math.Ceil(math.Log(2) * float64(n) / float64(num)))
	return n, k
}

// A standard bloom filter using the 64-bit FNV-1a hash function.
type Filter struct {
	*filter
	b *bitset.Bitset
}

// Check whether data was previously added to the filter. Returns true if
// yes, with a false positive chance near the ratio specified upon creation
// of the filter. The result cannot be falsely negative.
func (f *Filter) Test(data []byte) bool {
	for _, i := range f.indices(data) {
		if !f.b.Test(i) {
			return false
		}
	}
	return true
}

// Add data to the filter.
func (f *Filter) Add(data []byte) {
	for _, i := range f.indices(data) {
		f.b.Set(i)
	}
}

// Create a bloom filter with an expected num number of items, and an acceptable
// false positive rate of fpRate, e.g. 0.01.
func New(num int, fpRate float64) *Filter {
	n, k := estimates(num, fpRate)
	f := &Filter{
		new(n, k),
		bitset.New(n),
	}
	return f
}

// A counting bloom filter using the 64-bit FNV-1a hash function. Supports
// removing items from the filter.
type CountingFilter struct {
	*filter
	b []*bitset.Bitset
}

// Checks whether data was previously added to the filter. Returns true if
// yes, with a false positive chance near the ratio specified upon creation
// of the filter. The result cannot cannot be falsely negative (unless one
// has removed an item that wasn't actually added to the filter previously.)
func (f *CountingFilter) Test(data []byte) bool {
	b := f.b[0]
	for _, v := range f.indices(data) {
		if !b.Test(v) {
			return false
		}
	}
	return true
}

// Adds data to the filter.
func (f *CountingFilter) Add(data []byte) {
	for _, v := range f.indices(data) {
		done := false
		for _, ov := range f.b {
			if !ov.Test(v) {
				done = true
				ov.Set(v)
			}
		}
		if !done {
			nb := bitset.New(f.b[0].Len())
			f.b = append(f.b, nb)
			nb.Set(v)
		}
	}
}

// Removes data from the filter. This exact data must have been previously added
// to the filter, or future results will be inconsistent.
func (f *CountingFilter) Remove(data []byte) {
	last := len(f.b) - 1
	for _, v := range f.indices(data) {
		for oi := last; oi >= 0; oi-- {
			ov := f.b[oi]
			if ov.Test(v) {
				ov.Clear(v)
				break
			}
		}
	}
}

// Create a counting bloom filter with an expected num number of items, and an
// acceptable false positive rate of fpRate. Counting bloom filters support
// the removal of items from the filter.
func NewCounting(num int, fpRate float64) *CountingFilter {
	n, k := estimates(num, fpRate)
	f := &CountingFilter{
		new(n, k),
		[]*bitset.Bitset{bitset.New(n)},
	}
	return f
}

// A layered bloom filter using the 64-bit FNV-1a hash function.
type LayeredFilter struct {
	*filter
	b []*bitset.Bitset
}

// Checks whether data was previously added to the filter. Returns the number of
// the last layer where the data was added, e.g. 1 for the first layer, and a
// boolean indicating whether the data was added to the filter at all. The check
// has a false positive chance near the ratio specified upon creation of the
// filter. The result cannot be falsely negative.
func (f *LayeredFilter) Test(data []byte) (int, bool) {
	is := f.indices(data)
	for i := len(f.b) - 1; i >= 0; i-- {
		v := f.b[i]
		last := len(is) - 1
		for oi, ov := range is {
			if !v.Test(ov) {
				break
			}
			if oi == last {
				// Every test was positive at this layer
				return i + 1, true
			}
		}
	}
	return 0, false
}

// Adds data to the filter. Returns the number of the layer where the data
// was added, e.g. 1 for the first layer.
func (f *LayeredFilter) Add(data []byte) int {
	is := f.indices(data)
	var (
		i int
		v *bitset.Bitset
	)
	for i, v = range f.b {
		here := false
		for _, ov := range is {
			if here {
				v.Set(ov)
			} else if !v.Test(ov) {
				here = true
				v.Set(ov)
			}
		}
		if here {
			return i + 1
		}
	}
	nb := bitset.New(f.b[0].Len())
	f.b = append(f.b, nb)
	for _, v := range is {
		nb.Set(v)
	}
	return i + 2
}

// Create a layered bloom filter with an expected num number of items, and an
// acceptable false positive rate of fpRate. Layered bloom filters can be used
// to keep track of a certain, arbitrary count of items, e.g. to check if some
// given data was added to the filter 10 times or less.
func NewLayered(num int, fpRate float64) *LayeredFilter {
	n, k := estimates(num, fpRate)
	f := &LayeredFilter{
		new(n, k),
		[]*bitset.Bitset{bitset.New(n)},
	}
	return f
}
<commit_msg>Stop after we've found an empty spot in CountingFilter<commit_after>package bloom

import (
	"github.com/pmylund/go-bitset"

	"encoding/binary"
	"hash"
	"hash/fnv"
	"math"
)

type filter struct {
	n uint32
	k uint32
	h hash.Hash64
}

func (f *filter) indices(data []byte) []uint32 {
	f.h.Reset()
	f.h.Write(data)
	d := f.h.Sum(nil)
	a := binary.BigEndian.Uint32(d[0:4])
	b := binary.BigEndian.Uint32(d[4:8])
	is := make([]uint32, f.k)
	for i := uint32(0); i < f.k; i++ {
		is[i] = (a + b*i) % f.n
	}
	return is
}

func new(n, k uint32) *filter {
	return &filter{
		n: n,
		k: k,
		h: fnv.New64a(),
	}
}

func estimates(num int, fpRate float64) (uint32, uint32) {
	n := uint32(-1 * float64(num) * math.Log(fpRate) / math.Pow(math.Log(2), 2))
	k := uint32(math.Ceil(math.Log(2) * float64(n) / float64(num)))
	return n, k
}

// A standard bloom filter using the 64-bit FNV-1a hash function.
type Filter struct {
	*filter
	b *bitset.Bitset
}

// Check whether data was previously added to the filter. Returns true if
// yes, with a false positive chance near the ratio specified upon creation
// of the filter. The result cannot be falsely negative.
func (f *Filter) Test(data []byte) bool {
	for _, i := range f.indices(data) {
		if !f.b.Test(i) {
			return false
		}
	}
	return true
}

// Add data to the filter.
func (f *Filter) Add(data []byte) {
	for _, i := range f.indices(data) {
		f.b.Set(i)
	}
}

// Create a bloom filter with an expected num number of items, and an acceptable
// false positive rate of fpRate, e.g. 0.01.
func New(num int, fpRate float64) *Filter {
	n, k := estimates(num, fpRate)
	f := &Filter{
		new(n, k),
		bitset.New(n),
	}
	return f
}

// A counting bloom filter using the 64-bit FNV-1a hash function. Supports
// removing items from the filter.
type CountingFilter struct {
	*filter
	b []*bitset.Bitset
}

// Checks whether data was previously added to the filter. Returns true if
// yes, with a false positive chance near the ratio specified upon creation
// of the filter. The result cannot cannot be falsely negative (unless one
// has removed an item that wasn't actually added to the filter previously.)
func (f *CountingFilter) Test(data []byte) bool {
	b := f.b[0]
	for _, v := range f.indices(data) {
		if !b.Test(v) {
			return false
		}
	}
	return true
}

// Adds data to the filter.
func (f *CountingFilter) Add(data []byte) {
	for _, v := range f.indices(data) {
		done := false
		for _, ov := range f.b {
			if !ov.Test(v) {
				done = true
				ov.Set(v)
				break
			}
		}
		if !done {
			nb := bitset.New(f.b[0].Len())
			f.b = append(f.b, nb)
			nb.Set(v)
		}
	}
}

// Removes data from the filter. This exact data must have been previously added
// to the filter, or future results will be inconsistent.
func (f *CountingFilter) Remove(data []byte) {
	last := len(f.b) - 1
	for _, v := range f.indices(data) {
		for oi := last; oi >= 0; oi-- {
			ov := f.b[oi]
			if ov.Test(v) {
				ov.Clear(v)
				break
			}
		}
	}
}

// Create a counting bloom filter with an expected num number of items, and an
// acceptable false positive rate of fpRate. Counting bloom filters support
// the removal of items from the filter.
func NewCounting(num int, fpRate float64) *CountingFilter {
	n, k := estimates(num, fpRate)
	f := &CountingFilter{
		new(n, k),
		[]*bitset.Bitset{bitset.New(n)},
	}
	return f
}

// A layered bloom filter using the 64-bit FNV-1a hash function.
type LayeredFilter struct {
	*filter
	b []*bitset.Bitset
}

// Checks whether data was previously added to the filter. Returns the number of
// the last layer where the data was added, e.g. 1 for the first layer, and a
// boolean indicating whether the data was added to the filter at all. The check
// has a false positive chance near the ratio specified upon creation of the
// filter. The result cannot be falsely negative.
func (f *LayeredFilter) Test(data []byte) (int, bool) {
	is := f.indices(data)
	for i := len(f.b) - 1; i >= 0; i-- {
		v := f.b[i]
		last := len(is) - 1
		for oi, ov := range is {
			if !v.Test(ov) {
				break
			}
			if oi == last {
				// Every test was positive at this layer
				return i + 1, true
			}
		}
	}
	return 0, false
}

// Adds data to the filter. Returns the number of the layer where the data
// was added, e.g. 1 for the first layer.
func (f *LayeredFilter) Add(data []byte) int {
	is := f.indices(data)
	var (
		i int
		v *bitset.Bitset
	)
	for i, v = range f.b {
		here := false
		for _, ov := range is {
			if here {
				v.Set(ov)
			} else if !v.Test(ov) {
				here = true
				v.Set(ov)
			}
		}
		if here {
			return i + 1
		}
	}
	nb := bitset.New(f.b[0].Len())
	f.b = append(f.b, nb)
	for _, v := range is {
		nb.Set(v)
	}
	return i + 2
}

// Create a layered bloom filter with an expected num number of items, and an
// acceptable false positive rate of fpRate. Layered bloom filters can be used
// to keep track of a certain, arbitrary count of items, e.g. to check if some
// given data was added to the filter 10 times or less.
func NewLayered(num int, fpRate float64) *LayeredFilter {
	n, k := estimates(num, fpRate)
	f := &LayeredFilter{
		new(n, k),
		[]*bitset.Bitset{bitset.New(n)},
	}
	return f
}
<|endoftext|>