<commit_before>package wats

import (
	"encoding/json"
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"testing"
	"time"

	. "github.com/cloudfoundry-incubator/cf-test-helpers/workflowhelpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	"github.com/cloudfoundry-incubator/cf-test-helpers/generator"
	"github.com/cloudfoundry-incubator/cf-test-helpers/helpers"
)

const (
	DEFAULT_TIMEOUT = 45 * time.Second
	CF_PUSH_TIMEOUT = 3 * time.Minute
)

var (
	appName            string
	config             *watsConfig
	environment        *ReproducibleTestSuiteSetup
	hwcBuildPackURL    = "https://github.com/cloudfoundry-incubator/hwc-buildpack/releases/download/v2.3.2/hwc_buildpack-cached-v2.3.2.zip"
	binaryBuildPackURL = "https://github.com/cloudfoundry/binary-buildpack/releases/download/v1.0.10/binary_buildpack-cached-v1.0.10.zip"
)

func guidForAppName(appName string) string {
	cfApp := cf.Cf("app", appName, "--guid")
	Expect(cfApp.Wait()).To(Exit(0))

	appGuid := strings.TrimSpace(string(cfApp.Out.Contents()))
	Expect(appGuid).NotTo(Equal(""))
	return appGuid
}

func guidForSpaceName(spaceName string) string {
	cfSpace := cf.Cf("space", spaceName, "--guid")
	Expect(cfSpace.Wait()).To(Exit(0))

	spaceGuid := strings.TrimSpace(string(cfSpace.Out.Contents()))
	Expect(spaceGuid).NotTo(Equal(""))
	return spaceGuid
}

func enableDiego(appName string) {
	guid := guidForAppName(appName)
	Eventually(cf.Cf("curl", "/v2/apps/"+guid, "-X", "PUT", "-d", `{"diego": true}`)).Should(Exit(0))
}

func disableHealthCheck(appName string) {
	guid := guidForAppName(appName)
	Eventually(cf.Cf("curl", "/v2/apps/"+guid, "-X", "PUT", "-d", `{"health_check_type":"none"}`)).Should(Exit(0))
}

func TestDiegoWindows(t *testing.T) {
	RegisterFailHandler(Fail)

	SetDefaultEventuallyTimeout(time.Minute)
	SetDefaultEventuallyPollingInterval(time.Second)

	var err error
	config, err = LoadWatsConfig()
	if err != nil {
		t.Fatalf("could not load WATS config", err)
	}

	if config.NumWindowsCells == 0 {
		t.Fatalf("Please provide 'num_windows_cells' as a property in the integration config JSON (The number of windows cells in tested deployment)")
	}

	environment = NewTestSuiteSetup(config)

	BeforeSuite(func() {
		environment.Setup()
		binaryBuildpackVersion := getBuildpackVersion("binary_buildpack")
		if versionGreaterThan(binaryBuildpackVersion, 1, 0, 10) {
			binaryBuildPackURL = "binary_buildpack"
		}

		hwcBuildpackVersion := getBuildpackVersion("hwc_buildpack")
		if versionGreaterThan(hwcBuildpackVersion, 2, 3, 2) {
			hwcBuildPackURL = "hwc_buildpack"
		}

		if config.GetIsolationSegmentName() != "" {
			AsUser(environment.AdminUserContext(), environment.ShortTimeout(), func() {
				isoSegGuid := createOrGetIsolationSegment(config.GetIsolationSegmentName())
				attachIsolationSegmentToOrg(environment, isoSegGuid)
				attachIsolationSegmentToSpace(environment, isoSegGuid)
			})
		}
	})

	AfterSuite(func() {
		environment.Teardown()
	})

	BeforeEach(func() {
		Eventually(cf.Cf("apps").Out).Should(Say("No apps found"))
		appName = generator.PrefixedRandomName(config.GetNamePrefix(), "APP")
	})

	AfterEach(func() {
		Eventually(cf.Cf("logs", appName, "--recent")).Should(Exit())
		Eventually(cf.Cf("delete", appName, "-f")).Should(Exit(0))
	})

	componentName := "DiegoWindows"

	rs := []Reporter{}

	if config.GetArtifactsDirectory() != "" {
		helpers.EnableCFTrace(config, componentName)
		rs = append(rs, helpers.NewJUnitReporter(config, componentName))
	}

	RunSpecsWithDefaultAndCustomReporters(t, componentName, rs)
}
func getBuildpackVersion(name string) string {
	buildpack := cf.Cf("curl", fmt.Sprintf("/v2/buildpacks?q=name:%s", name))
	Expect(buildpack.Wait()).To(Exit(0))
	type Buildpack struct {
		Resources []struct {
			Entity struct {
				FileName string
			}
		}
	}
	var b Buildpack
	Expect(json.Unmarshal(buildpack.Out.Contents(), &b)).To(Succeed())
	if len(b.Resources) == 0 {
		return "0.0.0"
	}
	re := regexp.MustCompile(`[0-9]+\.[0-9]+\.[0-9]+`)
	return re.FindString(b.Resources[0].Entity.FileName)
}

func versionGreaterThan(version string, inputMajor, inputMinor, inputPatch int) bool {
	versions := strings.Split(version, ".")
	major, _ := strconv.Atoi(versions[0])
	if major > inputMajor {
		return true
	}
	minor, _ := strconv.Atoi(versions[1])
	if minor > inputMinor {
		return true
	}
	patch, _ := strconv.Atoi(versions[2])
	if patch > inputPatch {
		return true
	}
	return false
}

func createOrGetIsolationSegment(isolationSegmentName string) string {
	// This could go in cf-test-helpers
	guid := getIsolationSegmentGuid(isolationSegmentName)
	if guid == "" {
		Eventually(cf.Cf("curl", "/v3/isolation_segments/", "-X", "POST", "-d", fmt.Sprintf(`{"name":"%s"}`, isolationSegmentName))).Should(Exit(0))
		guid = getIsolationSegmentGuid(isolationSegmentName)
	}
	return guid
}

func getIsolationSegmentGuid(isolationSegmentName string) string {
	return getV3ResourceGuid(fmt.Sprintf("/v3/isolation_segments?names=%s", isolationSegmentName))
}

func attachIsolationSegmentToOrg(environment *ReproducibleTestSuiteSetup, isoSegGuid string) {
	orgGuid := getOrganizationGuid(environment.GetOrganizationName())
	response := cf.Cf(
		"curl",
		fmt.Sprintf("/v3/isolation_segments/%s/relationships/organizations", isoSegGuid),
		"-X",
		"POST",
		"-d",
		fmt.Sprintf(`{"data":[{"guid": "%s"}]}`, orgGuid),
	)
	Expect(response.Wait()).To(Exit(0))
}

func attachIsolationSegmentToSpace(environment *ReproducibleTestSuiteSetup, isoSegGuid string) {
	spaceGuid := getSpaceGuidForOrg(getOrganizationGuid(environment.GetOrganizationName()))
	response := cf.Cf(
		"curl",
		fmt.Sprintf("/v2/spaces/%s", spaceGuid),
		"-X",
		"PUT",
		"-d",
		fmt.Sprintf(`{"isolation_segment_guid":"%s"}`, isoSegGuid),
	)
	Expect(response.Wait()).To(Exit(0))
}

func getOrganizationGuid(organizationName string) string {
	return getV2ResourceGuid(fmt.Sprintf("/v2/organizations?q=name:%s", organizationName))
}

func getSpaceGuidForOrg(orgGuid string) string {
	return getV2ResourceGuid(fmt.Sprintf("/v2/organizations/%s/spaces", orgGuid))
}

func getV2ResourceGuid(endpoint string) string {
	response := cf.Cf("curl", endpoint, "-X", "GET")
	Expect(response.Wait()).To(Exit(0))
	var r struct {
		Resources []struct {
			Metadata struct {
				Guid string
			}
		}
	}
	Expect(json.Unmarshal(response.Out.Contents(), &r)).To(Succeed())
	Expect(len(r.Resources)).To(Equal(1))
	return r.Resources[0].Metadata.Guid
}

func getV3ResourceGuid(endpoint string) string {
	response := cf.Cf("curl", endpoint, "-X", "GET")
	Expect(response.Wait()).To(Exit(0))
	var r struct {
		Resources []struct {
			Guid string
		}
	}
	Expect(json.Unmarshal(response.Out.Contents(), &r)).To(Succeed())
	if len(r.Resources) == 0 {
		return ""
	}
	return r.Resources[0].Guid
}
<commit_msg>Bump binary buildpack to 1.0.11<commit_after>package wats

import (
	"encoding/json"
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"testing"
	"time"

	. "github.com/cloudfoundry-incubator/cf-test-helpers/workflowhelpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	"github.com/cloudfoundry-incubator/cf-test-helpers/generator"
	"github.com/cloudfoundry-incubator/cf-test-helpers/helpers"
)

const (
	DEFAULT_TIMEOUT = 45 * time.Second
	CF_PUSH_TIMEOUT = 3 * time.Minute
)

var (
	appName            string
	config             *watsConfig
	environment        *ReproducibleTestSuiteSetup
	hwcBuildPackURL    = "https://github.com/cloudfoundry-incubator/hwc-buildpack/releases/download/v2.3.2/hwc_buildpack-cached-v2.3.2.zip"
	binaryBuildPackURL = "https://github.com/cloudfoundry/binary-buildpack/releases/download/v1.0.11/binary_buildpack-cached-v1.0.11.zip"
)

func guidForAppName(appName string) string {
	cfApp := cf.Cf("app", appName, "--guid")
	Expect(cfApp.Wait()).To(Exit(0))

	appGuid := strings.TrimSpace(string(cfApp.Out.Contents()))
	Expect(appGuid).NotTo(Equal(""))
	return appGuid
}

func guidForSpaceName(spaceName string) string {
	cfSpace := cf.Cf("space", spaceName, "--guid")
	Expect(cfSpace.Wait()).To(Exit(0))

	spaceGuid := strings.TrimSpace(string(cfSpace.Out.Contents()))
	Expect(spaceGuid).NotTo(Equal(""))
	return spaceGuid
}

func enableDiego(appName string) {
	guid := guidForAppName(appName)
	Eventually(cf.Cf("curl", "/v2/apps/"+guid, "-X", "PUT", "-d", `{"diego": true}`)).Should(Exit(0))
}

func disableHealthCheck(appName string) {
	guid := guidForAppName(appName)
	Eventually(cf.Cf("curl", "/v2/apps/"+guid, "-X", "PUT", "-d", `{"health_check_type":"none"}`)).Should(Exit(0))
}

func TestDiegoWindows(t *testing.T) {
	RegisterFailHandler(Fail)

	SetDefaultEventuallyTimeout(time.Minute)
	SetDefaultEventuallyPollingInterval(time.Second)

	var err error
	config, err = LoadWatsConfig()
	if err != nil {
		t.Fatalf("could not load WATS config", err)
	}

	if config.NumWindowsCells == 0 {
		t.Fatalf("Please provide 'num_windows_cells' as a property in the integration config JSON (The number of windows cells in tested deployment)")
	}

	environment = NewTestSuiteSetup(config)

	BeforeSuite(func() {
		environment.Setup()
		binaryBuildpackVersion := getBuildpackVersion("binary_buildpack")
		if versionGreaterThan(binaryBuildpackVersion, 1, 0, 11) {
			binaryBuildPackURL = "binary_buildpack"
		}

		hwcBuildpackVersion := getBuildpackVersion("hwc_buildpack")
		if versionGreaterThan(hwcBuildpackVersion, 2, 3, 2) {
			hwcBuildPackURL = "hwc_buildpack"
		}

		if config.GetIsolationSegmentName() != "" {
			AsUser(environment.AdminUserContext(), environment.ShortTimeout(), func() {
				isoSegGuid := createOrGetIsolationSegment(config.GetIsolationSegmentName())
				attachIsolationSegmentToOrg(environment, isoSegGuid)
				attachIsolationSegmentToSpace(environment, isoSegGuid)
			})
		}
	})

	AfterSuite(func() {
		environment.Teardown()
	})

	BeforeEach(func() {
		Eventually(cf.Cf("apps").Out).Should(Say("No apps found"))
		appName = generator.PrefixedRandomName(config.GetNamePrefix(), "APP")
	})

	AfterEach(func() {
		Eventually(cf.Cf("logs", appName, "--recent")).Should(Exit())
		Eventually(cf.Cf("delete", appName, "-f")).Should(Exit(0))
	})

	componentName := "DiegoWindows"

	rs := []Reporter{}

	if config.GetArtifactsDirectory() != "" {
		helpers.EnableCFTrace(config, componentName)
		rs = append(rs, helpers.NewJUnitReporter(config, componentName))
	}

	RunSpecsWithDefaultAndCustomReporters(t, componentName, rs)
}
func getBuildpackVersion(name string) string {
	buildpack := cf.Cf("curl", fmt.Sprintf("/v2/buildpacks?q=name:%s", name))
	Expect(buildpack.Wait()).To(Exit(0))
	type Buildpack struct {
		Resources []struct {
			Entity struct {
				FileName string
			}
		}
	}
	var b Buildpack
	Expect(json.Unmarshal(buildpack.Out.Contents(), &b)).To(Succeed())
	if len(b.Resources) == 0 {
		return "0.0.0"
	}
	re := regexp.MustCompile(`[0-9]+\.[0-9]+\.[0-9]+`)
	return re.FindString(b.Resources[0].Entity.FileName)
}

func versionGreaterThan(version string, inputMajor, inputMinor, inputPatch int) bool {
	versions := strings.Split(version, ".")
	major, _ := strconv.Atoi(versions[0])
	if major > inputMajor {
		return true
	}
	minor, _ := strconv.Atoi(versions[1])
	if minor > inputMinor {
		return true
	}
	patch, _ := strconv.Atoi(versions[2])
	if patch > inputPatch {
		return true
	}
	return false
}

func createOrGetIsolationSegment(isolationSegmentName string) string {
	// This could go in cf-test-helpers
	guid := getIsolationSegmentGuid(isolationSegmentName)
	if guid == "" {
		Eventually(cf.Cf("curl", "/v3/isolation_segments/", "-X", "POST", "-d", fmt.Sprintf(`{"name":"%s"}`, isolationSegmentName))).Should(Exit(0))
		guid = getIsolationSegmentGuid(isolationSegmentName)
	}
	return guid
}

func getIsolationSegmentGuid(isolationSegmentName string) string {
	return getV3ResourceGuid(fmt.Sprintf("/v3/isolation_segments?names=%s", isolationSegmentName))
}

func attachIsolationSegmentToOrg(environment *ReproducibleTestSuiteSetup, isoSegGuid string) {
	orgGuid := getOrganizationGuid(environment.GetOrganizationName())
	response := cf.Cf(
		"curl",
		fmt.Sprintf("/v3/isolation_segments/%s/relationships/organizations", isoSegGuid),
		"-X",
		"POST",
		"-d",
		fmt.Sprintf(`{"data":[{"guid": "%s"}]}`, orgGuid),
	)
	Expect(response.Wait()).To(Exit(0))
}

func attachIsolationSegmentToSpace(environment *ReproducibleTestSuiteSetup, isoSegGuid string) {
	spaceGuid := getSpaceGuidForOrg(getOrganizationGuid(environment.GetOrganizationName()))
	response := cf.Cf(
		"curl",
		fmt.Sprintf("/v2/spaces/%s", spaceGuid),
		"-X",
		"PUT",
		"-d",
		fmt.Sprintf(`{"isolation_segment_guid":"%s"}`, isoSegGuid),
	)
	Expect(response.Wait()).To(Exit(0))
}

func getOrganizationGuid(organizationName string) string {
	return getV2ResourceGuid(fmt.Sprintf("/v2/organizations?q=name:%s", organizationName))
}

func getSpaceGuidForOrg(orgGuid string) string {
	return getV2ResourceGuid(fmt.Sprintf("/v2/organizations/%s/spaces", orgGuid))
}

func getV2ResourceGuid(endpoint string) string {
	response := cf.Cf("curl", endpoint, "-X", "GET")
	Expect(response.Wait()).To(Exit(0))
	var r struct {
		Resources []struct {
			Metadata struct {
				Guid string
			}
		}
	}
	Expect(json.Unmarshal(response.Out.Contents(), &r)).To(Succeed())
	Expect(len(r.Resources)).To(Equal(1))
	return r.Resources[0].Metadata.Guid
}

func getV3ResourceGuid(endpoint string) string {
	response := cf.Cf("curl", endpoint, "-X", "GET")
	Expect(response.Wait()).To(Exit(0))
	var r struct {
		Resources []struct {
			Guid string
		}
	}
	Expect(json.Unmarshal(response.Out.Contents(), &r)).To(Succeed())
	if len(r.Resources) == 0 {
		return ""
	}
	return r.Resources[0].Guid
}
<|endoftext|>


<commit_before>package api

import (
	"context"
	"encoding/json"
	"net/http"

	"github.com/go-chi/chi"
	"github.com/gobuffalo/uuid"
	"github.com/netlify/gotrue/conf"
	"github.com/netlify/gotrue/models"
	"github.com/pkg/errors"
)

func (a *API) loadInstance(w http.ResponseWriter, r *http.Request) (context.Context, error) {
	instanceID, err := uuid.FromString(chi.URLParam(r, "instance_id"))
	if err != nil {
		return nil, badRequestError("Invalid instance ID")
	}
	logEntrySetField(r, "instance_id", instanceID)

	i, err := models.GetInstance(a.db, instanceID)
	if err != nil {
		if models.IsNotFoundError(err) {
			return nil, notFoundError("Instance not found")
		}
		return nil, internalServerError("Database error loading instance").WithInternalError(err)
	}

	return withInstance(r.Context(), i), nil
}

func (a *API) GetAppManifest(w http.ResponseWriter, r *http.Request) error {
	// TODO update to real manifest
	return sendJSON(w, http.StatusOK, map[string]string{
		"version":     a.version,
		"name":        "GoTrue",
		"description": "GoTrue is a user registration and authentication API",
	})
}

type InstanceRequestParams struct {
	UUID       uuid.UUID           `json:"uuid"`
	BaseConfig *conf.Configuration `json:"config"`
}

type InstanceResponse struct {
	models.Instance
	Endpoint string `json:"endpoint"`
	State    string `json:"state"`
}

func (a *API) CreateInstance(w http.ResponseWriter, r *http.Request) error {
	params := InstanceRequestParams{}
	if err := json.NewDecoder(r.Body).Decode(&params); err != nil {
		return badRequestError("Error decoding params: %v", err)
	}

	_, err := models.GetInstanceByUUID(a.db, params.UUID)
	if err != nil {
		if !models.IsNotFoundError(err) {
			return internalServerError("Database error looking up instance").WithInternalError(err)
		}
	} else {
		return badRequestError("An instance with that UUID already exists")
	}

	id, err := uuid.NewV4()
	if err != nil {
		return errors.Wrap(err, "Error generating id")
	}

	i := models.Instance{
		ID:         id,
		UUID:       params.UUID,
		BaseConfig: params.BaseConfig,
	}
	if err = a.db.Create(&i); err != nil {
		return internalServerError("Database error creating instance").WithInternalError(err)
	}

	// hide pass in response
	if i.BaseConfig != nil {
		i.BaseConfig.SMTP.Pass = ""
	}

	resp := InstanceResponse{
		Instance: i,
		Endpoint: a.config.API.Endpoint,
		State:    "active",
	}
	return sendJSON(w, http.StatusCreated, resp)
}

func (a *API) GetInstance(w http.ResponseWriter, r *http.Request) error {
	i := getInstance(r.Context())
	if i.BaseConfig != nil {
		i.BaseConfig.SMTP.Pass = ""
	}
	return sendJSON(w, http.StatusOK, i)
}

func (a *API) UpdateInstance(w http.ResponseWriter, r *http.Request) error {
	i := getInstance(r.Context())

	params := InstanceRequestParams{}
	if err := json.NewDecoder(r.Body).Decode(&params); err != nil {
		return badRequestError("Error decoding params: %v", err)
	}

	if params.BaseConfig != nil {
		if err := i.UpdateConfig(a.db, params.BaseConfig); err != nil {
			return internalServerError("Database error updating instance").WithInternalError(err)
		}
		i.BaseConfig.SMTP.Pass = "" // hide from response
	}
	return sendJSON(w, http.StatusOK, i)
}

func (a *API) DeleteInstance(w http.ResponseWriter, r *http.Request) error {
	i := getInstance(r.Context())
	if err := models.DeleteInstance(a.db, i); err != nil {
		return internalServerError("Database error deleting instance").WithInternalError(err)
	}

	w.WriteHeader(http.StatusNoContent)
	return nil
}
<commit_msg>Fix hiding smtp cred in update call<commit_after>package api

import (
	"context"
	"encoding/json"
	"net/http"

	"github.com/go-chi/chi"
	"github.com/gobuffalo/uuid"
	"github.com/netlify/gotrue/conf"
	"github.com/netlify/gotrue/models"
	"github.com/pkg/errors"
)

func (a *API) loadInstance(w http.ResponseWriter, r *http.Request) (context.Context, error) {
	instanceID, err := uuid.FromString(chi.URLParam(r, "instance_id"))
	if err != nil {
		return nil, badRequestError("Invalid instance ID")
	}
	logEntrySetField(r, "instance_id", instanceID)

	i, err := models.GetInstance(a.db, instanceID)
	if err != nil {
		if models.IsNotFoundError(err) {
			return nil, notFoundError("Instance not found")
		}
		return nil, internalServerError("Database error loading instance").WithInternalError(err)
	}

	return withInstance(r.Context(), i), nil
}

func (a *API) GetAppManifest(w http.ResponseWriter, r *http.Request) error {
	// TODO update to real manifest
	return sendJSON(w, http.StatusOK, map[string]string{
		"version":     a.version,
		"name":        "GoTrue",
		"description": "GoTrue is a user registration and authentication API",
	})
}

type InstanceRequestParams struct {
	UUID       uuid.UUID           `json:"uuid"`
	BaseConfig *conf.Configuration `json:"config"`
}

type InstanceResponse struct {
	models.Instance
	Endpoint string `json:"endpoint"`
	State    string `json:"state"`
}

func (a *API) CreateInstance(w http.ResponseWriter, r *http.Request) error {
	params := InstanceRequestParams{}
	if err := json.NewDecoder(r.Body).Decode(&params); err != nil {
		return badRequestError("Error decoding params: %v", err)
	}

	_, err := models.GetInstanceByUUID(a.db, params.UUID)
	if err != nil {
		if !models.IsNotFoundError(err) {
			return internalServerError("Database error looking up instance").WithInternalError(err)
		}
	} else {
		return badRequestError("An instance with that UUID already exists")
	}

	id, err := uuid.NewV4()
	if err != nil {
		return errors.Wrap(err, "Error generating id")
	}

	i := models.Instance{
		ID:         id,
		UUID:       params.UUID,
		BaseConfig: params.BaseConfig,
	}
	if err = a.db.Create(&i); err != nil {
		return internalServerError("Database error creating instance").WithInternalError(err)
	}

	// hide pass in response
	if i.BaseConfig != nil {
		i.BaseConfig.SMTP.Pass = ""
	}

	resp := InstanceResponse{
		Instance: i,
		Endpoint: a.config.API.Endpoint,
		State:    "active",
	}
	return sendJSON(w, http.StatusCreated, resp)
}

func (a *API) GetInstance(w http.ResponseWriter, r *http.Request) error {
	i := getInstance(r.Context())
	if i.BaseConfig != nil {
		i.BaseConfig.SMTP.Pass = ""
	}
	return sendJSON(w, http.StatusOK, i)
}

func (a *API) UpdateInstance(w http.ResponseWriter, r *http.Request) error {
	i := getInstance(r.Context())

	params := InstanceRequestParams{}
	if err := json.NewDecoder(r.Body).Decode(&params); err != nil {
		return badRequestError("Error decoding params: %v", err)
	}

	if params.BaseConfig != nil {
		if err := i.UpdateConfig(a.db, params.BaseConfig); err != nil {
			return internalServerError("Database error updating instance").WithInternalError(err)
		}
	}

	// Hide SMTP credential from response
	if i.BaseConfig != nil {
		i.BaseConfig.SMTP.Pass = ""
	}
	return sendJSON(w, http.StatusOK, i)
}

func (a *API) DeleteInstance(w http.ResponseWriter, r *http.Request) error {
	i := getInstance(r.Context())
	if err := models.DeleteInstance(a.db, i); err != nil {
		return internalServerError("Database error deleting instance").WithInternalError(err)
	}

	w.WriteHeader(http.StatusNoContent)
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"net/http"

	"github.com/labstack/echo"
	"github.com/sys-cat/Kinsokujiko"
)

type Kinsoku struct {
	Sentence string `json:"sentence" form:"sentence"`
}

func main() {
	e := echo.New()
	e.POST("/", func(c echo.Context) error {
		u := new(Kinsoku)
		if err := c.Bind(u); err != nil {
			return c.String(http.StatusNotFound, err.Error())
		}
		kin := Kinsokujiko.Tokenize(Kinsokujiko.Master{u.Sentence})
		return c.JSON(http.StatusOK, kin)
	})
	e.Logger.Fatal(e.Start(":9090"))
}
<commit_msg>add urls<commit_after>package main

import (
	"net/http"

	"github.com/labstack/echo"
	"github.com/sys-cat/Kinsokujiko"
)

type Kinsoku struct {
	Sentence string `json:"sentence" form:"sentence"`
}

func main() {
	e := echo.New()
	e.POST("/", func(c echo.Context) error {
		u := new(Kinsoku)
		if err := c.Bind(u); err != nil {
			return c.String(http.StatusNotFound, err.Error())
		}
		kin := Kinsokujiko.Tokenize(Kinsokujiko.Master{u.Sentence})
		return c.JSON(http.StatusOK, kin)
	})
	// Dictionary
	e.GET("/dictionary/read", func(c echo.Context) error {
		return c.JSON(http.StatusOK, "/dictionary/read")
	})
	e.POST("/dictionary/update", func(c echo.Context) error {
		return c.JSON(http.StatusOK, "/dictionary/update")
	})
	// Targets
	e.PUT("/targets/create", func(c echo.Context) error {
		return c.JSON(http.StatusOK, "/targets/create")
	})
	e.GET("/targets/read", func(c echo.Context) error {
		return c.JSON(http.StatusOK, "/targets/read")
	})
	e.POST("/targets/update", func(c echo.Context) error {
		return c.JSON(http.StatusOK, "/targets/update")
	})
	e.DELETE("/targets/delete", func(c echo.Context) error {
		return c.JSON(http.StatusOK, "/targets/delete")
	})
	e.Logger.Fatal(e.Start(":9090"))
}
<|endoftext|>


<commit_before>package langserver

import (
	"context"
	"fmt"
	"go/build"
	"go/parser"
	"go/token"
	"go/types"
	"log"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"sync"

	"golang.org/x/tools/go/buildutil"
	"golang.org/x/tools/go/loader"

	"github.com/neelance/parallel"
	opentracing "github.com/opentracing/opentracing-go"
	"github.com/opentracing/opentracing-go/ext"
	"github.com/sourcegraph/go-langserver/langserver/internal/refs"
	"github.com/sourcegraph/go-langserver/pkg/lspext"
	"github.com/sourcegraph/jsonrpc2"
)

func (h *LangHandler) handleWorkspaceReference(ctx context.Context, conn JSONRPC2Conn, req *jsonrpc2.Request, params lspext.WorkspaceReferenceParams) ([]lspext.ReferenceInformation, error) {
	rootPath := h.FilePath(h.init.RootPath)
	bctx := h.OverlayBuildContext(ctx, h.defaultBuildContext(), !h.init.NoOSFileSystemAccess)

	var pkgPat string
	if h.init.RootImportPath == "" {
		// Go stdlib (empty root import path)
		pkgPat = "..."
	} else {
		// All other Go packages.
		pkgPat = h.init.RootImportPath + "/..."
	}

	var parallelism int
	if envWorkspaceReferenceParallelism != "" {
		var err error
		parallelism, err = strconv.Atoi(envWorkspaceReferenceParallelism)
		if err != nil {
			return nil, err
		}
	} else {
		parallelism = runtime.NumCPU() / 4 // 1/4 CPU
	}
	if parallelism < 1 {
		parallelism = 1
	}

	// Perform typechecking.
	var (
		fset = token.NewFileSet()
		pkgs []string
	)
	for pkg := range buildutil.ExpandPatterns(bctx, []string{pkgPat}) {
		// Ignore any vendor package so we can avoid scanning it for external
		// references, per the workspace/reference spec. This saves us a
		// considerable amount of work.
		bpkg, err := bctx.Import(pkg, rootPath, build.FindOnly)
		if err != nil && !isMultiplePackageError(err) {
			log.Printf("skipping possible package %s: %s", pkg, err)
			continue
		}
		if IsVendorDir(bpkg.Dir) {
			continue
		}
		pkgs = append(pkgs, pkg)
	}
	prog, err := h.externalRefsTypecheck(ctx, bctx, conn, fset, pkgs)
	if err != nil {
		return nil, err
	}

	// Collect external references.
	results := refResultSorter{results: make([]lspext.ReferenceInformation, 0)}
	par := parallel.NewRun(parallelism)
	for _, pkg := range prog.Imported {
		par.Acquire()
		go func(pkg *loader.PackageInfo) {
			defer par.Release()
			// Prevent any uncaught panics from taking the entire server down.
			defer func() {
				if r := recover(); r != nil {
					// Same as net/http
					const size = 64 << 10
					buf := make([]byte, size)
					buf = buf[:runtime.Stack(buf, false)]
					log.Printf("ignoring panic serving %v for pkg %v: %v\n%s", req.Method, pkg, r, buf)
					return
				}
			}()
			err := h.externalRefsFromPkg(ctx, bctx, conn, fset, pkg, rootPath, &results)
			if err != nil {
				log.Printf("externalRefsFromPkg: %v: %v", pkg, err)
			}
		}(pkg)
	}
	_ = par.Wait()

	sort.Sort(&results) // sort to provide consistent results

	// TODO: We calculate all the results and then throw them away. If we ever
	// decide to begin using limiting, we can improve the performance of this
	// dramatically. For now, it lives in the spec just so that other
	// implementations are aware it may need to be done and to design with that
	// in mind.
	if len(results.results) > params.Limit && params.Limit > 0 {
		results.results = results.results[:params.Limit]
	}
	return results.results, nil
}

func (h *LangHandler) externalRefsTypecheck(ctx context.Context, bctx *build.Context, conn JSONRPC2Conn, fset *token.FileSet, pkgs []string) (prog *loader.Program, err error) {
	span, ctx := opentracing.StartSpanFromContext(ctx, "externalRefsTypecheck")
	defer func() {
		if err != nil {
			ext.Error.Set(span, true)
			span.SetTag("err", err.Error())
		}
		span.Finish()
	}()

	// Configure the loader.
	var typeErrs []error
	conf := loader.Config{
		Fset: fset,
		TypeChecker: types.Config{
			DisableUnusedImportCheck: true,
			FakeImportC:              true,
			Error: func(err error) {
				typeErrs = append(typeErrs, err)
			},
		},
		Build:       bctx,
		AllowErrors: true,
		ParserMode:  parser.AllErrors | parser.ParseComments, // prevent parser from bailing out
		FindPackage: func(bctx *build.Context, importPath, fromDir string, mode build.ImportMode) (*build.Package, error) {
			// When importing a package, ignore any
			// MultipleGoErrors. This occurs, e.g., when you have a
			// main.go with "// +build ignore" that imports the
			// non-main package in the same dir.
			bpkg, err := bctx.Import(importPath, fromDir, mode)
			if err != nil && !isMultiplePackageError(err) {
				return bpkg, err
			}
			return bpkg, nil
		},
	}
	for _, path := range pkgs {
		conf.Import(path)
	}

	// Load and typecheck the packages.
	prog, err = conf.Load()
	if err != nil && prog == nil {
		return nil, err
	}

	// Publish typechecking error diagnostics.
	diags, err := errsToDiagnostics(typeErrs, prog)
	if err != nil {
		return nil, err
	}
	if len(diags) > 0 {
		go func() {
			if err := h.publishDiagnostics(ctx, conn, diags); err != nil {
				log.Printf("warning: failed to send diagnostics: %s.", err)
			}
		}()
	}
	return prog, nil
}

// externalRefsFromPkg collects all the external references from the specified
// package and returns the results.
func (h *LangHandler) externalRefsFromPkg(ctx context.Context, bctx *build.Context, conn JSONRPC2Conn, fs *token.FileSet, pkg *loader.PackageInfo, rootPath string, results *refResultSorter) (err error) {
	span, ctx := opentracing.StartSpanFromContext(ctx, "externalRefsFromPkg")
	defer func() {
		if err != nil {
			ext.Error.Set(span, true)
			span.SetTag("err", err.Error())
		}
		span.Finish()
	}()
	span.SetTag("pkg", pkg)

	pkgInWorkspace := func(path string) bool {
		return PathHasPrefix(path, h.init.RootImportPath)
	}

	// Compute external references.
	cfg := &refs.Config{
		FileSet:  fs,
		Pkg:      pkg.Pkg,
		PkgFiles: pkg.Files,
		Info:     &pkg.Info,
	}
	refsErr := cfg.Refs(func(r *refs.Ref) {
		var defName, defContainerName string
		if fields := strings.Fields(r.Def.Path); len(fields) > 0 {
			defName = fields[0]
			defContainerName = strings.Join(fields[1:], " ")
		}
		if defContainerName == "" {
			defContainerName = r.Def.PackageName
		}

		defPkg, err := bctx.Import(r.Def.ImportPath, rootPath, build.FindOnly)
		if err != nil {
			// Log the error, and flag it as one in the trace -- but do not
			// halt execution (hopefully, it is limited to a small subset of
			// the data).
			ext.Error.Set(span, true)
			err := fmt.Errorf("externalRefsFromPkg: failed to import %v: %v", r.Def.ImportPath, err)
			log.Println(err)
			span.SetTag("error", err.Error())
			return
		}

		// If the symbol the reference is to is defined within this workspace,
		// exclude it. We only emit refs to symbols that are external to the
		// workspace.
		if pkgInWorkspace(defPkg.ImportPath) {
			return
		}

		results.resultsMu.Lock()
		results.results = append(results.results, lspext.ReferenceInformation{
			Name:          defName,
			ContainerName: defContainerName,
			URI:           "file://" + defPkg.Dir,
			Location:      goRangeToLSPLocation(fs, token.Pos(r.Position.Offset), token.Pos(r.Position.Offset+len(defName)-1)),
		})
		results.resultsMu.Unlock()
	})
	if refsErr != nil {
		// Trace the error, but do not consider it a true error. In many cases
		// it is a problem with the user's code, not our external reference
		// finding code.
		span.SetTag("err", fmt.Sprintf("externalRefsFromPkg: external refs failed: %v: %v", pkg, refsErr))
	}
	return nil
}

// refResultSorter is a utility struct for collecting, filtering, and
// sorting external reference results.
type refResultSorter struct {
	results   []lspext.ReferenceInformation
	resultsMu sync.Mutex
}

func (s *refResultSorter) Len() int      { return len(s.results) }
func (s *refResultSorter) Swap(i, j int) { s.results[i], s.results[j] = s.results[j], s.results[i] }
func (s *refResultSorter) Less(i, j int) bool {
	return s.results[i].Location.URI < s.results[j].Location.URI
}
<commit_msg>langserver: make tests pass / further demonstrate invalid token.Position.Offset usage<commit_after>package langserver

import (
	"context"
	"fmt"
	"go/build"
	"go/parser"
	"go/token"
	"go/types"
	"log"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"sync"

	"golang.org/x/tools/go/buildutil"
	"golang.org/x/tools/go/loader"

	"github.com/neelance/parallel"
	opentracing "github.com/opentracing/opentracing-go"
	"github.com/opentracing/opentracing-go/ext"
	"github.com/sourcegraph/go-langserver/langserver/internal/refs"
	"github.com/sourcegraph/go-langserver/pkg/lspext"
	"github.com/sourcegraph/jsonrpc2"
)

func (h *LangHandler) handleWorkspaceReference(ctx context.Context, conn JSONRPC2Conn, req *jsonrpc2.Request, params lspext.WorkspaceReferenceParams) ([]lspext.ReferenceInformation, error) {
	rootPath := h.FilePath(h.init.RootPath)
	bctx := h.OverlayBuildContext(ctx, h.defaultBuildContext(), !h.init.NoOSFileSystemAccess)

	var pkgPat string
	if h.init.RootImportPath == "" {
		// Go stdlib (empty root import path)
		pkgPat = "..."
	} else {
		// All other Go packages.
		pkgPat = h.init.RootImportPath + "/..."
	}

	var parallelism int
	if envWorkspaceReferenceParallelism != "" {
		var err error
		parallelism, err = strconv.Atoi(envWorkspaceReferenceParallelism)
		if err != nil {
			return nil, err
		}
	} else {
		parallelism = runtime.NumCPU() / 4 // 1/4 CPU
	}
	if parallelism < 1 {
		parallelism = 1
	}

	// Perform typechecking.
	var (
		fset = token.NewFileSet()
		pkgs []string
	)
	for pkg := range buildutil.ExpandPatterns(bctx, []string{pkgPat}) {
		// Ignore any vendor package so we can avoid scanning it for external
		// references, per the workspace/reference spec. This saves us a
		// considerable amount of work.
		bpkg, err := bctx.Import(pkg, rootPath, build.FindOnly)
		if err != nil && !isMultiplePackageError(err) {
			log.Printf("skipping possible package %s: %s", pkg, err)
			continue
		}
		if IsVendorDir(bpkg.Dir) {
			continue
		}
		pkgs = append(pkgs, pkg)
	}
	prog, err := h.externalRefsTypecheck(ctx, bctx, conn, fset, pkgs)
	if err != nil {
		return nil, err
	}

	// Collect external references.
	results := refResultSorter{results: make([]lspext.ReferenceInformation, 0)}
	par := parallel.NewRun(parallelism)
	for _, pkg := range prog.Imported {
		par.Acquire()
		go func(pkg *loader.PackageInfo) {
			defer par.Release()
			// Prevent any uncaught panics from taking the entire server down.
			defer func() {
				if r := recover(); r != nil {
					// Same as net/http
					const size = 64 << 10
					buf := make([]byte, size)
					buf = buf[:runtime.Stack(buf, false)]
					log.Printf("ignoring panic serving %v for pkg %v: %v\n%s", req.Method, pkg, r, buf)
					return
				}
			}()
			err := h.externalRefsFromPkg(ctx, bctx, conn, fset, pkg, rootPath, &results)
			if err != nil {
				log.Printf("externalRefsFromPkg: %v: %v", pkg, err)
			}
		}(pkg)
	}
	_ = par.Wait()

	sort.Sort(&results) // sort to provide consistent results

	// TODO: We calculate all the results and then throw them away. If we ever
	// decide to begin using limiting, we can improve the performance of this
	// dramatically. For now, it lives in the spec just so that other
	// implementations are aware it may need to be done and to design with that
	// in mind.
	if len(results.results) > params.Limit && params.Limit > 0 {
		results.results = results.results[:params.Limit]
	}
	return results.results, nil
}

func (h *LangHandler) externalRefsTypecheck(ctx context.Context, bctx *build.Context, conn JSONRPC2Conn, fset *token.FileSet, pkgs []string) (prog *loader.Program, err error) {
	span, ctx := opentracing.StartSpanFromContext(ctx, "externalRefsTypecheck")
	defer func() {
		if err != nil {
			ext.Error.Set(span, true)
			span.SetTag("err", err.Error())
		}
		span.Finish()
	}()

	// Configure the loader.
	var typeErrs []error
	conf := loader.Config{
		Fset: fset,
		TypeChecker: types.Config{
			DisableUnusedImportCheck: true,
			FakeImportC:              true,
			Error: func(err error) {
				typeErrs = append(typeErrs, err)
			},
		},
		Build:       bctx,
		AllowErrors: true,
		ParserMode:  parser.AllErrors | parser.ParseComments, // prevent parser from bailing out
		FindPackage: func(bctx *build.Context, importPath, fromDir string, mode build.ImportMode) (*build.Package, error) {
			// When importing a package, ignore any
			// MultipleGoErrors. This occurs, e.g., when you have a
			// main.go with "// +build ignore" that imports the
			// non-main package in the same dir.
			bpkg, err := bctx.Import(importPath, fromDir, mode)
			if err != nil && !isMultiplePackageError(err) {
				return bpkg, err
			}
			return bpkg, nil
		},
	}
	for _, path := range pkgs {
		conf.Import(path)
	}

	// Load and typecheck the packages.
	prog, err = conf.Load()
	if err != nil && prog == nil {
		return nil, err
	}

	// Publish typechecking error diagnostics.
	diags, err := errsToDiagnostics(typeErrs, prog)
	if err != nil {
		return nil, err
	}
	if len(diags) > 0 {
		go func() {
			if err := h.publishDiagnostics(ctx, conn, diags); err != nil {
				log.Printf("warning: failed to send diagnostics: %s.", err)
			}
		}()
	}
	return prog, nil
}

// externalRefsFromPkg collects all the external references from the specified
// package and returns the results.
func (h *LangHandler) externalRefsFromPkg(ctx context.Context, bctx *build.Context, conn JSONRPC2Conn, fs *token.FileSet, pkg *loader.PackageInfo, rootPath string, results *refResultSorter) (err error) {
	span, ctx := opentracing.StartSpanFromContext(ctx, "externalRefsFromPkg")
	defer func() {
		if err != nil {
			ext.Error.Set(span, true)
			span.SetTag("err", err.Error())
		}
		span.Finish()
	}()
	span.SetTag("pkg", pkg)

	pkgInWorkspace := func(path string) bool {
		return PathHasPrefix(path, h.init.RootImportPath)
	}

	// Compute external references.
	cfg := &refs.Config{
		FileSet:  fs,
		Pkg:      pkg.Pkg,
		PkgFiles: pkg.Files,
		Info:     &pkg.Info,
	}
	refsErr := cfg.Refs(func(r *refs.Ref) {
		var defName, defContainerName string
		if fields := strings.Fields(r.Def.Path); len(fields) > 0 {
			defName = fields[0]
			defContainerName = strings.Join(fields[1:], " ")
		}
		if defContainerName == "" {
			defContainerName = r.Def.PackageName
		}

		defPkg, err := bctx.Import(r.Def.ImportPath, rootPath, build.FindOnly)
		if err != nil {
			// Log the error, and flag it as one in the trace -- but do not
			// halt execution (hopefully, it is limited to a small subset of
			// the data).
			ext.Error.Set(span, true)
			err := fmt.Errorf("externalRefsFromPkg: failed to import %v: %v", r.Def.ImportPath, err)
			log.Println(err)
			span.SetTag("error", err.Error())
			return
		}

		// If the symbol the reference is to is defined within this workspace,
		// exclude it. We only emit refs to symbols that are external to the
		// workspace.
		if pkgInWorkspace(defPkg.ImportPath) {
			return
		}

		results.resultsMu.Lock()
		results.results = append(results.results, lspext.ReferenceInformation{
			Name:          defName,
			ContainerName: defContainerName,
			URI:           "file://" + defPkg.Dir,
			// TODO: This code is EXTREMELY invalid!
			Location: goRangeToLSPLocation(fs, token.Pos(r.Position.Offset)+1, token.Pos(r.Position.Offset+len(defName)-1)),
		})
		results.resultsMu.Unlock()
	})
	if refsErr != nil {
		// Trace the error, but do not consider it a true error. In many cases
		// it is a problem with the user's code, not our external reference
		// finding code.
		span.SetTag("err", fmt.Sprintf("externalRefsFromPkg: external refs failed: %v: %v", pkg, refsErr))
	}
	return nil
}

// refResultSorter is a utility struct for collecting, filtering, and
// sorting external reference results.
type refResultSorter struct {
	results   []lspext.ReferenceInformation
	resultsMu sync.Mutex
}

func (s *refResultSorter) Len() int      { return len(s.results) }
func (s *refResultSorter) Swap(i, j int) { s.results[i], s.results[j] = s.results[j], s.results[i] }
func (s *refResultSorter) Less(i, j int) bool {
	return s.results[i].Location.URI < s.results[j].Location.URI
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math"
	"math/rand"

	"sort"
)

import "github.com/golang/glog"

import (
	"skia.googlesource.com/buildbot.git/perf/go/ctrace"
	"skia.googlesource.com/buildbot.git/perf/go/kmeans"
	"skia.googlesource.com/buildbot.git/perf/go/types"
)

const (
	MAX_SAMPLE_TRACES_PER_CLUSTER = 5

	// K is the k in k-means.
	K = 50

	// MAX_KMEANS_ITERATIONS is the maximum number of k-means iterations to run.
	MAX_KMEANS_ITERATIONS = 100

	// KMEAN_EPSILON is the smallest change in the k-means total error we will
	// accept per iteration.  If the change in error falls below KMEAN_EPSILON
	// the iteration will terminate.
	KMEAN_EPSILON = 1.0

	// INTERESTING_THRESHHOLD is the threshhold value beyond which
	// StepFit.Regression values become interesting, i.e. they may indicate real
	// regressions or improvements.
	INTERESTING_THRESHHOLD = 150.0
)

// ValueWeight is a weight proportional to the number of times the parameter
// Value appears in a cluster. Used in ClusterSummary.
type ValueWeight struct {
	Value  string
	Weight int
}

// StepFit stores information on the best Step Function fit on a trace.
type StepFit struct {
	// LeastSquares is the Least Squares error for a step function curve fit to the trace.
	LeastSquares float64

	// TurningPoint is the index where the Step Function changes value.
	TurningPoint int

	// StepSize is the size of the step in the step function. Negative values
	// indicate a step up, i.e. they look like a performance regression in the
	// trace, as opposed to positive values which look like performance
	// improvements.
	StepSize float64

	// The "Regression" value is calculated as Step Size / Least Squares Error.
	//
	// The better the fit the larger the number returned, because LSE
	// gets smaller with a better fit. The higher the Step Size the
	// larger the number returned.
	Regression float64

	// Status of the cluster.
	//
	// Values can be "High", "Low", and "Uninteresting"
	Status string
}

// ClusterSummary is a summary of a single cluster of traces.
type ClusterSummary struct {
	// Traces contains at most MAX_SAMPLE_TRACES_PER_CLUSTER sample traces, the first is the centroid.
	Traces [][][]float64

	// Keys of all the members of the Cluster.
	Keys []string

	// ParamSummaries is a summary of all the parameters in the cluster.
	ParamSummaries [][]ValueWeight

	// StepFit is info on the fit of the centroid to a step function.
	StepFit *StepFit
}

// ClusterSummaries is one summary for each cluster that the k-means clustering
// found.
type ClusterSummaries struct {
	Clusters         []*ClusterSummary
	StdDevThreshhold float64
	K                int
}

func NewClusterSummaries() *ClusterSummaries {
	return &ClusterSummaries{
		Clusters:         []*ClusterSummary{},
		StdDevThreshhold: ctrace.MIN_STDDEV,
		K:                K,
	}
}

// chooseK chooses a random sample of k observations. Used as the starting
// point for the k-means clustering.
func chooseK(observations []kmeans.Clusterable, k int) []kmeans.Clusterable {
	popN := len(observations)
	centroids := make([]kmeans.Clusterable, k)
	for i := 0; i < k; i++ {
		o := observations[rand.Intn(popN)].(*ctrace.ClusterableTrace)
		cp := &ctrace.ClusterableTrace{
			Key:    "I'm a centroid",
			Values: make([]float64, len(o.Values)),
		}
		copy(cp.Values, o.Values)
		centroids[i] = cp
	}
	return centroids
}

// traceToFlot converts the data into a format acceptable to the Flot plotting
// library.
//
// Flot expects data formatted as an array of [x, y] pairs.
func traceToFlot(t *ctrace.ClusterableTrace) [][]float64 {
	ret := make([][]float64, len(t.Values))
	for i, x := range t.Values {
		ret[i] = []float64{float64(i), x}
	}
	return ret
}

// ValueWeightSortable is a utility class for sorting the ValueWeight's by Weight.
type ValueWeightSortable []ValueWeight

func (p ValueWeightSortable) Len() int           { return len(p) }
func (p ValueWeightSortable) Less(i, j int) bool { return p[i].Weight > p[j].Weight } // Descending.
func (p ValueWeightSortable) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

// getParamSummaries summaries all the parameters for all observations in a cluster.
//
// The return value is an array of []ValueWeight's, one []ValueWeight per
// parameter. The members of each []ValueWeight are sorted by the Weight, with
// higher Weight's first.
func getParamSummaries(cluster []kmeans.Clusterable) [][]ValueWeight {
	// For each cluster member increment each parameters count.
	type ValueMap map[string]int
	counts := map[string]ValueMap{}
	clusterSize := float64(len(cluster))
	// First figure out what parameters and values appear in the cluster.
	for _, o := range cluster {
		for k, v := range o.(*ctrace.ClusterableTrace).Params {
			if v == "" {
				continue
			}
			if _, ok := counts[k]; !ok {
				counts[k] = ValueMap{}
				counts[k][v] = 0
			}
			counts[k][v] += 1
		}
	}
	// Now calculate the weights for each parameter value.  The weight of each
	// value is proportional to the number of times it appears on an observation
	// versus all other values for the same parameter.
	ret := make([][]ValueWeight, 0)
	for _, count := range counts {
		weights := []ValueWeight{}
		for value, weight := range count {
			weights = append(weights, ValueWeight{
				Value:  value,
				Weight: int(14*float64(weight)/clusterSize) + 12,
			})
		}
		sort.Sort(ValueWeightSortable(weights))
		ret = append(ret, weights)
	}

	return ret
}

// average calculates and returns the average value of the given []float64.
func average(xs []float64) float64 {
	total := 0.0
	for _, v := range xs {
		total += v
	}
	return total / float64(len(xs))
}

// sse calculates and returns the sum squared error from the given base of []float64.
func sse(xs []float64, base float64) float64 {
	total := 0.0
	for _, v := range xs {
		total += math.Pow(v-base, 2)
	}
	return total
}

// getStepFit takes one []float64 trace and calculates and returns a StepFit.
//
// See StepFit for a description of the values being calculated.
func getStepFit(trace []float64) *StepFit {
	lse := math.MaxFloat64
	stepSize := -1.0
	turn := 0
	for i, _ := range trace {
		if i == 0 {
			continue
		}
		y0 := average(trace[:i])
		y1 := average(trace[i:])
		if y0 == y1 {
			continue
		}
		d := math.Sqrt(sse(trace[:i], y0)+sse(trace[i:], y1)) / float64(len(trace))
		if d < lse {
			lse = d
			stepSize = (y0 - y1)
			turn = i
		}
	}
	regression := stepSize / lse
	status := "Uninteresting"
	if regression > INTERESTING_THRESHHOLD {
		status = "High"
	} else if regression < -INTERESTING_THRESHHOLD {
		status = "Low"
	}
	return &StepFit{
		LeastSquares: lse,
		StepSize:     stepSize,
		TurningPoint: turn,
		Regression:   regression,
		Status:       status,
	}
}

type SortableClusterable struct {
	Cluster  kmeans.Clusterable
	Distance float64
}

type SortableClusterableSlice []*SortableClusterable

func (p SortableClusterableSlice) Len() int           { return len(p) }
func (p SortableClusterableSlice) Less(i, j int) bool { return p[i].Distance < p[j].Distance }
func (p SortableClusterableSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

type SortableClusterSummarySlice []*ClusterSummary

func (p SortableClusterSummarySlice) Len() int { return len(p) }
func (p SortableClusterSummarySlice) Less(i, j int) bool {
	return p[i].StepFit.Regression < p[j].StepFit.Regression
}
func (p SortableClusterSummarySlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }

// GetClusterSummaries returns a summaries for each cluster.
func GetClusterSummaries(observations, centroids []kmeans.Clusterable) *ClusterSummaries {
	ret := &ClusterSummaries{
		Clusters: make([]*ClusterSummary, len(centroids)),
	}
	allClusters, _ := kmeans.GetClusters(observations, centroids)

	for i, cluster := range allClusters {
		// cluster is just an array of the observations for a given cluster.
		numSampleTraces := len(cluster)
		if numSampleTraces > MAX_SAMPLE_TRACES_PER_CLUSTER {
			numSampleTraces = MAX_SAMPLE_TRACES_PER_CLUSTER
		}
		summary := &ClusterSummary{
			Keys:           make([]string, len(cluster)-1),
			Traces:         make([][][]float64, numSampleTraces),
			ParamSummaries: getParamSummaries(cluster),
			// Try fit on the centroid.
			StepFit: getStepFit(cluster[0].(*ctrace.ClusterableTrace).Values),
		}
		for j, o := range cluster {
			if j != 0 {
				summary.Keys[j-1] = o.(*ctrace.ClusterableTrace).Key
			}
		}
		// First, sort the traces so they are order with the traces closest to the
		// centroid first.
		sc := []*SortableClusterable{}
		for j := 0; j < len(cluster); j++ {
			sc = append(sc, &SortableClusterable{Cluster: cluster[j], Distance: cluster[j].Distance(cluster[0])})
		}
		// Sort, but leave the centroid, the 0th element, unmoved.
		sort.Sort(SortableClusterableSlice(sc[1:]))

		for j, clusterable := range sc {
			if j == numSampleTraces {
				break
			}
			summary.Traces[j] = traceToFlot(clusterable.Cluster.(*ctrace.ClusterableTrace))
		}

		ret.Clusters[i] = summary
	}
	sort.Sort(SortableClusterSummarySlice(ret.Clusters))

	return ret
}

// Filter returns true if a trace should be included in clustering.
type Filter func(tr *types.Trace) bool

// calculateClusterSummaries runs k-means clustering over the trace shapes.
func calculateClusterSummaries(tile *types.Tile, k int, stddevThreshhold float64, filter Filter) (*ClusterSummaries, error) {
	lastCommitIndex := 0
	for i, c := range tile.Commits {
		if c.CommitTime != 0 {
			lastCommitIndex = i
		}
	}
	observations := make([]kmeans.Clusterable, 0, len(tile.Traces))
	for key, trace := range tile.Traces {
		if filter(trace) {
			observations = append(observations, ctrace.NewFullTrace(string(key), trace.Values[:lastCommitIndex], trace.Params, stddevThreshhold))
		}
	}
	if len(observations) == 0 {
		return nil, fmt.Errorf("Zero traces matched.")
	}

	// Create K starting centroids.
	centroids := chooseK(observations, k)
	// TODO(jcgregorio) Keep iterating until the total error stops changing.
	lastTotalError := 0.0
	for i := 0; i < MAX_KMEANS_ITERATIONS; i++ {
		centroids = kmeans.Do(observations, centroids, ctrace.CalculateCentroid)
		totalError := kmeans.TotalError(observations, centroids)
		glog.Infof("Total Error: %f\n", totalError)
		if math.Abs(totalError-lastTotalError) < KMEAN_EPSILON {
			break
		}
		lastTotalError = totalError
	}
	clusterSummaries := GetClusterSummaries(observations, centroids)
	clusterSummaries.K = k
	clusterSummaries.StdDevThreshhold = stddevThreshhold
	return clusterSummaries, nil
}
<commit_msg>Add Hash to ClusterSummary.<commit_after>package main

import (
	"fmt"
	"math"
	"math/rand"

	"sort"
)

import "github.com/golang/glog"

import (
	"skia.googlesource.com/buildbot.git/perf/go/ctrace"
	"skia.googlesource.com/buildbot.git/perf/go/kmeans"
	"skia.googlesource.com/buildbot.git/perf/go/types"
)

const (
	MAX_SAMPLE_TRACES_PER_CLUSTER = 5

	// K is the k in k-means.
	K = 50

	// MAX_KMEANS_ITERATIONS is the maximum number of k-means iterations to run.
	MAX_KMEANS_ITERATIONS = 100

	// KMEAN_EPSILON is the smallest change in the k-means total error we will
	// accept per iteration.  If the change in error falls below KMEAN_EPSILON
	// the iteration will terminate.
	KMEAN_EPSILON = 1.0

	// INTERESTING_THRESHHOLD is the threshhold value beyond which
	// StepFit.Regression values become interesting, i.e. they may indicate real
	// regressions or improvements.
	INTERESTING_THRESHHOLD = 150.0
)

// ValueWeight is a weight proportional to the number of times the parameter
// Value appears in a cluster. Used in ClusterSummary.
type ValueWeight struct {
	Value  string
	Weight int
}

// StepFit stores information on the best Step Function fit on a trace.
type StepFit struct {
	// LeastSquares is the Least Squares error for a step function curve fit to the trace.
	LeastSquares float64

	// TurningPoint is the index where the Step Function changes value.
	TurningPoint int

	// StepSize is the size of the step in the step function. Negative values
	// indicate a step up, i.e. they look like a performance regression in the
	// trace, as opposed to positive values which look like performance
	// improvements.
	StepSize float64

	// The "Regression" value is calculated as Step Size / Least Squares Error.
	//
	// The better the fit the larger the number returned, because LSE
	// gets smaller with a better fit. The higher the Step Size the
	// larger the number returned.
	Regression float64

	// Status of the cluster.
	//
	// Values can be "High", "Low", and "Uninteresting"
	Status string
}

// ClusterSummary is a summary of a single cluster of traces.
type ClusterSummary struct {
	// Traces contains at most MAX_SAMPLE_TRACES_PER_CLUSTER sample traces, the first is the centroid.
	Traces [][][]float64

	// Keys of all the members of the Cluster.
	Keys []string

	// ParamSummaries is a summary of all the parameters in the cluster.
	ParamSummaries [][]ValueWeight

	// StepFit is info on the fit of the centroid to a step function.
	StepFit *StepFit

	// Hash is the Git hash at the step point.
	Hash string
}

// ClusterSummaries is one summary for each cluster that the k-means clustering
// found.
type ClusterSummaries struct {
	Clusters         []*ClusterSummary
	StdDevThreshhold float64
	K                int
}

func NewClusterSummaries() *ClusterSummaries {
	return &ClusterSummaries{
		Clusters:         []*ClusterSummary{},
		StdDevThreshhold: ctrace.MIN_STDDEV,
		K:                K,
	}
}

// chooseK chooses a random sample of k observations. Used as the starting
// point for the k-means clustering.
func chooseK(observations []kmeans.Clusterable, k int) []kmeans.Clusterable {
	popN := len(observations)
	centroids := make([]kmeans.Clusterable, k)
	for i := 0; i < k; i++ {
		o := observations[rand.Intn(popN)].(*ctrace.ClusterableTrace)
		cp := &ctrace.ClusterableTrace{
			Key:    "I'm a centroid",
			Values: make([]float64, len(o.Values)),
		}
		copy(cp.Values, o.Values)
		centroids[i] = cp
	}
	return centroids
}

// traceToFlot converts the data into a format acceptable to the Flot plotting
// library.
//
// Flot expects data formatted as an array of [x, y] pairs.
func traceToFlot(t *ctrace.ClusterableTrace) [][]float64 {
	ret := make([][]float64, len(t.Values))
	for i, x := range t.Values {
		ret[i] = []float64{float64(i), x}
	}
	return ret
}

// ValueWeightSortable is a utility class for sorting the ValueWeight's by Weight.
type ValueWeightSortable []ValueWeight

func (p ValueWeightSortable) Len() int           { return len(p) }
func (p ValueWeightSortable) Less(i, j int) bool { return p[i].Weight > p[j].Weight } // Descending.
func (p ValueWeightSortable) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

// getParamSummaries summaries all the parameters for all observations in a cluster.
//
// The return value is an array of []ValueWeight's, one []ValueWeight per
// parameter. The members of each []ValueWeight are sorted by the Weight, with
// higher Weight's first.
func getParamSummaries(cluster []kmeans.Clusterable) [][]ValueWeight {
	// For each cluster member increment each parameters count.
	type ValueMap map[string]int
	counts := map[string]ValueMap{}
	clusterSize := float64(len(cluster))
	// First figure out what parameters and values appear in the cluster.
	for _, o := range cluster {
		for k, v := range o.(*ctrace.ClusterableTrace).Params {
			if v == "" {
				continue
			}
			if _, ok := counts[k]; !ok {
				counts[k] = ValueMap{}
				counts[k][v] = 0
			}
			counts[k][v] += 1
		}
	}
	// Now calculate the weights for each parameter value.  The weight of each
	// value is proportional to the number of times it appears on an observation
	// versus all other values for the same parameter.
	ret := make([][]ValueWeight, 0)
	for _, count := range counts {
		weights := []ValueWeight{}
		for value, weight := range count {
			weights = append(weights, ValueWeight{
				Value:  value,
				Weight: int(14*float64(weight)/clusterSize) + 12,
			})
		}
		sort.Sort(ValueWeightSortable(weights))
		ret = append(ret, weights)
	}

	return ret
}

// average calculates and returns the average value of the given []float64.
func average(xs []float64) float64 {
	total := 0.0
	for _, v := range xs {
		total += v
	}
	return total / float64(len(xs))
}

// sse calculates and returns the sum squared error from the given base of []float64.
func sse(xs []float64, base float64) float64 {
	total := 0.0
	for _, v := range xs {
		total += math.Pow(v-base, 2)
	}
	return total
}

// getStepFit takes one []float64 trace and calculates and returns a StepFit.
//
// See StepFit for a description of the values being calculated.
func getStepFit(trace []float64) *StepFit {
	lse := math.MaxFloat64
	stepSize := -1.0
	turn := 0
	for i, _ := range trace {
		if i == 0 {
			continue
		}
		y0 := average(trace[:i])
		y1 := average(trace[i:])
		if y0 == y1 {
			continue
		}
		d := math.Sqrt(sse(trace[:i], y0)+sse(trace[i:], y1)) / float64(len(trace))
		if d < lse {
			lse = d
			stepSize = (y0 - y1)
			turn = i
		}
	}
	regression := stepSize / lse
	status := "Uninteresting"
	if regression > INTERESTING_THRESHHOLD {
		status = "High"
	} else if regression < -INTERESTING_THRESHHOLD {
		status = "Low"
	}
	return &StepFit{
		LeastSquares: lse,
		StepSize:     stepSize,
		TurningPoint: turn,
		Regression:   regression,
		Status:       status,
	}
}

type SortableClusterable struct {
	Cluster  kmeans.Clusterable
	Distance float64
}

type SortableClusterableSlice []*SortableClusterable

func (p SortableClusterableSlice) Len() int           { return len(p) }
func (p SortableClusterableSlice) Less(i, j int) bool { return p[i].Distance < p[j].Distance }
func (p SortableClusterableSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

type SortableClusterSummarySlice []*ClusterSummary

func (p SortableClusterSummarySlice) Len() int { return len(p) }
func (p SortableClusterSummarySlice) Less(i, j int) bool {
	return p[i].StepFit.Regression < p[j].StepFit.Regression
}
func (p SortableClusterSummarySlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }

// GetClusterSummaries returns a summaries for each cluster.
func GetClusterSummaries(observations, centroids []kmeans.Clusterable, commits []*types.Commit) *ClusterSummaries {
	ret := &ClusterSummaries{
		Clusters: make([]*ClusterSummary, len(centroids)),
	}
	allClusters, _ := kmeans.GetClusters(observations, centroids)

	for i, cluster := range allClusters {
		// cluster is just an array of the observations for a given cluster.
		numSampleTraces := len(cluster)
		if numSampleTraces > MAX_SAMPLE_TRACES_PER_CLUSTER {
			numSampleTraces = MAX_SAMPLE_TRACES_PER_CLUSTER
		}
		stepFit := getStepFit(cluster[0].(*ctrace.ClusterableTrace).Values)
		summary := &ClusterSummary{
			Keys:           make([]string, len(cluster)-1),
			Traces:         make([][][]float64, numSampleTraces),
			ParamSummaries: getParamSummaries(cluster),
			StepFit:        stepFit,
			Hash:           commits[stepFit.TurningPoint].Hash,
		}
		for j, o := range cluster {
			if j != 0 {
				summary.Keys[j-1] = o.(*ctrace.ClusterableTrace).Key
			}
		}
		// First, sort the traces so they are order with the traces closest to the
		// centroid first.
		sc := []*SortableClusterable{}
		for j := 0; j < len(cluster); j++ {
			sc = append(sc, &SortableClusterable{Cluster: cluster[j], Distance: cluster[j].Distance(cluster[0])})
		}
		// Sort, but leave the centroid, the 0th element, unmoved.
		sort.Sort(SortableClusterableSlice(sc[1:]))

		for j, clusterable := range sc {
			if j == numSampleTraces {
				break
			}
			summary.Traces[j] = traceToFlot(clusterable.Cluster.(*ctrace.ClusterableTrace))
		}

		ret.Clusters[i] = summary
	}
	sort.Sort(SortableClusterSummarySlice(ret.Clusters))

	return ret
}

// Filter returns true if a trace should be included in clustering.
type Filter func(tr *types.Trace) bool

// calculateClusterSummaries runs k-means clustering over the trace shapes.
func calculateClusterSummaries(tile *types.Tile, k int, stddevThreshhold float64, filter Filter) (*ClusterSummaries, error) {
	lastCommitIndex := 0
	for i, c := range tile.Commits {
		if c.CommitTime != 0 {
			lastCommitIndex = i
		}
	}
	observations := make([]kmeans.Clusterable, 0, len(tile.Traces))
	for key, trace := range tile.Traces {
		if filter(trace) {
			observations = append(observations, ctrace.NewFullTrace(string(key), trace.Values[:lastCommitIndex], trace.Params, stddevThreshhold))
		}
	}
	if len(observations) == 0 {
		return nil, fmt.Errorf("Zero traces matched.")
	}

	// Create K starting centroids.
	centroids := chooseK(observations, k)
	// TODO(jcgregorio) Keep iterating until the total error stops changing.
	lastTotalError := 0.0
	for i := 0; i < MAX_KMEANS_ITERATIONS; i++ {
		centroids = kmeans.Do(observations, centroids, ctrace.CalculateCentroid)
		totalError := kmeans.TotalError(observations, centroids)
		glog.Infof("Total Error: %f\n", totalError)
		if math.Abs(totalError-lastTotalError) < KMEAN_EPSILON {
			break
		}
		lastTotalError = totalError
	}
	clusterSummaries := GetClusterSummaries(observations, centroids, tile.Commits)
	clusterSummaries.K = k
	clusterSummaries.StdDevThreshhold = stddevThreshhold
	return clusterSummaries, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Apprenda, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package lib

import (
	"log"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/ngpitt/blinkt"

	"k8s.io/client-go/tools/cache"
)

const (
	added     = iota
	updated   = iota
	deleted   = iota
	unchanged = iota
)

type Controller interface {
	Add(name, color string) bool
	Update(name, color string) bool
	Delete(name string) bool
	Watch(controller cache.Controller)
	Close()
}

type ControllerObj struct {
	brightness   float64
	resourceList []*resource
	blinkt       blinkt.Blinkt
}

type resource struct {
	name  string
	color string
	state int
}

func NewController(brightness float64) Controller {
	return &ControllerObj{
		brightness,
		make([]*resource, 0),
		blinkt.NewBlinkt(blinkt.Blue, brightness),
	}
}

func (o *ControllerObj) Add(name, color string) bool {
	o.resourceList = append(o.resourceList, &resource{name, color, added})
	o.updateBlinkt()
	return true
}

func (o *ControllerObj) Update(name, color string) bool {
	r := o.getResource(name)
	if r == nil || color == r.color {
		return false
	}
	r.color = color
	r.state = updated
	o.updateBlinkt()
	return true
}

func (o *ControllerObj) Delete(name string) bool {
	r := o.getResource(name)
	if r == nil {
		return false
	}
	r.state = deleted
	o.updateBlinkt()
	return true
}

func (o *ControllerObj) Watch(controller cache.Controller) {
	log.Println("Starting the Blinkt controller")
	stopCh := make(chan struct{}, 1)
	defer close(stopCh)
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, os.Interrupt, syscall.SIGTERM)
	go func() {
		<-sigs
		stopCh <- struct{}{}
	}()
	go controller.Run(stopCh)
	<-stopCh
}

func (o *ControllerObj) Close() {
	log.Println("Stopping the Blinkt controller")
	o.blinkt.Close(blinkt.Red, o.brightness)
}

func (o *ControllerObj) getResource(name string) *resource {
	for _, r := range o.resourceList {
		if r.name == name {
			return r
		}
	}
	return nil
}

func (o *ControllerObj) updateBlinkt() {
	i := 0
	for ; i < len(o.resourceList); i++ {
		r := o.resourceList[i]
		switch r.state {
		case added:
			fallthrough
		case updated:
			if i < 8 {
				o.blinkt.Flash(i, r.color, o.brightness, 2, 50*time.Millisecond)
				o.blinkt.Set(i, r.color, o.brightness)
			}
			r.state = unchanged
		case deleted:
			if i < 8 {
				o.blinkt.Flash(i, r.color, o.brightness, 2, 50*time.Millisecond)
			}
			o.resourceList = append(o.resourceList[:i], o.resourceList[i+1:]...)
			i--
		case unchanged:
			if i < 8 {
				o.blinkt.Set(i, r.color, o.brightness)
			}
		}
	}
	for ; i < 8; i++ {
		o.blinkt.Set(i, blinkt.Off, 0)
	}
	o.blinkt.Show()
}
<commit_msg>Updated signals<commit_after>// Copyright (c) 2017 Apprenda, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package lib

import (
	"log"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/ngpitt/blinkt"

	"k8s.io/client-go/tools/cache"
)

const (
	added     = iota
	updated   = iota
	deleted   = iota
	unchanged = iota
)

type Controller interface {
	Add(name, color string) bool
	Update(name, color string) bool
	Delete(name string) bool
	Watch(controller cache.Controller)
	Close()
}

type ControllerObj struct {
	brightness   float64
	resourceList []*resource
	blinkt       blinkt.Blinkt
}

type resource struct {
	name  string
	color string
	state int
}

func NewController(brightness float64) Controller {
	return &ControllerObj{
		brightness,
		make([]*resource, 0),
		blinkt.NewBlinkt(blinkt.Blue, brightness),
	}
}

func (o *ControllerObj) Add(name, color string) bool {
	o.resourceList = append(o.resourceList, &resource{name, color, added})
	o.updateBlinkt()
	return true
}

func (o *ControllerObj) Update(name, color string) bool {
	r := o.getResource(name)
	if r == nil || color == r.color {
		return false
	}
	r.color = color
	r.state = updated
	o.updateBlinkt()
	return true
}

func (o *ControllerObj) Delete(name string) bool {
	r := o.getResource(name)
	if r == nil {
		return false
	}
	r.state = deleted
	o.updateBlinkt()
	return true
}

func (o *ControllerObj) Watch(controller cache.Controller) {
	log.Println("Starting the Blinkt controller")
	stopCh := make(chan struct{}, 1)
	defer close(stopCh)
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
	go func() {
		<-sigs
		stopCh <- struct{}{}
	}()
	go controller.Run(stopCh)
	<-stopCh
}

func (o *ControllerObj) Close() {
	log.Println("Stopping the Blinkt controller")
	o.blinkt.Close(blinkt.Red, o.brightness)
}

func (o *ControllerObj) getResource(name string) *resource {
	for _, r := range o.resourceList {
		if r.name == name {
			return r
		}
	}
	return nil
}

func (o *ControllerObj) updateBlinkt() {
	i := 0
	for ; i < len(o.resourceList); i++ {
		r := o.resourceList[i]
		switch r.state {
		case added:
			fallthrough
		case updated:
			if i < 8 {
				o.blinkt.Flash(i, r.color, o.brightness, 2, 50*time.Millisecond)
				o.blinkt.Set(i, r.color, o.brightness)
			}
			r.state = unchanged
		case deleted:
			if i < 8 {
				o.blinkt.Flash(i, r.color, o.brightness, 2, 50*time.Millisecond)
			}
			o.resourceList = append(o.resourceList[:i], o.resourceList[i+1:]...)
			i--
		case unchanged:
			if i < 8 {
				o.blinkt.Set(i, r.color, o.brightness)
			}
		}
	}
	for ; i < 8; i++ {
		o.blinkt.Set(i, blinkt.Off, 0)
	}
	o.blinkt.Show()
}
<|endoftext|>


<commit_before>package libvirt

import (
	"github.com/alexzorin/libvirt-go"
	"github.com/mistifyio/mistify-agent/client"
	"github.com/mistifyio/mistify-agent/log"
	"github.com/mistifyio/mistify-agent/rpc"
	"net/http"
	"syscall"
)

var StateNames = map[int]string{
	libvirt.VIR_DOMAIN_NOSTATE:     "No State",
	libvirt.VIR_DOMAIN_RUNNING:     "Running",
	libvirt.VIR_DOMAIN_BLOCKED:     "Blocked",
	libvirt.VIR_DOMAIN_PAUSED:      "Paused",
	libvirt.VIR_DOMAIN_SHUTDOWN:    "Shutdown",
	libvirt.VIR_DOMAIN_CRASHED:     "Crashed",
	libvirt.VIR_DOMAIN_PMSUSPENDED: "Suspended",
	libvirt.VIR_DOMAIN_SHUTOFF:     "Shutoff",
}

type (
	Libvirt struct {
		uri         string
		connections chan *libvirt.VirConnection
		max         int
	}

	Domain struct {
		*libvirt.VirDomain
		State int
	}
)

func NewLibvirt(uri string, max int) (*Libvirt, error) {
	lv := &Libvirt{
		uri:         uri,
		max:         max,
		connections: make(chan *libvirt.VirConnection, max),
	}

	for i := 0; i < max; i++ {
		v, err := libvirt.NewVirConnection(uri)
		if err != nil {
			return nil, err
		}
		lv.connections <- &v
	}

	return lv, nil
}

func (lv *Libvirt) RunHTTP(port uint) error {
	server, err := rpc.NewServer(int(port))
	if err != nil {
		return err
	}

	server.RegisterService(lv)
	return server.ListenAndServe()
}

func (lv *Libvirt) getConnection() *libvirt.VirConnection {
	conn := <-lv.connections
	defer func() {
		lv.connections <- conn
	}()

	return conn
}

func (lv *Libvirt) LookupDomainByName(name string) (*libvirt.VirDomain, error) {
	conn := lv.getConnection()

	domain, err := conn.LookupDomainByName(name)
	if err != nil {
		return nil, err
	}

	return &domain, err
}

func (lv *Libvirt) NewDomain(guest *client.Guest) (*libvirt.VirDomain, error) {
	conn := lv.getConnection()

	xml, err := lv.DomainXML(guest)
	if err != nil {
		return nil, err
	}

	domain, err := conn.DomainDefineXML(xml)
	if err != nil {
		return nil, err
	}
	defer domain.Free()

	return &domain, err
}

func GetState(domain *libvirt.VirDomain) (int, error) {
	state, err := domain.GetState()
	if err != nil {
		return libvirt.VIR_DOMAIN_NOSTATE, err
	}

	return state[0], nil
}

func (lv *Libvirt) DomainWrapper(fn func(*libvirt.VirDomain, int) error) func(*http.Request, *rpc.GuestRequest, *rpc.GuestResponse) error {
	return func(r *http.Request, request *rpc.GuestRequest, response *rpc.GuestResponse) error {
		if request.Guest == nil || request.Guest.Id == "" {
			return syscall.EINVAL
		}

		domain, err := lv.LookupDomainByName(request.Guest.Id)
		if err != nil {
			return err
		}
		defer domain.Free()

		state, err := GetState(domain)
		if err != nil {
			return err
		}

		err = fn(domain, state)
		if err != nil {
			return err
		}

		*response = rpc.GuestResponse{
			Guest: request.Guest,
		}

		state, err = GetState(domain)
		if err != nil {
			return err
		}

		response.Guest.State = StateNames[state]

		return nil
	}
}

// Guest Actions

func (lv *Libvirt) Restart(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestResponse) error {
	return lv.Reboot(http, request, response)
}

func (lv *Libvirt) Poweroff(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestResponse) error {
	return lv.Shutdown(http, request, response)
}

func (lv *Libvirt) Delete(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestResponse) error {
	log.Info("Libvirt.Delete %s\n", request.Guest.Id)

	domain, err := lv.LookupDomainByName(request.Guest.Id)
	defer domain.Free()
	if err != nil {
		return err
	}

	err = domain.Destroy()
	if err != nil {
		return err
	}

	*response = rpc.GuestResponse{
		Guest: request.Guest,
	}

	return nil
}

func (lv *Libvirt) Create(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestResponse) error {
	log.Info("Libvirt.Create %s\n", request.Guest.Id)

	domain, err := lv.NewDomain(request.Guest)
	if err != nil {
		return err
	}
	defer domain.Free()

	err = domain.Create()
	if err != nil {
		return err
	}

	*response = rpc.GuestResponse{
		Guest: request.Guest,
	}

	state, err := GetState(domain)
	if err != nil {
		return err
	}

	response.Guest.State = StateNames[state]

	return nil
}

func (lv *Libvirt) Run(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestResponse) error {
	log.Info("Libvirt.Run %s\n", request.Guest.Id)

	return lv.DomainWrapper(func(domain *libvirt.VirDomain, state int) error {
		switch state {

		case libvirt.VIR_DOMAIN_RUNNING:
			// nothing to do

		case libvirt.VIR_DOMAIN_SHUTDOWN, libvirt.VIR_DOMAIN_SHUTOFF, libvirt.VIR_DOMAIN_BLOCKED, libvirt.VIR_DOMAIN_NOSTATE:
			return domain.Create()

		case libvirt.VIR_DOMAIN_PAUSED, libvirt.VIR_DOMAIN_PMSUSPENDED:
			return domain.Resume()
		}

		return nil
	})(http, request, response)
}

func (lv *Libvirt) Reboot(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestResponse) error {
	log.Info("Libvirt.Reboot %s\n", request.Guest.Id)

	return lv.DomainWrapper(func(domain *libvirt.VirDomain, state int) error {
		return domain.Reboot(0)
	})(http, request, response)
}

func (lv *Libvirt) Shutdown(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestResponse) error {
	log.Info("Libvirt.Shutdown %s\n", request.Guest.Id)

	return lv.DomainWrapper(func(domain *libvirt.VirDomain, state int) error {

		switch state {
		case libvirt.VIR_DOMAIN_SHUTDOWN, libvirt.VIR_DOMAIN_SHUTOFF:
			// nothing to do

		default:
			return domain.Shutdown()
		}

		return nil
	})(http, request, response)
}

func (lv *Libvirt) Status(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestResponse) error {
	log.Info("Libvirt.Status %s\n", request.Guest.Id)

	return lv.DomainWrapper(func(domain *libvirt.VirDomain, state int) error {
		// DomainWrapper gets the state already, no need to do anything here
		return nil
	})(http, request, response)
}

func (lv *Libvirt) CpuMetrics(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestMetricsResponse) error {
	var params libvirt.VirTypedParameters
	metrics := make([]*client.GuestCpuMetrics, request.Guest.Cpu)

	domain, err := lv.LookupDomainByName(request.Guest.Id)
	if err != nil {
		return err
	}
	defer domain.Free()

	nparams, err := domain.GetCPUStats(nil, 0, 0, uint32(request.Guest.Cpu), 0)
	if err != nil {
		return err
	}

	_, err = domain.GetCPUStats(&params, nparams, 0, uint32(request.Guest.Cpu), 0)
	if err != nil {
		return err
	}

	*response = rpc.GuestMetricsResponse{
		CPU: metrics,
	}
	return nil
}

func (lv *Libvirt) DiskMetrics(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestMetricsResponse) error {
	metrics := make(map[string]*client.GuestDiskMetrics, len(request.Guest.Disks))
	var params libvirt.VirTypedParameters

	domain, err := lv.LookupDomainByName(request.Guest.Id)
	if err != nil {
		return err
	}
	defer domain.Free()

	_, err = domain.BlockStatsFlags("", &params, 0, 0)
	if err != nil {
		return err
	}

	*response = rpc.GuestMetricsResponse{
		Disk: metrics,
	}
	return nil
}

func (lv *Libvirt) NicMetrics(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestMetricsResponse) error {
	metrics := make(map[string]*client.GuestNicMetrics, len(request.Guest.Nics))

	*response = rpc.GuestMetricsResponse{
		Nic: metrics,
	}
	return nil
}
<commit_msg>MIST-138: don't free here<commit_after>package libvirt

import (
	"github.com/alexzorin/libvirt-go"
	"github.com/mistifyio/mistify-agent/client"
	"github.com/mistifyio/mistify-agent/log"
	"github.com/mistifyio/mistify-agent/rpc"
	"net/http"
	"syscall"
)

var StateNames = map[int]string{
	libvirt.VIR_DOMAIN_NOSTATE:     "No State",
	libvirt.VIR_DOMAIN_RUNNING:     "Running",
	libvirt.VIR_DOMAIN_BLOCKED:     "Blocked",
	libvirt.VIR_DOMAIN_PAUSED:      "Paused",
	libvirt.VIR_DOMAIN_SHUTDOWN:    "Shutdown",
	libvirt.VIR_DOMAIN_CRASHED:     "Crashed",
	libvirt.VIR_DOMAIN_PMSUSPENDED: "Suspended",
	libvirt.VIR_DOMAIN_SHUTOFF:     "Shutoff",
}

type (
	Libvirt struct {
		uri         string
		connections chan *libvirt.VirConnection
		max         int
	}

	Domain struct {
		*libvirt.VirDomain
		State int
	}
)

func NewLibvirt(uri string, max int) (*Libvirt, error) {
	lv := &Libvirt{
		uri:         uri,
		max:         max,
		connections: make(chan *libvirt.VirConnection, max),
	}

	for i := 0; i < max; i++ {
		v, err := libvirt.NewVirConnection(uri)
		if err != nil {
			return nil, err
		}
		lv.connections <- &v
	}

	return lv, nil
}

func (lv *Libvirt) RunHTTP(port uint) error {
	server, err := rpc.NewServer(int(port))
	if err != nil {
		return err
	}

	server.RegisterService(lv)
	return server.ListenAndServe()
}

func (lv *Libvirt) getConnection() *libvirt.VirConnection {
	conn := <-lv.connections
	defer func() {
		lv.connections <- conn
	}()

	return conn
}

func (lv *Libvirt) LookupDomainByName(name string) (*libvirt.VirDomain, error) {
	conn := lv.getConnection()

	domain, err := conn.LookupDomainByName(name)
	if err != nil {
		return nil, err
	}

	return &domain, err
}

func (lv *Libvirt) NewDomain(guest *client.Guest) (*libvirt.VirDomain, error) {
	conn := lv.getConnection()

	xml, err := lv.DomainXML(guest)
	if err != nil {
		return nil, err
	}

	domain, err := conn.DomainDefineXML(xml)
	if err != nil {
		return nil, err
	}

	return &domain, err
}

func GetState(domain *libvirt.VirDomain) (int, error) {
	state, err := domain.GetState()
	if err != nil {
		return libvirt.VIR_DOMAIN_NOSTATE, err
	}

	return state[0], nil
}

func (lv *Libvirt) DomainWrapper(fn func(*libvirt.VirDomain, int) error) func(*http.Request, *rpc.GuestRequest, *rpc.GuestResponse) error {
	return func(r *http.Request, request *rpc.GuestRequest, response *rpc.GuestResponse) error {
		if request.Guest == nil || request.Guest.Id == "" {
			return syscall.EINVAL
		}

		domain, err := lv.LookupDomainByName(request.Guest.Id)
		if err != nil {
			return err
		}
		defer domain.Free()

		state, err := GetState(domain)
		if err != nil {
			return err
		}

		err = fn(domain, state)
		if err != nil {
			return err
		}

		*response = rpc.GuestResponse{
			Guest: request.Guest,
		}

		state, err = GetState(domain)
		if err != nil {
			return err
		}

		response.Guest.State = StateNames[state]

		return nil
	}
}

// Guest Actions

func (lv *Libvirt) Restart(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestResponse) error {
	return lv.Reboot(http, request, response)
}

func (lv *Libvirt) Poweroff(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestResponse) error {
	return lv.Shutdown(http, request, response)
}

func (lv *Libvirt) Delete(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestResponse) error {
	log.Info("Libvirt.Delete %s\n", request.Guest.Id)

	domain, err := lv.LookupDomainByName(request.Guest.Id)
	defer domain.Free()
	if err != nil {
		return err
	}

	err = domain.Destroy()
	if err != nil {
		return err
	}

	*response = rpc.GuestResponse{
		Guest: request.Guest,
	}

	return nil
}

func (lv *Libvirt) Create(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestResponse) error {
	log.Info("Libvirt.Create %s\n", request.Guest.Id)

	domain, err := lv.NewDomain(request.Guest)
	if err != nil {
		return err
	}
	defer domain.Free()

	err = domain.Create()
	if err != nil {
		return err
	}

	*response = rpc.GuestResponse{
		Guest: request.Guest,
	}

	state, err := GetState(domain)
	if err != nil {
		return err
	}

	response.Guest.State = StateNames[state]

	return nil
}

func (lv *Libvirt) Run(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestResponse) error {
	log.Info("Libvirt.Run %s\n", request.Guest.Id)

	return lv.DomainWrapper(func(domain *libvirt.VirDomain, state int) error {
		switch state {

		case libvirt.VIR_DOMAIN_RUNNING:
			// nothing to do

		case libvirt.VIR_DOMAIN_SHUTDOWN, libvirt.VIR_DOMAIN_SHUTOFF, libvirt.VIR_DOMAIN_BLOCKED, libvirt.VIR_DOMAIN_NOSTATE:
			return domain.Create()

		case libvirt.VIR_DOMAIN_PAUSED, libvirt.VIR_DOMAIN_PMSUSPENDED:
			return domain.Resume()
		}

		return nil
	})(http, request, response)
}

func (lv *Libvirt) Reboot(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestResponse) error {
	log.Info("Libvirt.Reboot %s\n", request.Guest.Id)

	return lv.DomainWrapper(func(domain *libvirt.VirDomain, state int) error {
		return domain.Reboot(0)
	})(http, request, response)
}

func (lv *Libvirt) Shutdown(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestResponse) error {
	log.Info("Libvirt.Shutdown %s\n", request.Guest.Id)

	return lv.DomainWrapper(func(domain *libvirt.VirDomain, state int) error {

		switch state {
		case libvirt.VIR_DOMAIN_SHUTDOWN, libvirt.VIR_DOMAIN_SHUTOFF:
			// nothing to do

		default:
			return domain.Shutdown()
		}

		return nil
	})(http, request, response)
}

func (lv *Libvirt) Status(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestResponse) error {
	log.Info("Libvirt.Status %s\n", request.Guest.Id)

	return lv.DomainWrapper(func(domain *libvirt.VirDomain, state int) error {
		// DomainWrapper gets the state already, no need to do anything here
		return nil
	})(http, request, response)
}

func (lv *Libvirt) CpuMetrics(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestMetricsResponse) error {
	var params libvirt.VirTypedParameters
	metrics := make([]*client.GuestCpuMetrics, request.Guest.Cpu)

	domain, err := lv.LookupDomainByName(request.Guest.Id)
	if err != nil {
		return err
	}
	defer domain.Free()

	nparams, err := domain.GetCPUStats(nil, 0, 0, uint32(request.Guest.Cpu), 0)
	if err != nil {
		return err
	}

	_, err = domain.GetCPUStats(&params, nparams, 0, uint32(request.Guest.Cpu), 0)
	if err != nil {
		return err
	}

	*response = rpc.GuestMetricsResponse{
		CPU: metrics,
	}
	return nil
}

func (lv *Libvirt) DiskMetrics(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestMetricsResponse) error {
	metrics := make(map[string]*client.GuestDiskMetrics, len(request.Guest.Disks))
	var params libvirt.VirTypedParameters

	domain, err := lv.LookupDomainByName(request.Guest.Id)
	if err != nil {
		return err
	}
	defer domain.Free()

	_, err = domain.BlockStatsFlags("", &params, 0, 0)
	if err != nil {
		return err
	}

	*response = rpc.GuestMetricsResponse{
		Disk: metrics,
	}
	return nil
}

func (lv *Libvirt) NicMetrics(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestMetricsResponse) error {
	metrics := make(map[string]*client.GuestNicMetrics, len(request.Guest.Nics))

	*response = rpc.GuestMetricsResponse{
		Nic: metrics,
	}
	return nil
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.
package mock

import (
	"reflect"
	"testing"

	_ "github.com/couchbaselabs/bleve/analysis/analyzers/standard_analyzer"
	"github.com/couchbaselabs/bleve/document"
	"github.com/couchbaselabs/bleve/index"
)

func TestCRUD(t *testing.T) {
	i := NewMockIndex()

	// create doc, assert doc count goes up
	doc1 := document.NewDocument("1")
	doc1.AddField(document.NewTextField("name", []byte("marty")))
	i.Update(doc1)
	count := i.DocCount()
	if count != 1 {
		t.Errorf("expected document count to be 1, was: %d", count)
	}

	// add another doc, assert doc count goes up again
	doc2 := document.NewDocument("2")
	doc2.AddField(document.NewTextField("name", []byte("bob")))
	i.Update(doc2)
	count = i.DocCount()
	if count != 2 {
		t.Errorf("expected document count to be 2, was: %d", count)
	}

	// search for doc with term that should exist
	expectedMatch := &index.TermFieldDoc{
		ID:   "1",
		Freq: 1,
		Norm: 1,
	}
	tfr, err := i.TermFieldReader([]byte("marty"), "name")
	if err != nil {
		t.Errorf("unexpected error: %v", err)
	}
	match, err := tfr.Next()
	if err != nil {
		t.Errorf("unexpected error: %v", err)
	}
	if !reflect.DeepEqual(expectedMatch, match) {
		t.Errorf("got %v, expected %v", match, expectedMatch)
	}
	nomatch, err := tfr.Next()
	if err != nil {
		t.Errorf("unexpected error: %v", err)
	}
	if nomatch != nil {
		t.Errorf("expected nil after last match")
	}

	// update doc, assert doc count doesn't go up
	doc1 = document.NewDocument("1")
	doc1.AddField(document.NewTextField("name", []byte("salad")))
	doc1.AddField(document.NewTextFieldWithIndexingOptions("desc", []byte("eat more rice"), document.INDEX_FIELD|document.INCLUDE_TERM_VECTORS))
	i.Update(doc1)
	count = i.DocCount()
	if count != 2 {
		t.Errorf("expected document count to be 2, was: %d", count)
	}

	// perform the original search again, should NOT find anything this time
	tfr, err = i.TermFieldReader([]byte("marty"), "name")
	if err != nil {
		t.Errorf("unexpected error: %v", err)
	}
	nomatch, err = tfr.Next()
	if err != nil {
		t.Errorf("unexpected error: %v", err)
	}
	if nomatch != nil {
		t.Errorf("expected no matches, found one")
		t.Logf("%v", i)
	}

	// delete a doc, ensure the count is 1
	err = i.Delete("2")
	if err != nil {
		t.Errorf("unexpected error: %v", err)
	}
	count = i.DocCount()
	if count != 1 {
		t.Errorf("expected document count to be 1, was: %d", count)
	}

	expectedMatch = &index.TermFieldDoc{
		ID:   "1",
		Freq: 1,
		Norm: 0.5773502691896258,
		Vectors: []*index.TermFieldVector{
			&index.TermFieldVector{
				Field: "desc",
				Pos:   3,
				Start: 9,
				End:   13,
			},
		},
	}
	tfr, err = i.TermFieldReader([]byte("rice"), "desc")
	if err != nil {
		t.Errorf("unexpected error: %v", err)
	}
	match, err = tfr.Next()
	if err != nil {
		t.Errorf("unexpected error: %v", err)
	}
	if !reflect.DeepEqual(expectedMatch, match) {
		t.Errorf("got %#v, expected %#v", match, expectedMatch)
	}
}
<commit_msg>improve coverage of the mock package<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.
package mock

import (
	"reflect"
	"testing"

	_ "github.com/couchbaselabs/bleve/analysis/analyzers/standard_analyzer"
	"github.com/couchbaselabs/bleve/document"
	"github.com/couchbaselabs/bleve/index"
)

func TestCRUD(t *testing.T) {
	// create a document to seed this mock index with
	doc1 := document.NewDocument("1")
	doc1.AddField(document.NewTextField("name", []byte("marty")))

	i := NewMockIndexWithDocs([]*document.Document{
		doc1,
	})

	// open it
	err := i.Open()
	if err != nil {
		t.Fatal(err)
	}

	// assert doc count is 1
	count := i.DocCount()
	if count != 1 {
		t.Errorf("expected document count to be 1, was: %d", count)
	}

	// add another doc, assert doc count goes up again
	doc2 := document.NewDocument("2")
	doc2.AddField(document.NewTextField("name", []byte("bob")))
	i.Update(doc2)
	count = i.DocCount()
	if count != 2 {
		t.Errorf("expected document count to be 2, was: %d", count)
	}

	// search for doc with term that should exist
	expectedMatch := &index.TermFieldDoc{
		ID:   "1",
		Freq: 1,
		Norm: 1,
	}
	tfr, err := i.TermFieldReader([]byte("marty"), "name")
	if err != nil {
		t.Errorf("unexpected error: %v", err)
	}
	match, err := tfr.Next()
	if err != nil {
		t.Errorf("unexpected error: %v", err)
	}
	if !reflect.DeepEqual(expectedMatch, match) {
		t.Errorf("got %v, expected %v", match, expectedMatch)
	}
	nomatch, err := tfr.Next()
	if err != nil {
		t.Errorf("unexpected error: %v", err)
	}
	if nomatch != nil {
		t.Errorf("expected nil after last match")
	}
	tfr.Close()

	// update doc, assert doc count doesn't go up
	doc1 = document.NewDocument("1")
	doc1.AddField(document.NewTextField("name", []byte("salad")))
	doc1.AddField(document.NewTextFieldWithIndexingOptions("desc", []byte("eat more rice"), document.INDEX_FIELD|document.INCLUDE_TERM_VECTORS))
	i.Update(doc1)
	count = i.DocCount()
	if count != 2 {
		t.Errorf("expected document count to be 2, was: %d", count)
	}

	// perform the original search again, should NOT find anything this time
	tfr, err = i.TermFieldReader([]byte("marty"), "name")
	if err != nil {
		t.Errorf("unexpected error: %v", err)
	}
	nomatch, err = tfr.Next()
	if err != nil {
		t.Errorf("unexpected error: %v", err)
	}
	if nomatch != nil {
		t.Errorf("expected no matches, found one")
		t.Logf("%v", i)
	}
	tfr.Close()

	// delete a doc, ensure the count is 1
	err = i.Delete("2")
	if err != nil {
		t.Errorf("unexpected error: %v", err)
	}
	count = i.DocCount()
	if count != 1 {
		t.Errorf("expected document count to be 1, was: %d", count)
	}

	expectedMatch = &index.TermFieldDoc{
		ID:   "1",
		Freq: 1,
		Norm: 0.5773502691896258,
		Vectors: []*index.TermFieldVector{
			&index.TermFieldVector{
				Field: "desc",
				Pos:   3,
				Start: 9,
				End:   13,
			},
		},
	}
	tfr, err = i.TermFieldReader([]byte("rice"), "desc")
	if err != nil {
		t.Errorf("unexpected error: %v", err)
	}
	match, err = tfr.Next()
	if err != nil {
		t.Errorf("unexpected error: %v", err)
	}
	if !reflect.DeepEqual(expectedMatch, match) {
		t.Errorf("got %#v, expected %#v", match, expectedMatch)
	}
	tfr.Close()

	// now test usage of advance
	// add another doc,
	doc5 := document.NewDocument("5")
	doc5.AddField(document.NewTextField("name", []byte("salad")))
	i.Update(doc5)
	tfr, err = i.TermFieldReader([]byte("salad"), "name")
	if err != nil {
		t.Errorf("Error accessing term field reader: %v", err)
	}

	readerCount := tfr.Count()
	if readerCount != 2 {
		t.Errorf("expected 2 docs in reader, got %d", readerCount)
	}

	match, err = tfr.Advance("1")
	if err != nil {
		t.Errorf("unexpected error: %v", err)
	}
	if match.ID != "1" {
		t.Errorf("Expected ID '1', got '%s'", match.ID)
	}
	match, err = tfr.Advance("7")
	if err != nil {
		t.Errorf("unexpected error: %v", err)
	}
	if match != nil {
		t.Errorf("expected nil, got %v", match)
	}
	// try to do it again
	match, err = tfr.Advance("7")
	if err != nil {
		t.Errorf("unexpected error: %v", err)
	}
	if match != nil {
		t.Errorf("expected nil, got %v", match)
	}
	tfr.Close()

	// close it
	i.Close()
}
<|endoftext|>


<commit_before>// Copyright 2018 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package libhttpserver

import (
	"net/http"
	"strings"
)

type contentTypeOverridingResponseWriter struct {
	original http.ResponseWriter
}

var _ http.ResponseWriter = (*contentTypeOverridingResponseWriter)(nil)

func newContentTypeOverridingResponseWriter(
	original http.ResponseWriter) *contentTypeOverridingResponseWriter {
	return &contentTypeOverridingResponseWriter{
		original: original,
	}
}

func (w *contentTypeOverridingResponseWriter) overrideMimeType(
	mimeType string) (newMimeType string) {
	// Send text/plain for all HTML and JS files to avoid them being executed
	// by the frontend WebView.
	ty := strings.ToLower(mimeType)
	switch {
	// First reject anything containing javascript/xml/html.
	case strings.Contains(ty, "javascript") ||
		strings.Contains(ty, "xml") ||
		strings.Contains(ty, "html"):
		return "text/plain"
	// Pass multimedia types through, and pdf too.
	case strings.HasPrefix(ty, "audio/") ||
		strings.HasPrefix(ty, "image/") ||
		strings.HasPrefix(ty, "video/") ||
		ty == "application/pdf":
		return ty
	// Otherwise text/plain.
	default:
		return "text/plain"
	}
}

func (w *contentTypeOverridingResponseWriter) override() {
	t := w.original.Header().Get("Content-Type")
	if len(t) > 0 {
		w.original.Header().Set("Content-Type", w.overrideMimeType(t))
	}
	w.original.Header().Set("X-Content-Type-Options", "nosniff")
}

func (w *contentTypeOverridingResponseWriter) Header() http.Header {
	return w.original.Header()
}

func (w *contentTypeOverridingResponseWriter) WriteHeader(statusCode int) {
	w.override()
	w.original.WriteHeader(statusCode)
}

func (w *contentTypeOverridingResponseWriter) Write(data []byte) (int, error) {
	w.override()
	return w.original.Write(data)
}

var additionalMimeTypes = map[string]string{
	".go":    "text/plain",
	".py":    "text/plain",
	".zsh":   "text/plain",
	".fish":  "text/plain",
	".cs":    "text/plain",
	".rb":    "text/plain",
	".m":     "text/plain",
	".mm":    "text/plain",
	".swift": "text/plain",
	".flow":  "text/plain",
	".php":   "text/plain",
	".pl":    "text/plain",
	".sh":    "text/plain",
	".js":    "text/plain",
	".json":  "text/plain",
	".sql":   "text/plain",
	".rs":    "text/plain",
	".xml":   "text/plain",
	".tex":   "text/plain",
	".pub":   "text/plain",
}
<commit_msg>Default to application/octet-stream for unknown types<commit_after>// Copyright 2018 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package libhttpserver

import (
	"net/http"
	"strings"
)

type contentTypeOverridingResponseWriter struct {
	original http.ResponseWriter
}

var _ http.ResponseWriter = (*contentTypeOverridingResponseWriter)(nil)

func newContentTypeOverridingResponseWriter(
	original http.ResponseWriter) *contentTypeOverridingResponseWriter {
	return &contentTypeOverridingResponseWriter{
		original: original,
	}
}

func (w *contentTypeOverridingResponseWriter) overrideMimeType(
	mimeType string) (newMimeType string) {
	// Send text/plain for all HTML and JS files to avoid them being executed
	// by the frontend WebView.
	ty := strings.ToLower(mimeType)
	switch {
	// First reject anything containing javascript/xml/html.
	case strings.Contains(ty, "javascript") ||
		strings.Contains(ty, "xml") ||
		strings.Contains(ty, "html"):
		return "text/plain"
	// Pass multimedia types through, and pdf too.
	case strings.HasPrefix(ty, "audio/") ||
		strings.HasPrefix(ty, "image/") ||
		strings.HasPrefix(ty, "video/") ||
		ty == "application/pdf" ||
		ty == "text/plain":
		return ty
	// Otherwise text/plain.
	default:
		return "application/octet-stream"
	}
}

func (w *contentTypeOverridingResponseWriter) override() {
	t := w.original.Header().Get("Content-Type")
	if len(t) > 0 {
		w.original.Header().Set("Content-Type", w.overrideMimeType(t))
	}
	w.original.Header().Set("X-Content-Type-Options", "nosniff")
}

func (w *contentTypeOverridingResponseWriter) Header() http.Header {
	return w.original.Header()
}

func (w *contentTypeOverridingResponseWriter) WriteHeader(statusCode int) {
	w.override()
	w.original.WriteHeader(statusCode)
}

func (w *contentTypeOverridingResponseWriter) Write(data []byte) (int, error) {
	w.override()
	return w.original.Write(data)
}

var additionalMimeTypes = map[string]string{
	".go":    "text/plain",
	".py":    "text/plain",
	".zsh":   "text/plain",
	".fish":  "text/plain",
	".cs":    "text/plain",
	".rb":    "text/plain",
	".m":     "text/plain",
	".mm":    "text/plain",
	".swift": "text/plain",
	".flow":  "text/plain",
	".php":   "text/plain",
	".pl":    "text/plain",
	".sh":    "text/plain",
	".js":    "text/plain",
	".json":  "text/plain",
	".sql":   "text/plain",
	".rs":    "text/plain",
	".xml":   "text/plain",
	".tex":   "text/plain",
	".pub":   "text/plain",
}
<|endoftext|>