<commit_before>package weed_server

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/filer2"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
	"os"
)

var (
	OS_UID = uint32(os.Getuid())
	OS_GID = uint32(os.Getgid())
)

type FilerPostResult struct {
	Name  string `json:"name,omitempty"`
	Size  uint32 `json:"size,omitempty"`
	Error string `json:"error,omitempty"`
	Fid   string `json:"fid,omitempty"`
	Url   string `json:"url,omitempty"`
}

func (fs *FilerServer) queryFileInfoByPath(w http.ResponseWriter, r *http.Request, path string) (fileId, urlLocation string, err error) {
	var entry *filer2.Entry
	entry, err = fs.filer.FindEntry(filer2.FullPath(path))
	if err == filer2.ErrNotFound {
		return "", "", nil
	}

	if err != nil {
		glog.V(0).Infoln("failing to find path in filer store", path, err.Error())
		writeJsonError(w, r, http.StatusInternalServerError, err)
		return
	}

	if len(entry.Chunks) == 0 {
		glog.V(1).Infof("empty entry: %s", path)
		w.WriteHeader(http.StatusNoContent)
	} else {
		fileId = entry.Chunks[0].FileId
		urlLocation, err = operation.LookupFileId(fs.filer.GetMaster(), fileId)
		if err != nil {
			glog.V(1).Infof("operation LookupFileId %s failed, err is %s", fileId, err.Error())
			w.WriteHeader(http.StatusNotFound)
		}
	}
	return
}

func (fs *FilerServer) assignNewFileInfo(w http.ResponseWriter, r *http.Request, replication, collection string, dataCenter string) (fileId, urlLocation string, err error) {
	ar := &operation.VolumeAssignRequest{
		Count:       1,
		Replication: replication,
		Collection:  collection,
		Ttl:         r.URL.Query().Get("ttl"),
		DataCenter:  dataCenter,
	}
	var altRequest *operation.VolumeAssignRequest
	if dataCenter != "" {
		altRequest = &operation.VolumeAssignRequest{
			Count:       1,
			Replication: replication,
			Collection:  collection,
			Ttl:         r.URL.Query().Get("ttl"),
			DataCenter:  "",
		}
	}

	assignResult, ae := operation.Assign(fs.filer.GetMaster(), ar, altRequest)
	if ae != nil {
		glog.Errorf("failing to assign a file id: %v", ae)
		writeJsonError(w, r, http.StatusInternalServerError, ae)
		err = ae
		return
	}
	fileId = assignResult.Fid
	urlLocation = "http://" + assignResult.Url + "/" + assignResult.Fid
	return
}

func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {

	query := r.URL.Query()
	replication := query.Get("replication")
	if replication == "" {
		replication = fs.option.DefaultReplication
	}
	collection := query.Get("collection")
	if collection == "" {
		collection = fs.option.Collection
	}
	dataCenter := query.Get("dataCenter")
	if dataCenter == "" {
		dataCenter = fs.option.DataCenter
	}

	if autoChunked := fs.autoChunk(w, r, replication, collection, dataCenter); autoChunked {
		return
	}

	fileId, urlLocation, err := fs.queryFileInfoByPath(w, r, r.URL.Path)
	if err == nil && fileId == "" {
		fileId, urlLocation, err = fs.assignNewFileInfo(w, r, replication, collection, dataCenter)
	}
	if err != nil || fileId == "" || urlLocation == "" {
		return
	}

	glog.V(0).Infof("request header %+v, urlLocation: %v", r.Header, urlLocation)

	u, _ := url.Parse(urlLocation)

	// This allows a client to generate a chunk manifest and submit it to the filer -- it is a little off
	// because they need to provide FIDs instead of file paths...
	cm, _ := strconv.ParseBool(query.Get("cm"))
	if cm {
		q := u.Query()
		q.Set("cm", "true")
		u.RawQuery = q.Encode()
	}
	glog.V(4).Infoln("post to", u)

	// send request to volume server
	request := &http.Request{
		Method:        r.Method,
		URL:           u,
		Proto:         r.Proto,
		ProtoMajor:    r.ProtoMajor,
		ProtoMinor:    r.ProtoMinor,
		Header:        r.Header,
		Body:          r.Body,
		Host:          r.Host,
		ContentLength: r.ContentLength,
	}
	resp, do_err := util.Do(request)
	if do_err != nil {
		glog.Errorf("failing to connect to volume server %s: %v, %+v", r.RequestURI, do_err, r.Method)
		writeJsonError(w, r, http.StatusInternalServerError, do_err)
		return
	}
	defer resp.Body.Close()
	resp_body, ra_err := ioutil.ReadAll(resp.Body)
	if ra_err != nil {
		glog.V(0).Infoln("failing to upload to volume server", r.RequestURI, ra_err.Error())
		writeJsonError(w, r, http.StatusInternalServerError, ra_err)
		return
	}
	glog.V(4).Infoln("post result", string(resp_body))
	var ret operation.UploadResult
	unmarshal_err := json.Unmarshal(resp_body, &ret)
	if unmarshal_err != nil {
		glog.V(0).Infoln("failing to read upload resonse", r.RequestURI, string(resp_body))
		writeJsonError(w, r, http.StatusInternalServerError, unmarshal_err)
		return
	}
	if ret.Error != "" {
		glog.V(0).Infoln("failing to post to volume server", r.RequestURI, ret.Error)
		writeJsonError(w, r, http.StatusInternalServerError, errors.New(ret.Error))
		return
	}

	// find correct final path
	path := r.URL.Path
	if strings.HasSuffix(path, "/") {
		if ret.Name != "" {
			path += ret.Name
		} else {
			operation.DeleteFile(fs.filer.GetMaster(), fileId, fs.jwt(fileId)) //clean up
			glog.V(0).Infoln("Can not to write to folder", path, "without a file name!")
			writeJsonError(w, r, http.StatusInternalServerError,
				errors.New("Can not to write to folder "+path+" without a file name"))
			return
		}
	}

	// update metadata in filer store
	glog.V(4).Infoln("saving", path, "=>", fileId)
	entry := &filer2.Entry{
		FullPath: filer2.FullPath(path),
		Attr: filer2.Attr{
			Mtime:       time.Now(),
			Crtime:      time.Now(),
			Mode:        0660,
			Uid:         OS_UID,
			Gid:         OS_GID,
			Replication: replication,
			Collection:  collection,
			TtlSec:      int32(util.ParseInt(r.URL.Query().Get("ttl"), 0)),
		},
		Chunks: []*filer_pb.FileChunk{{
			FileId: fileId,
			Size:   uint64(ret.Size),
			Mtime:  time.Now().UnixNano(),
		}},
	}
	if db_err := fs.filer.CreateEntry(entry); db_err != nil {
		operation.DeleteFile(fs.filer.GetMaster(), fileId, fs.jwt(fileId)) //clean up
		glog.V(0).Infof("failing to write %s to filer server : %v", path, db_err)
		writeJsonError(w, r, http.StatusInternalServerError, db_err)
		return
	}

	// send back post result
	reply := FilerPostResult{
		Name:  ret.Name,
		Size:  ret.Size,
		Error: ret.Error,
		Fid:   fileId,
		Url:   urlLocation,
	}
	writeJsonQuiet(w, r, http.StatusCreated, reply)
}

// curl -X DELETE http://localhost:8888/path/to
func (fs *FilerServer) DeleteHandler(w http.ResponseWriter, r *http.Request) {

	err := fs.filer.DeleteEntryMetaAndData(filer2.FullPath(r.URL.Path), false, true)
	if err != nil {
		glog.V(1).Infoln("deleting", r.URL.Path, ":", err.Error())
		writeJsonError(w, r, http.StatusInternalServerError, err)
		return
	}

	w.WriteHeader(http.StatusNoContent)
}
<commit_msg>add back recursively delete a folder<commit_after>package weed_server

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/filer2"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
	"os"
)

var (
	OS_UID = uint32(os.Getuid())
	OS_GID = uint32(os.Getgid())
)

type FilerPostResult struct {
	Name  string `json:"name,omitempty"`
	Size  uint32 `json:"size,omitempty"`
	Error string `json:"error,omitempty"`
	Fid   string `json:"fid,omitempty"`
	Url   string `json:"url,omitempty"`
}

func (fs *FilerServer) queryFileInfoByPath(w http.ResponseWriter, r *http.Request, path string) (fileId, urlLocation string, err error) {
	var entry *filer2.Entry
	entry, err = fs.filer.FindEntry(filer2.FullPath(path))
	if err == filer2.ErrNotFound {
		return "", "", nil
	}

	if err != nil {
		glog.V(0).Infoln("failing to find path in filer store", path, err.Error())
		writeJsonError(w, r, http.StatusInternalServerError, err)
		return
	}

	if len(entry.Chunks) == 0 {
		glog.V(1).Infof("empty entry: %s", path)
		w.WriteHeader(http.StatusNoContent)
	} else {
		fileId = entry.Chunks[0].FileId
		urlLocation, err = operation.LookupFileId(fs.filer.GetMaster(), fileId)
		if err != nil {
			glog.V(1).Infof("operation LookupFileId %s failed, err is %s", fileId, err.Error())
			w.WriteHeader(http.StatusNotFound)
		}
	}
	return
}

func (fs *FilerServer) assignNewFileInfo(w http.ResponseWriter, r *http.Request, replication, collection string, dataCenter string) (fileId, urlLocation string, err error) {
	ar := &operation.VolumeAssignRequest{
		Count:       1,
		Replication: replication,
		Collection:  collection,
		Ttl:         r.URL.Query().Get("ttl"),
		DataCenter:  dataCenter,
	}
	var altRequest *operation.VolumeAssignRequest
	if dataCenter != "" {
		altRequest = &operation.VolumeAssignRequest{
			Count:       1,
			Replication: replication,
			Collection:  collection,
			Ttl:         r.URL.Query().Get("ttl"),
			DataCenter:  "",
		}
	}

	assignResult, ae := operation.Assign(fs.filer.GetMaster(), ar, altRequest)
	if ae != nil {
		glog.Errorf("failing to assign a file id: %v", ae)
		writeJsonError(w, r, http.StatusInternalServerError, ae)
		err = ae
		return
	}
	fileId = assignResult.Fid
	urlLocation = "http://" + assignResult.Url + "/" + assignResult.Fid
	return
}

func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {

	query := r.URL.Query()
	replication := query.Get("replication")
	if replication == "" {
		replication = fs.option.DefaultReplication
	}
	collection := query.Get("collection")
	if collection == "" {
		collection = fs.option.Collection
	}
	dataCenter := query.Get("dataCenter")
	if dataCenter == "" {
		dataCenter = fs.option.DataCenter
	}

	if autoChunked := fs.autoChunk(w, r, replication, collection, dataCenter); autoChunked {
		return
	}

	fileId, urlLocation, err := fs.queryFileInfoByPath(w, r, r.URL.Path)
	if err == nil && fileId == "" {
		fileId, urlLocation, err = fs.assignNewFileInfo(w, r, replication, collection, dataCenter)
	}
	if err != nil || fileId == "" || urlLocation == "" {
		return
	}

	glog.V(0).Infof("request header %+v, urlLocation: %v", r.Header, urlLocation)

	u, _ := url.Parse(urlLocation)

	// This allows a client to generate a chunk manifest and submit it to the filer -- it is a little off
	// because they need to provide FIDs instead of file paths...
	cm, _ := strconv.ParseBool(query.Get("cm"))
	if cm {
		q := u.Query()
		q.Set("cm", "true")
		u.RawQuery = q.Encode()
	}
	glog.V(4).Infoln("post to", u)

	// send request to volume server
	request := &http.Request{
		Method:        r.Method,
		URL:           u,
		Proto:         r.Proto,
		ProtoMajor:    r.ProtoMajor,
		ProtoMinor:    r.ProtoMinor,
		Header:        r.Header,
		Body:          r.Body,
		Host:          r.Host,
		ContentLength: r.ContentLength,
	}
	resp, do_err := util.Do(request)
	if do_err != nil {
		glog.Errorf("failing to connect to volume server %s: %v, %+v", r.RequestURI, do_err, r.Method)
		writeJsonError(w, r, http.StatusInternalServerError, do_err)
		return
	}
	defer resp.Body.Close()
	resp_body, ra_err := ioutil.ReadAll(resp.Body)
	if ra_err != nil {
		glog.V(0).Infoln("failing to upload to volume server", r.RequestURI, ra_err.Error())
		writeJsonError(w, r, http.StatusInternalServerError, ra_err)
		return
	}
	glog.V(4).Infoln("post result", string(resp_body))
	var ret operation.UploadResult
	unmarshal_err := json.Unmarshal(resp_body, &ret)
	if unmarshal_err != nil {
		glog.V(0).Infoln("failing to read upload resonse", r.RequestURI, string(resp_body))
		writeJsonError(w, r, http.StatusInternalServerError, unmarshal_err)
		return
	}
	if ret.Error != "" {
		glog.V(0).Infoln("failing to post to volume server", r.RequestURI, ret.Error)
		writeJsonError(w, r, http.StatusInternalServerError, errors.New(ret.Error))
		return
	}

	// find correct final path
	path := r.URL.Path
	if strings.HasSuffix(path, "/") {
		if ret.Name != "" {
			path += ret.Name
		} else {
			operation.DeleteFile(fs.filer.GetMaster(), fileId, fs.jwt(fileId)) //clean up
			glog.V(0).Infoln("Can not to write to folder", path, "without a file name!")
			writeJsonError(w, r, http.StatusInternalServerError,
				errors.New("Can not to write to folder "+path+" without a file name"))
			return
		}
	}

	// update metadata in filer store
	glog.V(4).Infoln("saving", path, "=>", fileId)
	entry := &filer2.Entry{
		FullPath: filer2.FullPath(path),
		Attr: filer2.Attr{
			Mtime:       time.Now(),
			Crtime:      time.Now(),
			Mode:        0660,
			Uid:         OS_UID,
			Gid:         OS_GID,
			Replication: replication,
			Collection:  collection,
			TtlSec:      int32(util.ParseInt(r.URL.Query().Get("ttl"), 0)),
		},
		Chunks: []*filer_pb.FileChunk{{
			FileId: fileId,
			Size:   uint64(ret.Size),
			Mtime:  time.Now().UnixNano(),
		}},
	}
	if db_err := fs.filer.CreateEntry(entry); db_err != nil {
		operation.DeleteFile(fs.filer.GetMaster(), fileId, fs.jwt(fileId)) //clean up
		glog.V(0).Infof("failing to write %s to filer server : %v", path, db_err)
		writeJsonError(w, r, http.StatusInternalServerError, db_err)
		return
	}

	// send back post result
	reply := FilerPostResult{
		Name:  ret.Name,
		Size:  ret.Size,
		Error: ret.Error,
		Fid:   fileId,
		Url:   urlLocation,
	}
	writeJsonQuiet(w, r, http.StatusCreated, reply)
}

// curl -X DELETE http://localhost:8888/path/to
// curl -X DELETE http://localhost:8888/path/to?recursive=true
func (fs *FilerServer) DeleteHandler(w http.ResponseWriter, r *http.Request) {

	isRecursive := r.FormValue("recursive") == "true"

	err := fs.filer.DeleteEntryMetaAndData(filer2.FullPath(r.URL.Path), isRecursive, true)
	if err != nil {
		glog.V(1).Infoln("deleting", r.URL.Path, ":", err.Error())
		writeJsonError(w, r, http.StatusInternalServerError, err)
		return
	}

	w.WriteHeader(http.StatusNoContent)
}
<|endoftext|>


<commit_before>package template_test

import (
	"context"
	"fmt"
	"strings"
	"testing"

	"github.com/fabric8-services/fabric8-tenant/openshift"
	"github.com/fabric8-services/fabric8-tenant/template"
	"github.com/stretchr/testify/assert"
	yaml "gopkg.in/yaml.v2"
)

func TestFoundJenkins(t *testing.T) {
	c, err := template.Asset("template/fabric8-tenant-jenkins-openshift.yml")
	if err != nil {
		t.Fatalf("Asset template/fabric8-tenant-jenkins-openshift.yml not found")
	}

	cs := string(c)
	if !strings.Contains(cs, "jenkins") {
		t.Fatalf("Word jenkins not found in the template")
	}

	var template map[interface{}]interface{}
	err = yaml.Unmarshal(c, &template)
	if err != nil {
		t.Fatalf("Could not parse template as yaml")
	}

	params, ok := template["parameters"].([]interface{})
	if !ok {
		t.Fatalf("parameters not found")
	}

	assert.Equal(t, 6, len(params), "unknown number of parameters")
}

func TestFoundJenkinsQuotasOSO(t *testing.T) {
	c, err := template.Asset("template/fabric8-tenant-jenkins-quotas-oso-openshift.yml")
	if err != nil {
		t.Fatalf("Asset template/fabric8-tenant-jenkins-quotas-oso-openshift.yml not found")
	}

	cs := string(c)
	if !strings.Contains(cs, "Limit") {
		t.Fatalf("Word Limit not found in the resource")
	}

	var template map[interface{}]interface{}
	err = yaml.Unmarshal(c, &template)
	if err != nil {
		t.Fatalf("Could not parse resource as yaml")
	}
}

func TestFoundChe(t *testing.T) {
	c, err := template.Asset("template/fabric8-tenant-che-openshift.yml")
	if err != nil {
		t.Fatalf("Asset template/fabric8-tenant-che-openshift.yml not found")
	}

	cs := string(c)
	if !strings.Contains(cs, "che") {
		t.Fatalf("Word che not found in the template")
	}

	var template map[interface{}]interface{}
	err = yaml.Unmarshal(c, &template)
	if err != nil {
		t.Fatalf("Could not parse template as yaml")
	}

	params, ok := template["parameters"].([]interface{})
	if !ok {
		t.Fatalf("parameters not found")
	}

	assert.Equal(t, 10, len(params), "unknown number of parameters")
}

func TestFoundCheMultiTenant(t *testing.T) {
	c, err := template.Asset("template/fabric8-tenant-che-mt-openshift.yml")
	if err != nil {
		t.Fatalf("Asset template/fabric8-tenant-che-mt-openshift.yml not found")
	}

	cs := string(c)
	if !strings.Contains(cs, "claim-che-workspace") {
		t.Fatalf("Word claim-che-workspace not found in the template")
	}

	var template map[interface{}]interface{}
	err = yaml.Unmarshal(c, &template)
	if err != nil {
		t.Fatalf("Could not parse template as yaml")
	}

	params, ok := template["parameters"].([]interface{})
	if !ok {
		t.Fatalf("parameters not found")
	}

	assert.Equal(t, 5, len(params), "unknown number of parameters")
}

func TestFoundCheQuotasOSO(t *testing.T) {
	c, err := template.Asset("template/fabric8-tenant-che-quotas-oso-openshift.yml")
	if err != nil {
		t.Fatalf("Asset template/fabric8-tenant-che-quotas-oso-openshift.yml not found")
	}

	cs := string(c)
	if !strings.Contains(cs, "Limit") {
		t.Fatalf("Word Limit not found in the resource")
	}

	var template map[interface{}]interface{}
	err = yaml.Unmarshal(c, &template)
	if err != nil {
		t.Fatalf("Could not parse resource as yaml")
	}
}

func TestFoundTeam(t *testing.T) {
	c, err := template.Asset("template/fabric8-tenant-team-openshift.yml")
	if err != nil {
		t.Fatalf("Asset template/fabric8-tenant-team-openshift.yml not found")
	}

	cs := string(c)
	if !strings.Contains(cs, "team") {
		t.Fatalf("Word team not found in the template")
	}

	var template map[interface{}]interface{}
	err = yaml.Unmarshal(c, &template)
	if err != nil {
		t.Fatalf("Could not parse template as yaml")
	}

	params, ok := template["parameters"].([]interface{})
	if !ok {
		t.Fatalf("parameters not found")
	}
	// 1 parameter not used in Openshift templates but bleed through from k8
	assert.Equal(t, 8, len(params), "unknown number of parameters")
}

func TestStatusAPIJenkins(t *testing.T) {
	assert.NoError(t,
		contain(templates(t),
			openshift.ValKindDeploymentConfig,
			withSpecLabel("app", "jenkins"),
			withNamespaceLike("-jenkins")))
}

func TestStatusAPIChe(t *testing.T) {
	assert.NoError(t,
		contain(templates(t),
			openshift.ValKindDeploymentConfig,
			withSpecLabel("app", "che"),
			withNamespaceLike("-che")))
}

func templates(t *testing.T) []map[interface{}]interface{} {
	templs, err := openshift.LoadProcessedTemplates(context.Background(), openshift.Config{MasterUser: "master"}, "test", map[string]string{})
	assert.NoError(t, err)
	return templs
}

func contain(templtes []map[interface{}]interface{}, kind string, checks ...func(map[interface{}]interface{}) error) error {
	var err error
	for _, temp := range templtes {
		if openshift.GetKind(temp) == kind {
			err = nil
			for _, check := range checks {
				if e := check(temp); e != nil {
					err = e
				}
			}
			if err == nil {
				return nil
			}
		}
	}
	return fmt.Errorf("No template of kind %v found, cause %v", kind, err)
}

func withSpecLabel(name, value string) func(map[interface{}]interface{}) error {
	return func(temp map[interface{}]interface{}) error {
		val := openshift.GetLabel(openshift.GetTemplate(openshift.GetSpec(temp)), name)
		if val == value {
			return nil
		}
		return fmt.Errorf("No label named %v with value %v found", name, value)
	}
}

func withNamespaceLike(name string) func(map[interface{}]interface{}) error {
	return func(temp map[interface{}]interface{}) error {
		val := openshift.GetNamespace(temp)
		if strings.HasSuffix(val, name) {
			return nil
		}
		return fmt.Errorf("No namespace match for %v found", name)
	}
}
<commit_msg>Revert "Also modifies the test file"<commit_after>package template_test

import (
	"context"
	"fmt"
	"strings"
	"testing"

	"github.com/fabric8-services/fabric8-tenant/openshift"
	"github.com/fabric8-services/fabric8-tenant/template"
	"github.com/stretchr/testify/assert"
	yaml "gopkg.in/yaml.v2"
)

func TestFoundJenkins(t *testing.T) {
	c, err := template.Asset("template/fabric8-tenant-jenkins-openshift.yml")
	if err != nil {
		t.Fatalf("Asset template/fabric8-tenant-jenkins-openshift.yml not found")
	}

	cs := string(c)
	if !strings.Contains(cs, "jenkins") {
		t.Fatalf("Word jenkins not found in the template")
	}

	var template map[interface{}]interface{}
	err = yaml.Unmarshal(c, &template)
	if err != nil {
		t.Fatalf("Could not parse template as yaml")
	}

	params, ok := template["parameters"].([]interface{})
	if !ok {
		t.Fatalf("parameters not found")
	}

	assert.Equal(t, 6, len(params), "unknown number of parameters")
}

func TestFoundJenkinsQuotasOSO(t *testing.T) {
	c, err := template.Asset("template/fabric8-tenant-jenkins-quotas-oso-openshift.yml")
	if err != nil {
		t.Fatalf("Asset template/fabric8-tenant-jenkins-quotas-oso-openshift.yml not found")
	}

	cs := string(c)
	if !strings.Contains(cs, "Limit") {
		t.Fatalf("Word Limit not found in the resource")
	}

	var template map[interface{}]interface{}
	err = yaml.Unmarshal(c, &template)
	if err != nil {
		t.Fatalf("Could not parse resource as yaml")
	}
}

func TestFoundChe(t *testing.T) {
	c, err := template.Asset("template/fabric8-tenant-che-openshift.yml")
	if err != nil {
		t.Fatalf("Asset template/fabric8-tenant-che-openshift.yml not found")
	}

	cs := string(c)
	if !strings.Contains(cs, "che") {
		t.Fatalf("Word che not found in the template")
	}

	var template map[interface{}]interface{}
	err = yaml.Unmarshal(c, &template)
	if err != nil {
		t.Fatalf("Could not parse template as yaml")
	}

	params, ok := template["parameters"].([]interface{})
	if !ok {
		t.Fatalf("parameters not found")
	}

	assert.Equal(t, 10, len(params), "unknown number of parameters")
}

func TestFoundCheMultiTenant(t *testing.T) {
	c, err := template.Asset("template/fabric8-tenant-che-mt-openshift.yml")
	if err != nil {
		t.Fatalf("Asset template/fabric8-tenant-che-mt-openshift.yml not found")
	}

	cs := string(c)
	if !strings.Contains(cs, "claim-che-workspace") {
		t.Fatalf("Word claim-che-workspace not found in the template")
	}

	var template map[interface{}]interface{}
	err = yaml.Unmarshal(c, &template)
	if err != nil {
		t.Fatalf("Could not parse template as yaml")
	}

	params, ok := template["parameters"].([]interface{})
	if !ok {
		t.Fatalf("parameters not found")
	}

	assert.Equal(t, 3, len(params), "unknown number of parameters")
}

func TestFoundCheQuotasOSO(t *testing.T) {
	c, err := template.Asset("template/fabric8-tenant-che-quotas-oso-openshift.yml")
	if err != nil {
		t.Fatalf("Asset template/fabric8-tenant-che-quotas-oso-openshift.yml not found")
	}

	cs := string(c)
	if !strings.Contains(cs, "Limit") {
		t.Fatalf("Word Limit not found in the resource")
	}

	var template map[interface{}]interface{}
	err = yaml.Unmarshal(c, &template)
	if err != nil {
		t.Fatalf("Could not parse resource as yaml")
	}
}

func TestFoundTeam(t *testing.T) {
	c, err := template.Asset("template/fabric8-tenant-team-openshift.yml")
	if err != nil {
		t.Fatalf("Asset template/fabric8-tenant-team-openshift.yml not found")
	}

	cs := string(c)
	if !strings.Contains(cs, "team") {
		t.Fatalf("Word team not found in the template")
	}

	var template map[interface{}]interface{}
	err = yaml.Unmarshal(c, &template)
	if err != nil {
		t.Fatalf("Could not parse template as yaml")
	}

	params, ok := template["parameters"].([]interface{})
	if !ok {
		t.Fatalf("parameters not found")
	}
	// 1 parameter not used in Openshift templates but bleed through from k8
	assert.Equal(t, 8, len(params), "unknown number of parameters")
}

func TestStatusAPIJenkins(t *testing.T) {
	assert.NoError(t,
		contain(templates(t),
			openshift.ValKindDeploymentConfig,
			withSpecLabel("app", "jenkins"),
			withNamespaceLike("-jenkins")))
}

func TestStatusAPIChe(t *testing.T) {
	assert.NoError(t,
		contain(templates(t),
			openshift.ValKindDeploymentConfig,
			withSpecLabel("app", "che"),
			withNamespaceLike("-che")))
}

func templates(t *testing.T) []map[interface{}]interface{} {
	templs, err := openshift.LoadProcessedTemplates(context.Background(), openshift.Config{MasterUser: "master"}, "test", map[string]string{})
	assert.NoError(t, err)
	return templs
}

func contain(templtes []map[interface{}]interface{}, kind string, checks ...func(map[interface{}]interface{}) error) error {
	var err error
	for _, temp := range templtes {
		if openshift.GetKind(temp) == kind {
			err = nil
			for _, check := range checks {
				if e := check(temp); e != nil {
					err = e
				}
			}
			if err == nil {
				return nil
			}
		}
	}
	return fmt.Errorf("No template of kind %v found, cause %v", kind, err)
}

func withSpecLabel(name, value string) func(map[interface{}]interface{}) error {
	return func(temp map[interface{}]interface{}) error {
		val := openshift.GetLabel(openshift.GetTemplate(openshift.GetSpec(temp)), name)
		if val == value {
			return nil
		}
		return fmt.Errorf("No label named %v with value %v found", name, value)
	}
}

func withNamespaceLike(name string) func(map[interface{}]interface{}) error {
	return func(temp map[interface{}]interface{}) error {
		val := openshift.GetNamespace(temp)
		if strings.HasSuffix(val, name) {
			return nil
		}
		return fmt.Errorf("No namespace match for %v found", name)
	}
}
<|endoftext|>


<commit_before>package physical

import (
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/armon/go-metrics"
	"github.com/samuel/go-zookeeper/zk"
)

// ZookeeperBackend is a physical backend that stores data at specific
// prefix within Zookeeper. It is used in production situations as
// it allows Vault to run on multiple machines in a highly-available manner.
type ZookeeperBackend struct {
	datapath   string
	lockpath   string
	client *zk.Conn
}

// newZookeeperBackend constructs a Zookeeper backend using the given API client
// and the prefix in the KV store.
func newZookeeperBackend(conf map[string]string) (Backend, error) {
	// Get the path in Zookeeper
	basepath, ok := conf["path"]
	if !ok {
		basepath = "vault/"
	}

	// Ensure path is suffixed and prefixed (zk requires prefix /)
	if !strings.HasSuffix(basepath, "/") {
		basepath += "/"
	}
	if !strings.HasPrefix(basepath, "/") {
		basepath = "/" + basepath
	}
	datapath := basepath + "data/"
	lockpath := basepath + "lock/"

	// Configure the client, default to localhost instance
	var machines string
	machines, ok = conf["address"]
	if !ok {
		machines = "localhost:2181"
	}

	// Attempt to create the ZK client
	client, _, err := zk.Connect(strings.Split(machines, ","), time.Second)
	if err != nil {
		return nil, fmt.Errorf("client setup failed: %v", err)
	}

	// Setup the backend
	c := &ZookeeperBackend{
		datapath:   datapath,
		lockpath:   lockpath,
		client: client,
	}
	return c, nil
}

// ensurePath is used to create each node in the path hierarchy.
// We avoid calling this optimistically, and invoke it when we get
// an error during an operation
func (c *ZookeeperBackend) ensurePath(path string, value []byte) error {
	nodes := strings.Split(path, "/")
	acl := zk.WorldACL(zk.PermAll)
	fullPath := ""
	for index, node := range nodes {
		if strings.TrimSpace(node) != "" {
			fullPath += "/" + node
			isLastNode := index+1 == len(nodes)

			// set parent nodes to nil, leaf to value
			// this block reduces round trips by being smart on the leaf create/set
			if exists, _, _ := c.client.Exists(fullPath); !isLastNode && !exists {
				if _, err := c.client.Create(fullPath, nil, int32(0), acl); err != nil {
					return err
				}
			} else if isLastNode && !exists {
				if _, err := c.client.Create(fullPath, value, int32(0), acl); err != nil {
					return err
				}
			} else if isLastNode && exists {
				if _, err := c.client.Set(fullPath, value, int32(0)); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

// Put is used to insert or update an entry
func (c *ZookeeperBackend) Put(entry *Entry) error {
	defer metrics.MeasureSince([]string{"zookeeper", "put"}, time.Now())

	// Attempt to set the full path
	fullPath := c.datapath + entry.Key
	_, err := c.client.Set(fullPath, entry.Value, 0)

	// If we get ErrNoNode, we need to construct the path hierarchy
	if err == zk.ErrNoNode {
		return c.ensurePath(fullPath, entry.Value)
	}
	return err
}

// Get is used to fetch an entry
func (c *ZookeeperBackend) Get(key string) (*Entry, error) {
	defer metrics.MeasureSince([]string{"zookeeper", "get"}, time.Now())

	// Attempt to read the full path
	fullPath := c.datapath + key
	value, _, err := c.client.Get(fullPath)

	// Ignore if the node does not exist
	if err == zk.ErrNoNode {
		err = nil
	}
	if err != nil {
		return nil, err
	}

	// Handle a non-existing value
	if value == nil {
		return nil, nil
	}
	ent := &Entry{
		Key:   key,
		Value: value,
	}
	return ent, nil
}

// Delete is used to permanently delete an entry
func (c *ZookeeperBackend) Delete(key string) error {
	defer metrics.MeasureSince([]string{"zookeeper", "delete"}, time.Now())

	// Delete the full path
	fullPath := c.datapath + key
	err := c.client.Delete(fullPath, -1)

	// Mask if the node does not exist
	if err == zk.ErrNoNode {
		err = nil
	}
	return err
}

// List is used ot list all the keys under a given
// prefix, up to the next prefix.
func (c *ZookeeperBackend) List(prefix string) ([]string, error) {
	defer metrics.MeasureSince([]string{"zookeeper", "list"}, time.Now())

	// Query the children at the full path
	fullPath := strings.TrimSuffix(c.datapath+prefix, "/")
	result, _, err := c.client.Children(fullPath)

	// If the path nodes are missing, no children!
	if err == zk.ErrNoNode {
		return []string{}, nil
	}

	children := []string{}
	for _, key := range result {
		children = append(children, key)

		// Check if this entry has any child entries,
		// and append the slash which is what Vault depends on
		// for iteration
		nodeChildren, _, _ := c.client.Children(fullPath + "/" + key)
		if nodeChildren != nil && len(nodeChildren) > 0 {
			children = append(children, key+"/")
		}
	}
	sort.Strings(children)
	return children, nil
}

// LockWith is used for mutual exclusion based on the given key.
func (c *ZookeeperBackend) LockWith(key, value string) (Lock, error) {
	l := &ZookeeperHALock{
		in:    c,
		key:   key,
		value: value,
	}
	return l, nil
}

func (c *ZookeeperBackend) DetectHostAddr() (string, error) {
	// TODO: implement this!
	return "", nil
}

// ZookeeperHALock is a Zookeeper Lock implementation for the HABackend
type ZookeeperHALock struct {
	in    *ZookeeperBackend
	key   string
	value string

	held     bool
	leaderCh chan struct{}
	zkLock *zk.Lock
}

func (i *ZookeeperHALock) Lock(stopCh <-chan struct{}) (<-chan struct{}, error) {
	if i.held {
		return nil, fmt.Errorf("lock already held")
	}

	// Attempt an async acquisition
	didLock := make(chan struct{})
	failLock := make(chan error, 1)
	releaseCh := make(chan bool, 1)
	go func() {
		// Wait to acquire the lock in ZK
		acl := zk.WorldACL(zk.PermAll)
		lockpath := i.in.lockpath + i.key
		lock := zk.NewLock(i.in.client, lockpath, acl)
		err := lock.Lock()
		if err != nil {
			failLock <- err
			return
		}
		// Set node value
		err2 := i.in.ensurePath(lockpath, []byte(i.value))
		if err2 != nil {
			failLock <- err2
			lock.Unlock()
			return
		}
		i.zkLock = lock

		// Signal that lock is held
		close(didLock)

		// Handle an early abort
		release := <-releaseCh
		if release {
			lock.Unlock()
		}
	}()

	// Wait for lock acquisition, failure, or shutdown
	select {
	case <-didLock:
		releaseCh <- false
	case err := <-failLock:
		return nil, err
	case <-stopCh:
		releaseCh <- true
		return nil, nil
	}

	// Create the leader channel
	i.held = true
	i.leaderCh = make(chan struct{})

	// TODO: Watch for Events which could result in loss of our zkLock and close(i.leaderCh)

	return i.leaderCh, nil
}

func (i *ZookeeperHALock) Unlock() error {
	if !i.held {
		return nil
	}

	close(i.leaderCh)
	i.leaderCh = nil
	i.held = false
	i.zkLock.Unlock()
	return nil
}

func (i *ZookeeperHALock) Value() (bool, string, error) {
	lockpath := i.in.lockpath + i.key
	value, _, err := i.in.client.Get(lockpath)
	return i.held, string(value), err
}

<commit_msg>Implement HA lock loss detection for zookeeper physical backend<commit_after>package physical

import (
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/armon/go-metrics"
	"github.com/samuel/go-zookeeper/zk"
)

// ZookeeperBackend is a physical backend that stores data at specific
// prefix within Zookeeper. It is used in production situations as
// it allows Vault to run on multiple machines in a highly-available manner.
type ZookeeperBackend struct {
	datapath   string
	lockpath   string
	client *zk.Conn
}

// newZookeeperBackend constructs a Zookeeper backend using the given API client
// and the prefix in the KV store.
func newZookeeperBackend(conf map[string]string) (Backend, error) {
	// Get the path in Zookeeper
	basepath, ok := conf["path"]
	if !ok {
		basepath = "vault/"
	}

	// Ensure path is suffixed and prefixed (zk requires prefix /)
	if !strings.HasSuffix(basepath, "/") {
		basepath += "/"
	}
	if !strings.HasPrefix(basepath, "/") {
		basepath = "/" + basepath
	}
	datapath := basepath + "data/"
	lockpath := basepath + "lock/"

	// Configure the client, default to localhost instance
	var machines string
	machines, ok = conf["address"]
	if !ok {
		machines = "localhost:2181"
	}

	// Attempt to create the ZK client
	client, _, err := zk.Connect(strings.Split(machines, ","), time.Second)
	if err != nil {
		return nil, fmt.Errorf("client setup failed: %v", err)
	}

	// Setup the backend
	c := &ZookeeperBackend{
		datapath:   datapath,
		lockpath:   lockpath,
		client: client,
	}
	return c, nil
}

// ensurePath is used to create each node in the path hierarchy.
// We avoid calling this optimistically, and invoke it when we get
// an error during an operation
func (c *ZookeeperBackend) ensurePath(path string, value []byte) error {
	nodes := strings.Split(path, "/")
	acl := zk.WorldACL(zk.PermAll)
	fullPath := ""
	for index, node := range nodes {
		if strings.TrimSpace(node) != "" {
			fullPath += "/" + node
			isLastNode := index+1 == len(nodes)

			// set parent nodes to nil, leaf to value
			// this block reduces round trips by being smart on the leaf create/set
			if exists, _, _ := c.client.Exists(fullPath); !isLastNode && !exists {
				if _, err := c.client.Create(fullPath, nil, int32(0), acl); err != nil {
					return err
				}
			} else if isLastNode && !exists {
				if _, err := c.client.Create(fullPath, value, int32(0), acl); err != nil {
					return err
				}
			} else if isLastNode && exists {
				if _, err := c.client.Set(fullPath, value, int32(0)); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

// Put is used to insert or update an entry
func (c *ZookeeperBackend) Put(entry *Entry) error {
	defer metrics.MeasureSince([]string{"zookeeper", "put"}, time.Now())

	// Attempt to set the full path
	fullPath := c.datapath + entry.Key
	_, err := c.client.Set(fullPath, entry.Value, 0)

	// If we get ErrNoNode, we need to construct the path hierarchy
	if err == zk.ErrNoNode {
		return c.ensurePath(fullPath, entry.Value)
	}
	return err
}

// Get is used to fetch an entry
func (c *ZookeeperBackend) Get(key string) (*Entry, error) {
	defer metrics.MeasureSince([]string{"zookeeper", "get"}, time.Now())

	// Attempt to read the full path
	fullPath := c.datapath + key
	value, _, err := c.client.Get(fullPath)

	// Ignore if the node does not exist
	if err == zk.ErrNoNode {
		err = nil
	}
	if err != nil {
		return nil, err
	}

	// Handle a non-existing value
	if value == nil {
		return nil, nil
	}
	ent := &Entry{
		Key:   key,
		Value: value,
	}
	return ent, nil
}

// Delete is used to permanently delete an entry
func (c *ZookeeperBackend) Delete(key string) error {
	defer metrics.MeasureSince([]string{"zookeeper", "delete"}, time.Now())

	// Delete the full path
	fullPath := c.datapath + key
	err := c.client.Delete(fullPath, -1)

	// Mask if the node does not exist
	if err == zk.ErrNoNode {
		err = nil
	}
	return err
}

// List is used ot list all the keys under a given
// prefix, up to the next prefix.
func (c *ZookeeperBackend) List(prefix string) ([]string, error) {
	defer metrics.MeasureSince([]string{"zookeeper", "list"}, time.Now())

	// Query the children at the full path
	fullPath := strings.TrimSuffix(c.datapath+prefix, "/")
	result, _, err := c.client.Children(fullPath)

	// If the path nodes are missing, no children!
	if err == zk.ErrNoNode {
		return []string{}, nil
	}

	children := []string{}
	for _, key := range result {
		children = append(children, key)

		// Check if this entry has any child entries,
		// and append the slash which is what Vault depends on
		// for iteration
		nodeChildren, _, _ := c.client.Children(fullPath + "/" + key)
		if nodeChildren != nil && len(nodeChildren) > 0 {
			children = append(children, key+"/")
		}
	}
	sort.Strings(children)
	return children, nil
}

// LockWith is used for mutual exclusion based on the given key.
func (c *ZookeeperBackend) LockWith(key, value string) (Lock, error) {
	l := &ZookeeperHALock{
		in:    c,
		key:   key,
		value: value,
	}
	return l, nil
}

func (c *ZookeeperBackend) DetectHostAddr() (string, error) {
	// TODO: implement this!
	return "", nil
}

// ZookeeperHALock is a Zookeeper Lock implementation for the HABackend
type ZookeeperHALock struct {
	in    *ZookeeperBackend
	key   string
	value string

	held     bool
	leaderCh chan struct{}
	zkLock *zk.Lock
}

func (i *ZookeeperHALock) Lock(stopCh <-chan struct{}) (<-chan struct{}, error) {
	if i.held {
		return nil, fmt.Errorf("lock already held")
	}

	// Attempt an async acquisition
	didLock := make(chan struct{})
	failLock := make(chan error, 1)
	releaseCh := make(chan bool, 1)
	lockpath := i.in.lockpath + i.key
	go func() {
		// Wait to acquire the lock in ZK
		acl := zk.WorldACL(zk.PermAll)
		lock := zk.NewLock(i.in.client, lockpath, acl)
		err := lock.Lock()
		if err != nil {
			failLock <- err
			return
		}
		// Set node value
		err2 := i.in.ensurePath(lockpath, []byte(i.value))
		if err2 != nil {
			failLock <- err2
			lock.Unlock()
			return
		}
		i.zkLock = lock

		// Signal that lock is held
		close(didLock)

		// Handle an early abort
		release := <-releaseCh
		if release {
			lock.Unlock()
		}
	}()

	// Wait for lock acquisition, failure, or shutdown
	select {
	case <-didLock:
		releaseCh <- false
	case err := <-failLock:
		return nil, err
	case <-stopCh:
		releaseCh <- true
		return nil, nil
	}

	// Create the leader channel
	i.held = true
	i.leaderCh = make(chan struct{})

	// Watch for Events which could result in loss of our zkLock and close(i.leaderCh)
	currentVal, _, lockeventCh, err3 := i.in.client.GetW(lockpath)
	if err3 != nil {
		return nil, fmt.Errorf("unable to watch HA lock")
	}
	if i.value != string(currentVal) {
		return nil, fmt.Errorf("lost HA lock immediately before watch")
	}
	go func() {
		for {
			select {
			case event := <- lockeventCh:
				// Lost connection?
				if event.State != zk.StateConnected && event.State != zk.StateHasSession {
					close(i.leaderCh)
				}
				// Lost watch
				if event.Type == zk.EventNotWatching {
					close(i.leaderCh)
				}
				// Lock changed
				if event.Type == zk.EventNodeCreated || event.Type == zk.EventNodeDeleted || event.Type == zk.EventNodeDataChanged {
					close(i.leaderCh)
				}
			}
		}
	}()

	return i.leaderCh, nil
}

func (i *ZookeeperHALock) Unlock() error {
	if !i.held {
		return nil
	}

	close(i.leaderCh)
	i.leaderCh = nil
	i.held = false
	i.zkLock.Unlock()
	return nil
}

func (i *ZookeeperHALock) Value() (bool, string, error) {
	lockpath := i.in.lockpath + i.key
	value, _, err := i.in.client.Get(lockpath)
	return i.held, string(value), err
}

<|endoftext|>


<commit_before>package physical

import (
	"fmt"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/armon/go-metrics"
	"github.com/samuel/go-zookeeper/zk"
)

// ZookeeperBackend is a physical backend that stores data at specific
// prefix within Zookeeper. It is used in production situations as
// it allows Vault to run on multiple machines in a highly-available manner.
type ZookeeperBackend struct {
	path   string
	client *zk.Conn
}

// newZookeeperBackend constructs a Zookeeper backend using the given API client
// and the prefix in the KV store.
func newZookeeperBackend(conf map[string]string) (Backend, error) {
	// Get the path in Zookeeper
	path, ok := conf["path"]
	if !ok {
		path = "vault/"
	}

	// Ensure path is suffixed and prefixed (zk requires prefix /)
	if !strings.HasSuffix(path, "/") {
		path += "/"
	}
	if !strings.HasPrefix(path, "/") {
		path = "/" + path
	}

	// Configure the client, default to localhost instance
	var machines string
	machines, ok = conf["address"]
	if !ok {
		machines = "localhost:2181"
	}

	// Attempt to create the ZK client
	client, _, err := zk.Connect(strings.Split(machines, ","), time.Second)
	if err != nil {
		return nil, fmt.Errorf("client setup failed: %v", err)
	}

	// Setup the backend
	c := &ZookeeperBackend{
		path:   path,
		client: client,
	}
	return c, nil
}

// ensurePath is used to create each node in the path hierarchy.
// We avoid calling this optimistically, and invoke it when we get
// an error during an operation
func (c *ZookeeperBackend) ensurePath(path string, value []byte) error {
	nodes := strings.Split(path, "/")
	acl := zk.WorldACL(zk.PermAll)
	fullPath := ""
	for index, node := range nodes {
		if strings.TrimSpace(node) != "" {
			fullPath += "/" + node
			isLastNode := index+1 == len(nodes)

			// set parent nodes to nil, leaf to value
			// this block reduces round trips by being smart on the leaf create/set
			if exists, _, _ := c.client.Exists(fullPath); !isLastNode && !exists {
				if _, err := c.client.Create(fullPath, nil, int32(0), acl); err != nil {
					return err
				}
			} else if isLastNode && !exists {
				if _, err := c.client.Create(fullPath, value, int32(0), acl); err != nil {
					return err
				}
			} else if isLastNode && exists {
				if _, err := c.client.Set(fullPath, value, int32(-1)); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

// deletePath is a helper that will recursively delete
// a given path
func (c *ZookeeperBackend) deletePath(path string) error {
	children, _, _ := c.client.Children(path)

	for _, childPath := range children {
		err := c.deletePath(path + "/" + childPath)

		if err != nil {
			return err
		}
	}
	err := c.client.Delete(path, -1)
	if err != nil {
		return err
	}

	return nil
}

// Put is used to insert or update an entry
func (c *ZookeeperBackend) Put(entry *Entry) error {
	defer metrics.MeasureSince([]string{"zookeeper", "put"}, time.Now())

	// Attempt to set the full path
	fullPath := c.path + entry.Key
	_, err := c.client.Set(fullPath, entry.Value, -1)

	// If we get ErrNoNode, we need to construct the path hierarchy
	if err == zk.ErrNoNode {
		return c.ensurePath(fullPath, entry.Value)
	}
	return err
}

// Get is used to fetch an entry
func (c *ZookeeperBackend) Get(key string) (*Entry, error) {
	defer metrics.MeasureSince([]string{"zookeeper", "get"}, time.Now())

	// Attempt to read the full path
	fullPath := c.path + key
	value, _, err := c.client.Get(fullPath)

	// Ignore if the node does not exist
	if err == zk.ErrNoNode {
		err = nil
	}
	if err != nil {
		return nil, err
	}

	// Handle a non-existing value
	if value == nil {
		return nil, nil
	}
	ent := &Entry{
		Key:   key,
		Value: value,
	}
	return ent, nil
}

// Delete is used to permanently delete an entry
func (c *ZookeeperBackend) Delete(key string) error {
	defer metrics.MeasureSince([]string{"zookeeper", "delete"}, time.Now())

	// Delete the full path
	fullPath := c.path + key
	err := c.deletePath(fullPath)

	// Mask if the node does not exist
	if err == zk.ErrNoNode {
		err = nil
	}
	return err
}

// List is used ot list all the keys under a given
// prefix, up to the next prefix.
func (c *ZookeeperBackend) List(prefix string) ([]string, error) {
	defer metrics.MeasureSince([]string{"zookeeper", "list"}, time.Now())

	// Query the children at the full path
	fullPath := strings.TrimSuffix(c.path+prefix, "/")
	result, _, err := c.client.Children(fullPath)

	// If the path nodes are missing, no children!
	if err == zk.ErrNoNode {
		return []string{}, nil
	}

	children := []string{}
	for _, key := range result {
		children = append(children, key)

		// Check if this entry has any child entries,
		// and append the slash which is what Vault depends on
		// for iteration
		nodeChildren, _, _ := c.client.Children(fullPath + "/" + key)
		if nodeChildren != nil && len(nodeChildren) > 0 {
			children = append(children, key+"/")
		}
	}
	sort.Strings(children)
	return children, nil
}

// LockWith is used for mutual exclusion based on the given key.
func (c *ZookeeperBackend) LockWith(key, value string) (Lock, error) {
	l := &ZookeeperHALock{
		in:    c,
		key:   key,
		value: value,
	}
	return l, nil
}

// ZookeeperHALock is a Zookeeper Lock implementation for the HABackend
type ZookeeperHALock struct {
	in    *ZookeeperBackend
	key   string
	value string

	held      bool
	localLock sync.Mutex
	leaderCh  chan struct{}
	zkLock    *zk.Lock
}

func (i *ZookeeperHALock) Lock(stopCh <-chan struct{}) (<-chan struct{}, error) {
	i.localLock.Lock()
	defer i.localLock.Unlock()
	if i.held {
		return nil, fmt.Errorf("lock already held")
	}

	// Attempt an async acquisition
	didLock := make(chan struct{})
	failLock := make(chan error, 1)
	releaseCh := make(chan bool, 1)
	lockpath := i.in.path + i.key
	go i.attemptLock(lockpath, didLock, failLock, releaseCh)

	// Wait for lock acquisition, failure, or shutdown
	select {
	case <-didLock:
		releaseCh <- false
	case err := <-failLock:
		return nil, err
	case <-stopCh:
		releaseCh <- true
		return nil, nil
	}

	// Create the leader channel
	i.held = true
	i.leaderCh = make(chan struct{})

	// Watch for Events which could result in loss of our zkLock and close(i.leaderCh)
	currentVal, _, lockeventCh, err3 := i.in.client.GetW(lockpath)
	if err3 != nil {
		return nil, fmt.Errorf("unable to watch HA lock")
	}
	if i.value != string(currentVal) {
		return nil, fmt.Errorf("lost HA lock immediately before watch")
	}
	go i.monitorLock(lockeventCh)

	return i.leaderCh, nil
}

func (i *ZookeeperHALock) attemptLock(lockpath string, didLock chan struct{}, failLock chan error, releaseCh chan bool) {
	// Wait to acquire the lock in ZK
	acl := zk.WorldACL(zk.PermAll)
	lock := zk.NewLock(i.in.client, lockpath, acl)
	err := lock.Lock()
	if err != nil {
		failLock <- err
		return
	}
	// Set node value
	data := []byte(i.value)
	err2 := i.in.ensurePath(lockpath, data)
	if err2 != nil {
		failLock <- err2
		lock.Unlock()
		return
	}
	i.zkLock = lock

	// Signal that lock is held
	close(didLock)

	// Handle an early abort
	release := <-releaseCh
	if release {
		lock.Unlock()
	}
}

func (i *ZookeeperHALock) monitorLock(lockeventCh <-chan zk.Event) {
	for {
		select {
		case event := <- lockeventCh:
			// Lost connection?
			if event.State == zk.StateUnknown || event.State == zk.StateDisconnected ||  event.State == zk.StateConnecting || event.State == zk.StateAuthFailed || event.State == zk.StateConnectedReadOnly || event.State == zk.StateExpired {
				close(i.leaderCh)
				return
			}
			// Lost watch
			if event.Type == zk.EventNotWatching {
				close(i.leaderCh)
				return
			}
			// Lock changed
			if event.Type == zk.EventNodeCreated || event.Type == zk.EventNodeDeleted || event.Type == zk.EventNodeDataChanged {
				close(i.leaderCh)
				return
			}
		}
	}
}

func (i *ZookeeperHALock) Unlock() error {
	i.localLock.Lock()
	defer i.localLock.Unlock()
	if !i.held {
		return nil
	}

	i.held = false
	i.zkLock.Unlock()
	return nil
}

func (i *ZookeeperHALock) Value() (bool, string, error) {
	lockpath := i.in.path + i.key
	value, _, err := i.in.client.Get(lockpath)
	return (value != nil), string(value), err
}

<commit_msg>Cleaned up error handling and HA lock monitoring for zookeeper physical backend based on PR feedback.<commit_after>package physical

import (
	"fmt"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/armon/go-metrics"
	"github.com/samuel/go-zookeeper/zk"
)

// ZookeeperBackend is a physical backend that stores data at specific
// prefix within Zookeeper. It is used in production situations as
// it allows Vault to run on multiple machines in a highly-available manner.
type ZookeeperBackend struct {
	path   string
	client *zk.Conn
}

// newZookeeperBackend constructs a Zookeeper backend using the given API client
// and the prefix in the KV store.
func newZookeeperBackend(conf map[string]string) (Backend, error) {
	// Get the path in Zookeeper
	path, ok := conf["path"]
	if !ok {
		path = "vault/"
	}

	// Ensure path is suffixed and prefixed (zk requires prefix /)
	if !strings.HasSuffix(path, "/") {
		path += "/"
	}
	if !strings.HasPrefix(path, "/") {
		path = "/" + path
	}

	// Configure the client, default to localhost instance
	var machines string
	machines, ok = conf["address"]
	if !ok {
		machines = "localhost:2181"
	}

	// Attempt to create the ZK client
	client, _, err := zk.Connect(strings.Split(machines, ","), time.Second)
	if err != nil {
		return nil, fmt.Errorf("client setup failed: %v", err)
	}

	// Setup the backend
	c := &ZookeeperBackend{
		path:   path,
		client: client,
	}
	return c, nil
}

// ensurePath is used to create each node in the path hierarchy.
// We avoid calling this optimistically, and invoke it when we get
// an error during an operation
func (c *ZookeeperBackend) ensurePath(path string, value []byte) error {
	nodes := strings.Split(path, "/")
	acl := zk.WorldACL(zk.PermAll)
	fullPath := ""
	for index, node := range nodes {
		if strings.TrimSpace(node) != "" {
			fullPath += "/" + node
			isLastNode := index+1 == len(nodes)

			// set parent nodes to nil, leaf to value
			// this block reduces round trips by being smart on the leaf create/set
			if exists, _, _ := c.client.Exists(fullPath); !isLastNode && !exists {
				if _, err := c.client.Create(fullPath, nil, int32(0), acl); err != nil {
					return err
				}
			} else if isLastNode && !exists {
				if _, err := c.client.Create(fullPath, value, int32(0), acl); err != nil {
					return err
				}
			} else if isLastNode && exists {
				if _, err := c.client.Set(fullPath, value, int32(-1)); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

// deletePath is a helper that will recursively delete
// a given path
func (c *ZookeeperBackend) deletePath(path string) error {
	children, _, _ := c.client.Children(path)

	for _, childPath := range children {
		err := c.deletePath(path + "/" + childPath)

		if err != nil {
			return err
		}
	}
	err := c.client.Delete(path, -1)
	if err != nil {
		return err
	}

	return nil
}

// Put is used to insert or update an entry
func (c *ZookeeperBackend) Put(entry *Entry) error {
	defer metrics.MeasureSince([]string{"zookeeper", "put"}, time.Now())

	// Attempt to set the full path
	fullPath := c.path + entry.Key
	_, err := c.client.Set(fullPath, entry.Value, -1)

	// If we get ErrNoNode, we need to construct the path hierarchy
	if err == zk.ErrNoNode {
		return c.ensurePath(fullPath, entry.Value)
	}
	return err
}

// Get is used to fetch an entry
func (c *ZookeeperBackend) Get(key string) (*Entry, error) {
	defer metrics.MeasureSince([]string{"zookeeper", "get"}, time.Now())

	// Attempt to read the full path
	fullPath := c.path + key
	value, _, err := c.client.Get(fullPath)

	// Ignore if the node does not exist
	if err == zk.ErrNoNode {
		err = nil
	}
	if err != nil {
		return nil, err
	}

	// Handle a non-existing value
	if value == nil {
		return nil, nil
	}
	ent := &Entry{
		Key:   key,
		Value: value,
	}
	return ent, nil
}

// Delete is used to permanently delete an entry
func (c *ZookeeperBackend) Delete(key string) error {
	defer metrics.MeasureSince([]string{"zookeeper", "delete"}, time.Now())

	// Delete the full path
	fullPath := c.path + key
	err := c.deletePath(fullPath)

	// Mask if the node does not exist
	if err == zk.ErrNoNode {
		err = nil
	}
	return err
}

// List is used ot list all the keys under a given
// prefix, up to the next prefix.
func (c *ZookeeperBackend) List(prefix string) ([]string, error) {
	defer metrics.MeasureSince([]string{"zookeeper", "list"}, time.Now())

	// Query the children at the full path
	fullPath := strings.TrimSuffix(c.path+prefix, "/")
	result, _, err := c.client.Children(fullPath)

	// If the path nodes are missing, no children!
	if err == zk.ErrNoNode {
		return []string{}, nil
	}

	children := []string{}
	for _, key := range result {
		children = append(children, key)

		// Check if this entry has any child entries,
		// and append the slash which is what Vault depends on
		// for iteration
		nodeChildren, _, _ := c.client.Children(fullPath + "/" + key)
		if nodeChildren != nil && len(nodeChildren) > 0 {
			children = append(children, key+"/")
		}
	}
	sort.Strings(children)
	return children, nil
}

// LockWith is used for mutual exclusion based on the given key.
func (c *ZookeeperBackend) LockWith(key, value string) (Lock, error) {
	l := &ZookeeperHALock{
		in:    c,
		key:   key,
		value: value,
	}
	return l, nil
}

// ZookeeperHALock is a Zookeeper Lock implementation for the HABackend
type ZookeeperHALock struct {
	in    *ZookeeperBackend
	key   string
	value string

	held      bool
	localLock sync.Mutex
	leaderCh  chan struct{}
	zkLock    *zk.Lock
}

func (i *ZookeeperHALock) Lock(stopCh <-chan struct{}) (<-chan struct{}, error) {
	i.localLock.Lock()
	defer i.localLock.Unlock()
	if i.held {
		return nil, fmt.Errorf("lock already held")
	}

	// Attempt an async acquisition
	didLock := make(chan struct{})
	failLock := make(chan error, 1)
	releaseCh := make(chan bool, 1)
	lockpath := i.in.path + i.key
	go i.attemptLock(lockpath, didLock, failLock, releaseCh)

	// Wait for lock acquisition, failure, or shutdown
	select {
	case <-didLock:
		releaseCh <- false
	case err := <-failLock:
		return nil, err
	case <-stopCh:
		releaseCh <- true
		return nil, nil
	}

	// Create the leader channel
	i.held = true
	i.leaderCh = make(chan struct{})

	// Watch for Events which could result in loss of our zkLock and close(i.leaderCh)
	currentVal, _, lockeventCh, err := i.in.client.GetW(lockpath)
	if err != nil {
		return nil, fmt.Errorf("unable to watch HA lock: %v", err)
	}
	if i.value != string(currentVal) {
		return nil, fmt.Errorf("lost HA lock immediately before watch")
	}
	go i.monitorLock(lockeventCh, i.leaderCh)

	return i.leaderCh, nil
}

func (i *ZookeeperHALock) attemptLock(lockpath string, didLock chan struct{}, failLock chan error, releaseCh chan bool) {
	// Wait to acquire the lock in ZK
	acl := zk.WorldACL(zk.PermAll)
	lock := zk.NewLock(i.in.client, lockpath, acl)
	err := lock.Lock()
	if err != nil {
		failLock <- err
		return
	}
	// Set node value
	data := []byte(i.value)
	err = i.in.ensurePath(lockpath, data)
	if err != nil {
		failLock <- err
		lock.Unlock()
		return
	}
	i.zkLock = lock

	// Signal that lock is held
	close(didLock)

	// Handle an early abort
	release := <-releaseCh
	if release {
		lock.Unlock()
	}
}

func (i *ZookeeperHALock) monitorLock(lockeventCh <-chan zk.Event, leaderCh chan struct{}) {
	for {
		select {
		case event := <- lockeventCh:
			// Lost connection?
			switch event.State {
			case zk.StateConnected:
			case zk.StateSyncConnected:
			case zk.StateHasSession:
			default:
				close(leaderCh)
				return
			}

			// Lost lock?
			switch event.Type {
			case zk.EventNodeChildrenChanged:
			case zk.EventSession:
			default:
				close(leaderCh)
				return
			}
		}
	}
}

func (i *ZookeeperHALock) Unlock() error {
	i.localLock.Lock()
	defer i.localLock.Unlock()
	if !i.held {
		return nil
	}

	i.held = false
	i.zkLock.Unlock()
	return nil
}

func (i *ZookeeperHALock) Value() (bool, string, error) {
	lockpath := i.in.path + i.key
	value, _, err := i.in.client.Get(lockpath)
	return (value != nil), string(value), err
}

<|endoftext|>


<commit_before>package alerts

import (
	"context"
	"fmt"
	"strings"
	"time"

	o "github.com/onsi/gomega"
	configclient "github.com/openshift/client-go/config/clientset/versioned"
	"github.com/openshift/origin/pkg/synthetictests/allowedalerts"
	"github.com/openshift/origin/test/extended/util/disruption"
	helper "github.com/openshift/origin/test/extended/util/prometheus"
	prometheusv1 "github.com/prometheus/client_golang/api/prometheus/v1"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/kubernetes/test/e2e/framework"
)

type allowedAlertsFunc func(configclient.Interface) (allowedFiringWithBugs, allowedFiring, allowedPendingWithBugs, allowedPending helper.MetricConditions)

func CheckAlerts(allowancesFunc allowedAlertsFunc, prometheusClient prometheusv1.API, configClient configclient.Interface, startTime time.Time) {
	firingAlertsWithBugs, allowedFiringAlerts, pendingAlertsWithBugs, allowedPendingAlerts :=
		allowancesFunc(configClient)

	// we exclude alerts that have their own separate tests.
	for _, alertTest := range allowedalerts.AllAlertTests(context.TODO(), nil, 0) {
		switch alertTest.AlertState() {
		case allowedalerts.AlertPending:
			// a pending test covers pending and everything above (firing)
			allowedPendingAlerts = append(allowedPendingAlerts,
				helper.MetricCondition{
					Selector: map[string]string{"alertname": alertTest.AlertName()},
					Text:     "has a separate e2e test",
				},
			)
			allowedFiringAlerts = append(allowedFiringAlerts,
				helper.MetricCondition{
					Selector: map[string]string{"alertname": alertTest.AlertName()},
					Text:     "has a separate e2e test",
				},
			)
		case allowedalerts.AlertInfo:
			// an info test covers all firing
			allowedFiringAlerts = append(allowedFiringAlerts,
				helper.MetricCondition{
					Selector: map[string]string{"alertname": alertTest.AlertName()},
					Text:     "has a separate e2e test",
				},
			)
		}
	}

	knownViolations := sets.NewString()
	unexpectedViolations := sets.NewString()
	unexpectedViolationsAsFlakes := sets.NewString()
	debug := sets.NewString()

	testDuration := time.Now().Sub(startTime).Round(time.Second)

	// Invariant: No non-info level alerts should have fired during the upgrade
	firingAlertQuery := fmt.Sprintf(`
sort_desc(
count_over_time(ALERTS{alertstate="firing",severity!="info",alertname!~"Watchdog|AlertmanagerReceiversNotConfigured"}[%[1]s:1s])
) > 0
`, testDuration)
	result, err := helper.RunQuery(context.TODO(), prometheusClient, firingAlertQuery)
	o.Expect(err).NotTo(o.HaveOccurred(), "unable to check firing alerts during upgrade")
	for _, series := range result.Data.Result {
		labels := helper.StripLabels(series.Metric, "alertname", "alertstate", "prometheus")
		violation := fmt.Sprintf("alert %s fired for %s seconds with labels: %s", series.Metric["alertname"], series.Value, helper.LabelsAsSelector(labels))
		if cause := allowedFiringAlerts.Matches(series); cause != nil {
			debug.Insert(fmt.Sprintf("%s (allowed: %s)", violation, cause.Text))
			continue
		}
		if cause := firingAlertsWithBugs.Matches(series); cause != nil {
			knownViolations.Insert(fmt.Sprintf("%s (open bug: %s)", violation, cause.Text))
		} else {
			unexpectedViolations.Insert(violation)
		}
	}

	// Invariant: There should be no pending alerts 1m after the upgrade completes
	pendingAlertQuery := fmt.Sprintf(`
sort_desc(
  time() * ALERTS + 1
  -
  last_over_time((
    time() * ALERTS{alertname!~"Watchdog|AlertmanagerReceiversNotConfigured",alertstate="pending",severity!="info"}
    unless
    ALERTS offset 1s
  )[%[1]s:1s])
)
`, testDuration)
	result, err = helper.RunQuery(context.TODO(), prometheusClient, pendingAlertQuery)
	o.Expect(err).NotTo(o.HaveOccurred(), "unable to retrieve pending alerts after upgrade")
	for _, series := range result.Data.Result {
		labels := helper.StripLabels(series.Metric, "alertname", "alertstate", "prometheus")
		violation := fmt.Sprintf("alert %s pending for %s seconds with labels: %s", series.Metric["alertname"], series.Value, helper.LabelsAsSelector(labels))
		if cause := allowedPendingAlerts.Matches(series); cause != nil {
			debug.Insert(fmt.Sprintf("%s (allowed: %s)", violation, cause.Text))
			continue
		}
		if cause := pendingAlertsWithBugs.Matches(series); cause != nil {
			knownViolations.Insert(fmt.Sprintf("%s (open bug: %s)", violation, cause.Text))
		} else {
			// treat pending errors as a flake right now because we are still trying to determine the scope
			// TODO: move this to unexpectedViolations later
			unexpectedViolationsAsFlakes.Insert(violation)
		}
	}

	if len(debug) > 0 {
		framework.Logf("Alerts were detected during upgrade which are allowed:\n\n%s", strings.Join(debug.List(), "\n"))
	}
	if len(unexpectedViolations) > 0 {
		framework.Failf("Unexpected alerts fired or pending during the upgrade:\n\n%s", strings.Join(unexpectedViolations.List(), "\n"))
	}
	if flakes := sets.NewString().Union(knownViolations).Union(unexpectedViolations).Union(unexpectedViolationsAsFlakes); len(flakes) > 0 {
		disruption.FrameworkFlakef(f, "Unexpected alert behavior during upgrade:\n\n%s", strings.Join(flakes.List(), "\n"))

		/*
			framework.Logf(format, options...)
			f.TestSummaries = append(f.TestSummaries, flakeSummary(fmt.Sprintf(format, options...)))
		*/
	}
	framework.Logf("No alerts fired during upgrade")

}
<commit_msg>wip: Fork logic on flake reporting, unsure what the implications of choosing here are<commit_after>package alerts

import (
	"context"
	"fmt"
	"strings"
	"time"

	o "github.com/onsi/gomega"
	configclient "github.com/openshift/client-go/config/clientset/versioned"
	"github.com/openshift/origin/pkg/synthetictests/allowedalerts"
	testresult "github.com/openshift/origin/pkg/test/ginkgo/result"
	"github.com/openshift/origin/test/extended/util/disruption"
	helper "github.com/openshift/origin/test/extended/util/prometheus"
	prometheusv1 "github.com/prometheus/client_golang/api/prometheus/v1"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/kubernetes/test/e2e/framework"
)

type allowedAlertsFunc func(configclient.Interface) (allowedFiringWithBugs, allowedFiring, allowedPendingWithBugs, allowedPending helper.MetricConditions)

func CheckAlerts(allowancesFunc allowedAlertsFunc, prometheusClient prometheusv1.API, configClient configclient.Interface, startTime time.Time, f *framework.Framework) {
	firingAlertsWithBugs, allowedFiringAlerts, pendingAlertsWithBugs, allowedPendingAlerts :=
		allowancesFunc(configClient)

	// we exclude alerts that have their own separate tests.
	for _, alertTest := range allowedalerts.AllAlertTests(context.TODO(), nil, 0) {
		switch alertTest.AlertState() {
		case allowedalerts.AlertPending:
			// a pending test covers pending and everything above (firing)
			allowedPendingAlerts = append(allowedPendingAlerts,
				helper.MetricCondition{
					Selector: map[string]string{"alertname": alertTest.AlertName()},
					Text:     "has a separate e2e test",
				},
			)
			allowedFiringAlerts = append(allowedFiringAlerts,
				helper.MetricCondition{
					Selector: map[string]string{"alertname": alertTest.AlertName()},
					Text:     "has a separate e2e test",
				},
			)
		case allowedalerts.AlertInfo:
			// an info test covers all firing
			allowedFiringAlerts = append(allowedFiringAlerts,
				helper.MetricCondition{
					Selector: map[string]string{"alertname": alertTest.AlertName()},
					Text:     "has a separate e2e test",
				},
			)
		}
	}

	knownViolations := sets.NewString()
	unexpectedViolations := sets.NewString()
	unexpectedViolationsAsFlakes := sets.NewString()
	debug := sets.NewString()

	testDuration := time.Now().Sub(startTime).Round(time.Second)

	// Invariant: No non-info level alerts should have fired during the upgrade
	firingAlertQuery := fmt.Sprintf(`
sort_desc(
count_over_time(ALERTS{alertstate="firing",severity!="info",alertname!~"Watchdog|AlertmanagerReceiversNotConfigured"}[%[1]s:1s])
) > 0
`, testDuration)
	result, err := helper.RunQuery(context.TODO(), prometheusClient, firingAlertQuery)
	o.Expect(err).NotTo(o.HaveOccurred(), "unable to check firing alerts during upgrade")
	for _, series := range result.Data.Result {
		labels := helper.StripLabels(series.Metric, "alertname", "alertstate", "prometheus")
		violation := fmt.Sprintf("alert %s fired for %s seconds with labels: %s", series.Metric["alertname"], series.Value, helper.LabelsAsSelector(labels))
		if cause := allowedFiringAlerts.Matches(series); cause != nil {
			debug.Insert(fmt.Sprintf("%s (allowed: %s)", violation, cause.Text))
			continue
		}
		if cause := firingAlertsWithBugs.Matches(series); cause != nil {
			knownViolations.Insert(fmt.Sprintf("%s (open bug: %s)", violation, cause.Text))
		} else {
			unexpectedViolations.Insert(violation)
		}
	}

	// Invariant: There should be no pending alerts 1m after the upgrade completes
	pendingAlertQuery := fmt.Sprintf(`
sort_desc(
  time() * ALERTS + 1
  -
  last_over_time((
    time() * ALERTS{alertname!~"Watchdog|AlertmanagerReceiversNotConfigured",alertstate="pending",severity!="info"}
    unless
    ALERTS offset 1s
  )[%[1]s:1s])
)
`, testDuration)
	result, err = helper.RunQuery(context.TODO(), prometheusClient, pendingAlertQuery)
	o.Expect(err).NotTo(o.HaveOccurred(), "unable to retrieve pending alerts after upgrade")
	for _, series := range result.Data.Result {
		labels := helper.StripLabels(series.Metric, "alertname", "alertstate", "prometheus")
		violation := fmt.Sprintf("alert %s pending for %s seconds with labels: %s", series.Metric["alertname"], series.Value, helper.LabelsAsSelector(labels))
		if cause := allowedPendingAlerts.Matches(series); cause != nil {
			debug.Insert(fmt.Sprintf("%s (allowed: %s)", violation, cause.Text))
			continue
		}
		if cause := pendingAlertsWithBugs.Matches(series); cause != nil {
			knownViolations.Insert(fmt.Sprintf("%s (open bug: %s)", violation, cause.Text))
		} else {
			// treat pending errors as a flake right now because we are still trying to determine the scope
			// TODO: move this to unexpectedViolations later
			unexpectedViolationsAsFlakes.Insert(violation)
		}
	}

	if len(debug) > 0 {
		framework.Logf("Alerts were detected during upgrade which are allowed:\n\n%s", strings.Join(debug.List(), "\n"))
	}
	if len(unexpectedViolations) > 0 {
		framework.Failf("Unexpected alerts fired or pending during the upgrade:\n\n%s", strings.Join(unexpectedViolations.List(), "\n"))
	}
	if flakes := sets.NewString().Union(knownViolations).Union(unexpectedViolations).Union(unexpectedViolationsAsFlakes); len(flakes) > 0 {
		// TODO: The two tests that had this duplicated code had slightly different ways of reporting flakes
		// that I do not fully understand the implications of. Fork the logic here.
		if f != nil {
			// when called from alert.go within an UpgradeTest with a framework available
			// f.TestSummaries is the part I'm unsure about here.
			disruption.FrameworkFlakef(f, "Unexpected alert behavior during upgrade:\n\n%s", strings.Join(flakes.List(), "\n"))
		} else {
			// when called from prometheus.go with no framework available
			testresult.Flakef("Unexpected alert behavior during test:\n\n%s", strings.Join(flakes.List(), "\n"))
		}
	}
	framework.Logf("No alerts fired during upgrade")

}
<|endoftext|>


<commit_before>package ganglia

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"os"
	"regexp"
	"runtime"
	"strconv"
	"sync"
	"syscall"
	"time"

	"github.com/fastly/go-utils/debug"
	"github.com/fastly/go-utils/stopper"
	"github.com/fastly/go-utils/vlog"
	"github.com/jbuchbinder/go-gmetric/gmetric"
)

const (
	String = gmetric.VALUE_STRING
	Ushort = gmetric.VALUE_UNSIGNED_SHORT
	Short  = gmetric.VALUE_SHORT
	Uint   = gmetric.VALUE_UNSIGNED_INT
	Int    = gmetric.VALUE_INT
	Float  = gmetric.VALUE_FLOAT
	Double = gmetric.VALUE_DOUBLE
)

var (
	GmondConfig string
	Interval    time.Duration
	// set using Configure(), which should be called before any other functions
	groupName string

	gmondChannelRe  = regexp.MustCompile("udp_send_channel\\s*{([^}]+)}")
	gmondHostPortRe = regexp.MustCompile("(host|port)\\s*=\\s*(\\S+)")

	globalReporter struct {
		sync.Once
		*Reporter
	}
)

func init() {
	flag.StringVar(&GmondConfig, "gmond-config", "/etc/ganglia/gmond.conf", "location of gmond.conf")
	flag.DurationVar(&Interval, "ganglia-interval", 9*time.Second, "time between gmetric updates")
}

type gmetricSample struct {
	value interface{}
	when  time.Time
}
type Reporter struct {
	*stopper.ChanStopper
	prefix    string
	callbacks []ReporterCallback
	previous  map[string]gmetricSample
}

// Configure sets the group name that will be used in ganglia calls and whether to
// turn on more verbose logging. The verbose flag is used for the vlog package, so
// setting it here will override previous options.
func Configure(GroupName string, Verbose bool) {
	groupName = GroupName
	vlog.Verbose = Verbose
}

// MetricSender takes the following parameters:
//   name: an arbitrary metric name
//   value: the metric's current value
//   metricType: one of GmetricString, GmetricUshort, GmetricShort, GmetricUint, GmetricInt, GmetricFloat, or GmetricDouble
//   units: a label to include on the metric's Y axis
//   rate: if true, send the rate relative to the last sample instead of an absolute value
type MetricSender func(name string, value string, metricType uint32, units string, rate bool)

type ReporterCallback func(MetricSender)

// Gmetric returns a global Reporter that clients may hook into by
// calling AddCallback.
func Gmetric() *Reporter {
	globalReporter.Do(func() {
		globalReporter.Reporter = NewGangliaReporter(Interval)
		globalReporter.AddCallback(CommonGmetrics)
	})
	return globalReporter.Reporter
}

// Convenience wrapper for Gmetric().AddCallback():
//
//   AddGmetrics(func(gmetric MetricSender) {
// 	   gmetric("profit", "1000000.00", GmetricFloat, "dollars", true)
//   })
func AddGmetrics(callback ReporterCallback) {
	Gmetric().AddCallback(callback)
}

func NewGmetric() (*gmetric.Gmetric, error) {
	b, err := ioutil.ReadFile(GmondConfig)
	if err != nil {
		return nil, err
	}
	stanzas := gmondChannelRe.FindAllStringSubmatch(string(b), -1)
	if len(stanzas) == 0 {
		return nil, fmt.Errorf("No udp_send_channel stanzas found in %s", GmondConfig)
	}

	servers := make([]gmetric.GmetricServer, 0)
	for _, stanza := range stanzas {
		var host, port string
		for _, match := range gmondHostPortRe.FindAllStringSubmatch(stanza[1], 2) {
			if match[1] == "host" {
				host = match[2]
			} else if match[1] == "port" {
				port = match[2]
			}
		}
		if host == "" || port == "" {
			return nil, fmt.Errorf("Missing host or port from %s stanza %q", GmondConfig, stanza[0])
		}
		portNum, err := strconv.Atoi(port)
		if err != nil {
			return nil, err
		}
		ips, err := net.LookupIP(host)
		if err != nil {
			return nil, err
		}
		for _, ip := range ips {
			vlog.VLogf("Reporting to Ganglia server at %s:%d", ip, portNum)
			servers = append(servers, gmetric.GmetricServer{ip, portNum})
		}
	}

	// see http://sourceforge.net/apps/trac/ganglia/wiki/gmetric_spoofing
	hostname, _ := os.Hostname()
	spoofName := fmt.Sprintf("%s:%s", hostname, hostname)

	gm := gmetric.Gmetric{Spoof: spoofName}
	for _, server := range servers {
		gm.AddServer(server)
	}
	return &gm, nil
}

// NewGangliaReporter return a Reporter object which calls callback every
// interval with the given group name. callback is passed a Gmetric whose
// servers are initialized from the hosts gmond.conf. Calling Stop on the
// Reporter will cease its operation.
func NewGangliaReporter(interval time.Duration) *Reporter {
	// set before the call to NewGmetric so VLogf in NewGmetric works properly
	gm, err := NewGmetric()
	if err != nil {
		vlog.VLogfQuiet("ganglia", "Couldn't start Ganglia reporter: %s", err)
		return nil
	} else if gm == nil {
		return nil
	}
	stopper := stopper.NewChanStopper()
	gr := &Reporter{stopper, "", make([]ReporterCallback, 0), make(map[string]gmetricSample)}
	go func() {
		defer stopper.Done()
		for {
			select {
			case <-stopper.Chan:
				return
			case <-time.After(interval):
				go func() {
					// SendMetric "opens" and "closes" UDP connections each
					// time, but since we expect the callback to send several
					// metrics at once, avoid that here.
					conns := gm.OpenConnections()
					n := 0
					sender := func(name string, value string, metricType uint32, units string, rate bool) {
						v := value
						if rate {
							prev, exists := gr.previous[name]
							units += "/sec"

							now := time.Now()

							switch metricType {
							case Ushort, Short, Uint, Int:
								i, err := strconv.Atoi(value)
								if err != nil {
									vlog.VLogfQuiet(name, "Value %q doesn't look like an int: %s", value, err)
									return
								}
								gr.previous[name] = gmetricSample{i, now}
								if !exists {
									return
								}
								delta := i - prev.value.(int)
								elapsed := time.Now().Sub(prev.when).Seconds()
								v = fmt.Sprint(float64(delta) / elapsed)
								// upgrade to a float to avoid loss of precision
								metricType = Float

							case Float, Double:
								f, err := strconv.ParseFloat(value, 64)
								if err != nil {
									vlog.VLogfQuiet(name, "Value %q doesn't look like a float: %s", value, err)
									return
								}
								gr.previous[name] = gmetricSample{f, now}
								if !exists {
									return
								}
								delta := f - prev.value.(float64)
								elapsed := time.Now().Sub(prev.when).Seconds()
								v = fmt.Sprint(delta / elapsed)

							case String:
								vlog.VLogfQuiet(name, "Can't compute deltas for string metric %q", value)
								return
							}
						}

						n++
						gm.SendMetricPackets(
							gr.prefix+name, v, metricType, units,
							gmetric.SLOPE_BOTH,
							uint32(interval.Seconds()), // tmax is the expected reporting interval
							0, // dmax is the time to keep values in tsdb; 0 means forever
							groupName,
							gmetric.PACKET_BOTH, conns,
						)
						if debug.On() {
							if rate {
								log.Printf("gmetric: name=%q, rate=%q, value=%q, type=%d, units=%q, slope=%d, tmax=%d, dmax=%v, group=%q, packet=%d",
									gr.prefix+name, v, value, metricType, units, gmetric.SLOPE_BOTH,
									uint32(interval.Seconds()), 0, groupName, gmetric.PACKET_BOTH,
								)
							} else {
								log.Printf("gmetric: name=%q, value=%q, type=%d, units=%q, slope=%d, tmax=%d, dmax=%v, group=%q, packet=%d",
									gr.prefix+name, v, metricType, units, gmetric.SLOPE_BOTH,
									uint32(interval.Seconds()), 0, groupName, gmetric.PACKET_BOTH,
								)
							}
						}
					}
					defer gm.CloseConnections(conns)
					for _, callback := range gr.callbacks {
						callback(sender)
					}
					if debug.On() {
						log.Printf("Published %d metrics to Ganglia", n)
					}
				}()
			}
		}
	}()
	return gr
}

func (gr *Reporter) AddCallback(callback ReporterCallback) {
	if gr == nil {
		return
	}
	gr.callbacks = append(gr.callbacks, callback)
}

func (gr *Reporter) SetPrefix(prefix string) {
	if gr == nil {
		return
	}
	gr.prefix = prefix
}

func (g *Reporter) Stop() {
	if g == nil {
		return
	}
	g.Stop()
}

func CommonGmetrics(gmetric MetricSender) {
	gmetric("goroutines", fmt.Sprintf("%d", runtime.NumGoroutine()), Uint, "num", false)

	var mem runtime.MemStats
	runtime.ReadMemStats(&mem)
	gmetric("mem_alloc", fmt.Sprintf("%d", mem.Alloc), Uint, "bytes", false)
	gmetric("mem_sys", fmt.Sprintf("%d", mem.Sys), Uint, "bytes", false)
	gmetric("mem_gc_pause_last", fmt.Sprintf("%.6f", float64(mem.PauseNs[(mem.NumGC+255)%256])/1e6), Float, "ms", false)
	var gcPauseMax uint64
	for _, v := range mem.PauseNs {
		if v > gcPauseMax {
			gcPauseMax = v
		}
	}
	gmetric("mem_gc_pause_max", fmt.Sprintf("%.6f", float64(gcPauseMax)/1e6), Float, "ms", false)
	gmetric("mem_gc_pause_total", fmt.Sprintf("%.6f", float64(mem.PauseTotalNs)/1e6), Float, "ms", true)
	since := time.Now().Sub(time.Unix(0, int64(mem.LastGC))).Seconds()
	gmetric("mem_gc_pause_since", fmt.Sprintf("%.6f", since), Float, "sec", false)

	var r syscall.Rusage
	if syscall.Getrusage(syscall.RUSAGE_SELF, &r) == nil {
		gmetric("rusage_utime", fmt.Sprintf("%.6f", float64(r.Utime.Nano())/1e9), Float, "cpusecs", true)
		gmetric("rusage_stime", fmt.Sprintf("%.6f", float64(r.Stime.Nano())/1e9), Float, "cpusecs", true)
	}
}
<commit_msg>remove groupName global, add NewGangliaReporterWithOptions<commit_after>package ganglia

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"os"
	"regexp"
	"runtime"
	"strconv"
	"sync"
	"syscall"
	"time"

	"github.com/fastly/go-utils/debug"
	"github.com/fastly/go-utils/stopper"
	"github.com/fastly/go-utils/vlog"
	"github.com/jbuchbinder/go-gmetric/gmetric"
)

const (
	String = gmetric.VALUE_STRING
	Ushort = gmetric.VALUE_UNSIGNED_SHORT
	Short  = gmetric.VALUE_SHORT
	Uint   = gmetric.VALUE_UNSIGNED_INT
	Int    = gmetric.VALUE_INT
	Float  = gmetric.VALUE_FLOAT
	Double = gmetric.VALUE_DOUBLE
)

var (
	GmondConfig string
	Interval    time.Duration

	gmondChannelRe  = regexp.MustCompile("udp_send_channel\\s*{([^}]+)}")
	gmondHostPortRe = regexp.MustCompile("(host|port)\\s*=\\s*(\\S+)")

	globalReporter struct {
		sync.Once
		*Reporter
	}
)

func init() {
	flag.StringVar(&GmondConfig, "gmond-config", "/etc/ganglia/gmond.conf", "location of gmond.conf")
	flag.DurationVar(&Interval, "ganglia-interval", 9*time.Second, "time between gmetric updates")
}

type gmetricSample struct {
	value interface{}
	when  time.Time
}
type Reporter struct {
	*stopper.ChanStopper
	prefix    string
	callbacks []ReporterCallback
	previous  map[string]gmetricSample
	groupName string
}

// MetricSender takes the following parameters:
//   name: an arbitrary metric name
//   value: the metric's current value
//   metricType: one of GmetricString, GmetricUshort, GmetricShort, GmetricUint, GmetricInt, GmetricFloat, or GmetricDouble
//   units: a label to include on the metric's Y axis
//   rate: if true, send the rate relative to the last sample instead of an absolute value
type MetricSender func(name string, value string, metricType uint32, units string, rate bool)

type ReporterCallback func(MetricSender)

// Gmetric returns a global Reporter that clients may hook into by
// calling AddCallback.
func Gmetric() *Reporter {
	globalReporter.Do(func() {
		globalReporter.Reporter = NewGangliaReporter(Interval)
		globalReporter.AddCallback(CommonGmetrics)
	})
	return globalReporter.Reporter
}

// Convenience wrapper for Gmetric().AddCallback():
//
//   AddGmetrics(func(gmetric MetricSender) {
// 	   gmetric("profit", "1000000.00", GmetricFloat, "dollars", true)
//   })
func AddGmetrics(callback ReporterCallback) {
	Gmetric().AddCallback(callback)
}

func NewGmetric() (*gmetric.Gmetric, error) {
	b, err := ioutil.ReadFile(GmondConfig)
	if err != nil {
		return nil, err
	}
	stanzas := gmondChannelRe.FindAllStringSubmatch(string(b), -1)
	if len(stanzas) == 0 {
		return nil, fmt.Errorf("No udp_send_channel stanzas found in %s", GmondConfig)
	}

	servers := make([]gmetric.GmetricServer, 0)
	for _, stanza := range stanzas {
		var host, port string
		for _, match := range gmondHostPortRe.FindAllStringSubmatch(stanza[1], 2) {
			if match[1] == "host" {
				host = match[2]
			} else if match[1] == "port" {
				port = match[2]
			}
		}
		if host == "" || port == "" {
			return nil, fmt.Errorf("Missing host or port from %s stanza %q", GmondConfig, stanza[0])
		}
		portNum, err := strconv.Atoi(port)
		if err != nil {
			return nil, err
		}
		ips, err := net.LookupIP(host)
		if err != nil {
			return nil, err
		}
		for _, ip := range ips {
			vlog.VLogf("Reporting to Ganglia server at %s:%d", ip, portNum)
			servers = append(servers, gmetric.GmetricServer{ip, portNum})
		}
	}

	// see http://sourceforge.net/apps/trac/ganglia/wiki/gmetric_spoofing
	hostname, _ := os.Hostname()
	spoofName := fmt.Sprintf("%s:%s", hostname, hostname)

	gm := gmetric.Gmetric{Spoof: spoofName}
	for _, server := range servers {
		gm.AddServer(server)
	}
	return &gm, nil
}

// NewGangliaReporter returns a Reporter object which calls callback every
// interval with the given group name. callback is passed a Gmetric whose
// servers are initialized from the hosts gmond.conf. Calling Stop on the
// Reporter will cease its operation.
func NewGangliaReporter(interval time.Duration) *Reporter {
	return NewGangliaReporterWithOptions(interval, "", false)
}

// NewGangliaReporterWithOptions is NewGangliaReporter with the groupName
// and verbose parameters explicit.
func NewGangliaReporterWithOptions(interval time.Duration, groupName string, verbose bool) *Reporter {
	// set before the call to NewGmetric so VLogf in NewGmetric works properly
	vlog.Verbose = verbose
	gm, err := NewGmetric()
	if err != nil {
		vlog.VLogfQuiet("ganglia", "Couldn't start Ganglia reporter: %s", err)
		return nil
	} else if gm == nil {
		return nil
	}
	stopper := stopper.NewChanStopper()
	gr := &Reporter{stopper, "", make([]ReporterCallback, 0), make(map[string]gmetricSample), groupName}
	go func() {
		defer stopper.Done()
		for {
			select {
			case <-stopper.Chan:
				return
			case <-time.After(interval):
				go func() {
					// SendMetric "opens" and "closes" UDP connections each
					// time, but since we expect the callback to send several
					// metrics at once, avoid that here.
					conns := gm.OpenConnections()
					n := 0
					sender := func(name string, value string, metricType uint32, units string, rate bool) {
						v := value
						if rate {
							prev, exists := gr.previous[name]
							units += "/sec"

							now := time.Now()

							switch metricType {
							case Ushort, Short, Uint, Int:
								i, err := strconv.Atoi(value)
								if err != nil {
									vlog.VLogfQuiet(name, "Value %q doesn't look like an int: %s", value, err)
									return
								}
								gr.previous[name] = gmetricSample{i, now}
								if !exists {
									return
								}
								delta := i - prev.value.(int)
								elapsed := time.Now().Sub(prev.when).Seconds()
								v = fmt.Sprint(float64(delta) / elapsed)
								// upgrade to a float to avoid loss of precision
								metricType = Float

							case Float, Double:
								f, err := strconv.ParseFloat(value, 64)
								if err != nil {
									vlog.VLogfQuiet(name, "Value %q doesn't look like a float: %s", value, err)
									return
								}
								gr.previous[name] = gmetricSample{f, now}
								if !exists {
									return
								}
								delta := f - prev.value.(float64)
								elapsed := time.Now().Sub(prev.when).Seconds()
								v = fmt.Sprint(delta / elapsed)

							case String:
								vlog.VLogfQuiet(name, "Can't compute deltas for string metric %q", value)
								return
							}
						}

						n++
						gm.SendMetricPackets(
							gr.prefix+name, v, metricType, units,
							gmetric.SLOPE_BOTH,
							uint32(interval.Seconds()), // tmax is the expected reporting interval
							0, // dmax is the time to keep values in tsdb; 0 means forever
							groupName,
							gmetric.PACKET_BOTH, conns,
						)
						if debug.On() {
							if rate {
								log.Printf("gmetric: name=%q, rate=%q, value=%q, type=%d, units=%q, slope=%d, tmax=%d, dmax=%v, group=%q, packet=%d",
									gr.prefix+name, v, value, metricType, units, gmetric.SLOPE_BOTH,
									uint32(interval.Seconds()), 0, groupName, gmetric.PACKET_BOTH,
								)
							} else {
								log.Printf("gmetric: name=%q, value=%q, type=%d, units=%q, slope=%d, tmax=%d, dmax=%v, group=%q, packet=%d",
									gr.prefix+name, v, metricType, units, gmetric.SLOPE_BOTH,
									uint32(interval.Seconds()), 0, groupName, gmetric.PACKET_BOTH,
								)
							}
						}
					}
					defer gm.CloseConnections(conns)
					for _, callback := range gr.callbacks {
						callback(sender)
					}
					if debug.On() {
						log.Printf("Published %d metrics to Ganglia", n)
					}
				}()
			}
		}
	}()
	return gr
}

func (gr *Reporter) AddCallback(callback ReporterCallback) {
	if gr == nil {
		return
	}
	gr.callbacks = append(gr.callbacks, callback)
}

func (gr *Reporter) SetPrefix(prefix string) {
	if gr == nil {
		return
	}
	gr.prefix = prefix
}

func (g *Reporter) Stop() {
	if g == nil {
		return
	}
	g.Stop()
}

func CommonGmetrics(gmetric MetricSender) {
	gmetric("goroutines", fmt.Sprintf("%d", runtime.NumGoroutine()), Uint, "num", false)

	var mem runtime.MemStats
	runtime.ReadMemStats(&mem)
	gmetric("mem_alloc", fmt.Sprintf("%d", mem.Alloc), Uint, "bytes", false)
	gmetric("mem_sys", fmt.Sprintf("%d", mem.Sys), Uint, "bytes", false)
	gmetric("mem_gc_pause_last", fmt.Sprintf("%.6f", float64(mem.PauseNs[(mem.NumGC+255)%256])/1e6), Float, "ms", false)
	var gcPauseMax uint64
	for _, v := range mem.PauseNs {
		if v > gcPauseMax {
			gcPauseMax = v
		}
	}
	gmetric("mem_gc_pause_max", fmt.Sprintf("%.6f", float64(gcPauseMax)/1e6), Float, "ms", false)
	gmetric("mem_gc_pause_total", fmt.Sprintf("%.6f", float64(mem.PauseTotalNs)/1e6), Float, "ms", true)
	since := time.Now().Sub(time.Unix(0, int64(mem.LastGC))).Seconds()
	gmetric("mem_gc_pause_since", fmt.Sprintf("%.6f", since), Float, "sec", false)

	var r syscall.Rusage
	if syscall.Getrusage(syscall.RUSAGE_SELF, &r) == nil {
		gmetric("rusage_utime", fmt.Sprintf("%.6f", float64(r.Utime.Nano())/1e9), Float, "cpusecs", true)
		gmetric("rusage_stime", fmt.Sprintf("%.6f", float64(r.Stime.Nano())/1e9), Float, "cpusecs", true)
	}
}
<|endoftext|>


<commit_before>package LevenshteinTrie

import (
	"fmt"
	"sort"
	"unicode/utf8"
)

func Min(a ...int) int {
	min := int(^uint(0) >> 1) // largest int
	for _, i := range a {
		if i < min {
			min = i
		}
	}
	return min
}
func Max(a ...int) int {
	max := int(0)
	for _, i := range a {
		if i > max {
			max = i
		}
	}
	return max
}

type TrieNode struct {
	letter   rune //Equivalent to int32
	children map[rune]*TrieNode
	final    bool
	text     string
}

func (t *TrieNode) String() string {
	s := fmt.Sprintf("%#U\n", t.letter)
	for _, v := range t.children {
		s += fmt.Sprintf("-%#U\n", v)
	}
	return s
}

func NewTrie() *TrieNode {
	return &TrieNode{children: make(map[rune]*TrieNode)}
}

func (root *TrieNode) InsertText(text string) {
	if root == nil {
		return
	}

	currNode := root //Starts at root
	for i, w := 0, 0; i < len(text); i += w {
		runeValue, width := utf8.DecodeRuneInString(text[i:])
		final := false
		if width+i == len(text) {
			final = true
		}
		w = width

		currNode = NewTrieNode(currNode, runeValue, final, text)
	}
}

func NewTrieNode(t *TrieNode, runeValue rune, final bool, text string) *TrieNode {
	node, exists := t.children[runeValue]
	if !exists {
		node = &TrieNode{letter: runeValue, children: make(map[rune]*TrieNode)}
		t.children[runeValue] = node
	}
	if final {
		node.final = true
		node.text = text
	}
	return node
}

func (t *TrieNode) SearchSuffix(query string) []string {
	var curr *TrieNode
	var ok bool

	curr = t
	//first, find the end of the prefix
	for _, letter := range query {
		if curr != nil {
			curr, ok = curr.children[letter]
			if ok {
				//do nothing
			}

		} else {
			return nil
		}
	}

	candidates := getsuffixr(curr)

	return candidates
}

func getsuffixr(n *TrieNode) []string {
	if n == nil {
		return nil
	}

	candidates := make([]string, 0)
	if n.final == true {
		candidates = append(candidates, n.text)
	}

	for _, childNode := range n.children {
		candidates = append(candidates, getsuffixr(childNode)...)
	}
	return candidates
}

type QueryResult struct {
	Val      string
	Distance int
}

func (q QueryResult) String() string {
	return fmt.Sprintf("Val: %s, Dist: %d\n", q.Val, q.Distance)
}

type ByDistance []QueryResult

func (a ByDistance) Len() int           { return len(a) }
func (a ByDistance) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByDistance) Less(i, j int) bool { return a[i].Distance < a[j].Distance }

func (n *TrieNode) SearchLevenshtein(text string, distance int) []QueryResult {

	//initialize the first row for the dynamic programming alg
	l := utf8.RuneCount([]byte(text))
	currentRow := make([]int, l+1)

	for i := 0; i < len(currentRow); i++ {
		currentRow[i] = i
	}

	candidates := make([]QueryResult, 0)

	for letter, childNode := range n.children {
		candidates = append(candidates, searchlevr(childNode, currentRow, letter, []rune(text), distance)...)
	}

	sort.Sort(ByDistance(candidates))
	return candidates
}

func searchlevr(n *TrieNode, prevRow []int, letter rune, text []rune, maxDistance int) []QueryResult {
	columns := len(prevRow)
	currentRow := make([]int, columns)

	currentRow[0] = prevRow[0] + 1

	for col := 1; col < columns; col++ {
		if text[col-1] == letter {
			currentRow[col] = prevRow[col-1]
			continue
		}
		insertCost := currentRow[col-1] + 1
		deleteCost := prevRow[col] + 1
		replaceCost := prevRow[col-1] + 1

		currentRow[col] = Min(insertCost, deleteCost, replaceCost)
	}

	candidates := make([]QueryResult, 0)

	distance := currentRow[len(currentRow)-1]
	if distance <= maxDistance && n.final == true {
		candidates = append(candidates, QueryResult{Val: n.text, Distance: distance})
	}
	mi := Min(currentRow[1:]...)
	if mi <= maxDistance {
		for l, childNode := range n.children {
			candidates = append(candidates, searchlevr(childNode, currentRow, l, text, maxDistance)...)
		}
	}
	return candidates
}
<commit_msg>final changes<commit_after>package LevenshteinTrie

import (
	"fmt"
	"sort"
	"unicode/utf8"
)

func Min(a ...int) int {
	min := int(^uint(0) >> 1) // largest int
	for _, i := range a {
		if i < min {
			min = i
		}
	}
	return min
}
func Max(a ...int) int {
	max := int(0)
	for _, i := range a {
		if i > max {
			max = i
		}
	}
	return max
}

type TrieNode struct {
	letter   rune //Equivalent to int32
	children map[rune]*TrieNode
	final    bool
	text     string
}

func (t *TrieNode) String() string {
	s := fmt.Sprintf("%#U\n", t.letter)
	for _, v := range t.children {
		s += fmt.Sprintf("-%#U\n", v)
	}
	return s
}

func NewTrie() *TrieNode {
	return &TrieNode{children: make(map[rune]*TrieNode)}
}

func (root *TrieNode) InsertText(text string) {
	if root == nil {
		return
	}

	currNode := root //Starts at root
	for i, w := 0, 0; i < len(text); i += w {
		runeValue, width := utf8.DecodeRuneInString(text[i:])
		final := false
		if width+i == len(text) {
			final = true
		}
		w = width

		currNode = NewTrieNode(currNode, runeValue, final, text)
	}
}

func NewTrieNode(t *TrieNode, runeValue rune, final bool, text string) *TrieNode {
	node, exists := t.children[runeValue]
	if !exists {
		node = &TrieNode{letter: runeValue, children: make(map[rune]*TrieNode)}
		t.children[runeValue] = node
	}
	if final {
		node.final = true
		node.text = text
	}
	return node
}

func (t *TrieNode) Suffix(query string) []string {
	var curr *TrieNode
	var ok bool

	curr = t
	//first, find the end of the prefix
	for _, letter := range query {
		if curr != nil {
			curr, ok = curr.children[letter]
			if ok {
				//do nothing
			}

		} else {
			return nil
		}
	}

	candidates := getsuffixr(curr)

	return candidates
}

func getsuffixr(n *TrieNode) []string {
	if n == nil {
		return nil
	}

	candidates := make([]string, 0)
	if n.final == true {
		candidates = append(candidates, n.text)
	}

	for _, childNode := range n.children {
		candidates = append(candidates, getsuffixr(childNode)...)
	}
	return candidates
}

type QueryResult struct {
	Val      string
	Distance int
}

func (q QueryResult) String() string {
	return fmt.Sprintf("Val: %s, Dist: %d\n", q.Val, q.Distance)
}

type ByDistance []QueryResult

func (a ByDistance) Len() int           { return len(a) }
func (a ByDistance) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByDistance) Less(i, j int) bool { return a[i].Distance < a[j].Distance }

func (n *TrieNode) Levenshtein(text string, distance int) []QueryResult {

	//initialize the first row for the dynamic programming alg
	l := utf8.RuneCount([]byte(text))
	currentRow := make([]int, l+1)

	for i := 0; i < len(currentRow); i++ {
		currentRow[i] = i
	}

	candidates := make([]QueryResult, 0)

	for letter, childNode := range n.children {
		candidates = append(candidates, searchlevr(childNode, currentRow, letter, []rune(text), distance)...)
	}

	sort.Sort(ByDistance(candidates))
	return candidates
}

func searchlevr(n *TrieNode, prevRow []int, letter rune, text []rune, maxDistance int) []QueryResult {
	columns := len(prevRow)
	currentRow := make([]int, columns)

	currentRow[0] = prevRow[0] + 1

	for col := 1; col < columns; col++ {
		if text[col-1] == letter {
			currentRow[col] = prevRow[col-1]
			continue
		}
		insertCost := currentRow[col-1] + 1
		deleteCost := prevRow[col] + 1
		replaceCost := prevRow[col-1] + 1

		currentRow[col] = Min(insertCost, deleteCost, replaceCost)
	}

	candidates := make([]QueryResult, 0)

	distance := currentRow[len(currentRow)-1]
	if distance <= maxDistance && n.final == true {
		candidates = append(candidates, QueryResult{Val: n.text, Distance: distance})
	}
	mi := Min(currentRow[1:]...)
	if mi <= maxDistance {
		for l, childNode := range n.children {
			candidates = append(candidates, searchlevr(childNode, currentRow, l, text, maxDistance)...)
		}
	}
	return candidates
}
<|endoftext|>