<commit_before>package gc

import (
	"sync"
)

func indexOfResource(resources []Disposable, resource Disposable) int {
	for i, r := range resources {
		if r == resource {
			return i
		}
	}
	return -1
}

// A ResourceTracker is an object capable of tracking resources.
//
// This is the interface for the GarbageCollector that should be exposed to
// engines and plugins. So they can't initiate garbage collection.
type ResourceTracker interface {
	Register(resource Disposable)
	Unregister(resource Disposable) bool
}

// GarbageCollector can be used register Disposable resources which will then
// be diposed when not in use and the system is low on available disk space
// or memory.
type GarbageCollector struct {
	resources []Disposable
	m         sync.Mutex
}

// Register takes a Disposable resource for the GarbageCollector to manage.
//
// GarbageCollector will attempt to to call resource.Dispose() at any time,
// you should return ErrDisposableInUse if the resource is in use.
func (gc *GarbageCollector) Register(resource Disposable) {
	gc.m.Lock()
	defer gc.m.Unlock()
	if indexOfResource(gc.resources, resource) == -1 {
		gc.resources = append(gc.resources, resource)
	}
}

// Unregister will inform the GarbageCollector to stop tracking the given
// resource. Returns true if the resource was tracked, notice that the resource
// maybe have been disposed of while waiting for the GC lock. Say if the GC was
// running when you made this call.
//
// Note, you don't have to use this method. When you resouce is in a state
// where you don't want it to be disposed just ensure that Dispose() returns
// ErrDisposableInUse.
func (gc *GarbageCollector) Unregister(resource Disposable) bool {
	gc.m.Lock()
	defer gc.m.Unlock()
	i := indexOfResource(gc.resources, resource)
	if i != -1 {
		gc.resources = append(gc.resources[:i], gc.resources[i+1:]...)
		return true
	}
	return false
}

// Collect runs garbage collection and reclaims resources, at this stage it just
// calls CollectAll(), but in the future this should dispose resources somewhat
// intelligently.
func (gc *GarbageCollector) Collect() error {
	return gc.CollectAll()
}

// CollectAll disposes all resources that can be disposed.
//
// All resources not returning: ErrDisposableInUse.
// This is useful for testing when implementing resources.
func (gc *GarbageCollector) CollectAll() error {
	gc.m.Lock()
	defer gc.m.Unlock()
	var resources []Disposable
	for i, resource := range gc.resources {
		err := resource.Dispose()
		if err != nil {
			if err != ErrDisposableInUse {
				gc.resources = append(resources, gc.resources[i-1:]...)
				return err
			}
			resources = append(resources, resource)
		}
	}
	gc.resources = resources
	return nil
}
<commit_msg>Modify gc.Collect() to only dispose as needed in a LRU fashion<commit_after>package gc

import (
	"sort"
	"sync"
)

func indexOfResource(resources []Disposable, resource Disposable) int {
	for i, r := range resources {
		if r == resource {
			return i
		}
	}
	return -1
}

type disposableSorter []Disposable

func (d disposableSorter) Len() int {
	return len(d)
}

func (d disposableSorter) Swap(i, j int) {
	d[i], d[j] = d[j], d[i]
}

func (d disposableSorter) Less(i, j int) bool {
	return d[i].LastUsed().Before(d[j].LastUsed())
}

// A ResourceTracker is an object capable of tracking resources.
//
// This is the interface for the GarbageCollector that should be exposed to
// engines and plugins. So they can't initiate garbage collection.
type ResourceTracker interface {
	Register(resource Disposable)
	Unregister(resource Disposable) bool
}

// GarbageCollector can be used register Disposable resources which will then
// be diposed when not in use and the system is low on available disk space
// or memory.
type GarbageCollector struct {
	resources        []Disposable
	m                sync.Mutex
	storageFolder    string
	minimumDiskSpace int64
	minimumMemory    int64
}

// New creates a GarbageCollector which uses storageFolder to test for available
// diskspace and tries to ensure that minimumDiskSpace and minimumMemory is
// satisfied after each call to Collect()
func New(storageFolder string, minimumDiskSpace, minimumMemory int64) *GarbageCollector {
	return &GarbageCollector{
		storageFolder:    storageFolder,
		minimumDiskSpace: minimumDiskSpace,
		minimumMemory:    minimumMemory,
	}
}

// Register takes a Disposable resource for the GarbageCollector to manage.
//
// GarbageCollector will attempt to to call resource.Dispose() at any time,
// you should return ErrDisposableInUse if the resource is in use.
func (gc *GarbageCollector) Register(resource Disposable) {
	gc.m.Lock()
	defer gc.m.Unlock()
	if indexOfResource(gc.resources, resource) == -1 {
		gc.resources = append(gc.resources, resource)
	}
}

// Unregister will inform the GarbageCollector to stop tracking the given
// resource. Returns true if the resource was tracked, notice that the resource
// maybe have been disposed of while waiting for the GC lock. Say if the GC was
// running when you made this call.
//
// Note, you don't have to use this method. When you resouce is in a state
// where you don't want it to be disposed just ensure that Dispose() returns
// ErrDisposableInUse.
func (gc *GarbageCollector) Unregister(resource Disposable) bool {
	gc.m.Lock()
	defer gc.m.Unlock()
	i := indexOfResource(gc.resources, resource)
	if i != -1 {
		gc.resources = append(gc.resources[:i], gc.resources[i+1:]...)
		return true
	}
	return false
}

// Collect runs garbage collection and reclaims resources, attempting to
// satisfy minimumMemory and minimumDiskSpace, if possible.
func (gc *GarbageCollector) Collect() error {
	gc.m.Lock()
	defer gc.m.Unlock()

	// Sort to get least-recently-used first
	sort.Sort(disposableSorter(gc.resources))

	var resources []Disposable
	for i, r := range gc.resources {
		var err error
		var size uint64

		if gc.needDiskSpace() {
			size, err = r.DiskSize()
			if err != nil && err != ErrDisposableSizeNotSupported {
				goto abort
			}
			if size > 0 || err == ErrDisposableSizeNotSupported {
				goto dispose
			}
		}

		if gc.needMemory() {
			size, err = r.MemorySize()
			if err != nil && err != ErrDisposableSizeNotSupported {
				goto abort
			}
			if size > 0 || err == ErrDisposableSizeNotSupported {
				goto dispose
			}
		}

		goto keep
	keep:
		resources = append(resources, r)
		continue
	abort:
		gc.resources = append(resources, gc.resources[i-1:]...)
		return err
	dispose:
		err = r.Dispose()
		if err != nil {
			if err != ErrDisposableInUse {
				goto abort
			}
			goto keep
		}
	}

	gc.resources = resources
	return nil
}

// CollectAll disposes all resources that can be disposed.
//
// All resources not returning: ErrDisposableInUse.
// This is useful for testing when implementing resources.
func (gc *GarbageCollector) CollectAll() error {
	gc.m.Lock()
	defer gc.m.Unlock()
	var resources []Disposable
	for i, resource := range gc.resources {
		err := resource.Dispose()
		if err != nil {
			if err != ErrDisposableInUse {
				gc.resources = append(resources, gc.resources[i-1:]...)
				return err
			}
			resources = append(resources, resource)
		}
	}
	gc.resources = resources
	return nil
}

// needDiskSpace returns true if we need to free diskspace
func (gc *GarbageCollector) needDiskSpace() bool {
	return true
}

// needMemory returns true if we need to free memory
func (gc *GarbageCollector) needMemory() bool {
	return true
}
<|endoftext|>


<commit_before>package scans

import (
	"encoding/json"
	"fmt"
	"strings"
	"time"

	"github.com/ion-channel/ionic/vulnerabilities"
)

// Results is an Ion Channel representation of the results from a
// scan summary.  It contains what type of results and the data pertaining to
// the results.
type Results struct {
	Type string      `json:"type" xml:"type"`
	Data interface{} `json:"data,omitempty" xml:"data,omitempty"`
}

type results struct {
	Type    string          `json:"type"`
	RawData json.RawMessage `json:"data"`
}

// UnmarshalJSON is a custom JSON unmarshaller implementation for the standard
// go json package to know how to properly interpret ScanSummaryResults from
// JSON.
func (r *Results) UnmarshalJSON(b []byte) error {
	var tr results
	err := json.Unmarshal(b, &tr)
	if err != nil {
		return err
	}

	r.Type = tr.Type

	switch strings.ToLower(tr.Type) {
	case "about_yml":
		var a AboutYMLResults
		err := json.Unmarshal(tr.RawData, &a)
		if err != nil {
			return fmt.Errorf("failed to unmarshall about yml results: %v", err)
		}

		r.Data = a
	case "community":
		var c []CommunityResults
		err := json.Unmarshal(tr.RawData, &c)
		if err != nil {
			return fmt.Errorf("failed to unmarshall community results: %v", err)
		}

		r.Data = c
	case "coverage", "external_coverage":
		var c CoverageResults
		err := json.Unmarshal(tr.RawData, &c)
		if err != nil {
			return fmt.Errorf("failed to unmarshall coverage results: %v", err)
		}

		r.Data = c
	case "dependency":
		var d DependencyResults
		err := json.Unmarshal(tr.RawData, &d)
		if err != nil {
			return fmt.Errorf("failed to unmarshall dependency results: %v", err)
		}

		r.Data = d
	case "ecosystems":
		var e EcosystemResults
		err := json.Unmarshal(tr.RawData, &e)
		if err != nil {
			return fmt.Errorf("failed to unmarshall ecosystems results: %v", err)
		}

		r.Data = e
	case "license":
		var l LicenseResults
		err := json.Unmarshal(tr.RawData, &l)
		if err != nil {
			return fmt.Errorf("failed to unmarshall license results: %v", err)
		}

		r.Data = l
	case "virus", "clamav":
		var v VirusResults
		err := json.Unmarshal(tr.RawData, &v)
		if err != nil {
			return fmt.Errorf("failed to unmarshall virus results: %v", err)
		}

		r.Data = v
	case "vulnerability":
		var v VulnerabilityResults
		err := json.Unmarshal(tr.RawData, &v)
		if err != nil {
			return fmt.Errorf("failed to unmarshall vulnerability results: %v", err)
		}

		r.Data = v
	case "external_vulnerability":
		var v ExternalVulnerabilitiesResults
		err := json.Unmarshal(tr.RawData, &v)
		if err != nil {
			return fmt.Errorf("failed to unmarshall external vulnerabilities results: %v", err)
		}

		r.Data = v
	case "difference":
		var v DifferenceResults
		err := json.Unmarshal(tr.RawData, &v)
		if err != nil {
			return fmt.Errorf("failed to unmarshall difference results: %v", err)
		}

		r.Data = v
	default:
		return fmt.Errorf("unsupported results type found: %v", tr.Type)
	}

	return nil
}

// AboutYMLResults represents the data collected from the AboutYML scan.  It
// includes a message and whether or not the About YML file found was valid or
// not.
type AboutYMLResults struct {
	Message string `json:"message" xml:"message"`
	Valid   bool   `json:"valid" xml:"valid"`
	Content string `json:"content" xml:"content"`
}

// CommunityResults represents the data collected from a community scan.  It
// represents all known data regarding the open community of a software project
type CommunityResults struct {
	Committers int    `json:"committers" xml:"committers"`
	Name       string `json:"name" xml:"name"`
	URL        string `json:"url" xml:"url"`
}

// CoverageResults represents the data collected from a code coverage scan.  It
// includes the value of the code coverage seen for the project.
type CoverageResults struct {
	Value float64 `json:"value" xml:"value"`
}

// DependencyResults represents the data collected from a dependency scan.  It
// includes a list of the dependencies seen and meta data counts about those
// dependencies seen.
type DependencyResults struct {
	Dependencies []struct {
		LatestVersion string `json:"latest_version" xml:"latest_version"`
		Org           string `json:"org" xml:"org"`
		Name          string `json:"name" xml:"name"`
		Type          string `json:"type" xml:"type"`
		Package       string `json:"package" xml:"package"`
		Version       string `json:"version" xml:"version"`
		Scope         string `json:"scope" xml:"scope"`
	} `json:"dependencies" xml:"dependencies"`
	Meta struct {
		FirstDegreeCount     int `json:"first_degree_count" xml:"first_degree_count"`
		NoVersionCount       int `json:"no_version_count" xml:"no_version_count"`
		TotalUniqueCount     int `json:"total_unique_count" xml:"total_unique_count"`
		UpdateAvailableCount int `json:"update_available_count" xml:"update_available_count"`
	} `json:"meta" xml:"meta"`
}

// DifferenceResults represents the checksum of a project.  It includes a checksum
// and flag indicating if there was a difference detected within that last 5 scans
type DifferenceResults struct {
	Checksum   string `json:"checksum" xml:"checksum"`
	Difference bool   `json:"difference" xml:"difference"`
}

// EcosystemResults represents the data collected from an ecosystems scan.  It
// include the name of the ecosystem and the number of lines seen for the given
// ecosystem.
type EcosystemResults struct {
	Ecosystems []struct {
		Ecosystem string `json:"ecosystem" xml:"ecosystem"`
		Lines     int    `json:"lines" xml:"lines"`
	} `json:"ecosystems" xml:"ecosystems"`
}

// ExternalVulnerabilitiesResults represents the data collected from an external
// vulnerability scan.  It includes the number of each vulnerability criticality
// seen within the project.
type ExternalVulnerabilitiesResults struct {
	Critical int `json:"critical" xml:"critical"`
	High     int `json:"high" xml:"high"`
	Medium   int `json:"medium" xml:"medium"`
	Low      int `json:"low" xml:"low"`
}

// LicenseResults represents the data collected from a license scan.  It
// includes the name and type of each license seen within the project.
type LicenseResults struct {
	License struct {
		Name string `json:"name" xml:"name"`
		Type []struct {
			Name string `json:"name" xml:"name"`
		} `json:"type" xml:"type"`
	} `json:"license" xml:"license"`
}

// VirusResults represents the data collected from a virus scan.  It includes
// information of the viruses seen and the virus scanner used.
type VirusResults struct {
	KnownViruses       int    `json:"known_viruses" xml:"known_viruses"`
	EngineVersion      string `json:"engine_version" xml:"engine_version"`
	ScannedDirectories int    `json:"scanned_directories" xml:"scanned_directories"`
	ScannedFiles       int    `json:"scanned_files" xml:"scanned_files"`
	InfectedFiles      int    `json:"infected_files" xml:"infected_files"`
	DataScanned        string `json:"data_scanned" xml:"data_scanned"`
	DataRead           string `json:"data_read" xml:"data_read"`
	Time               string `json:"time" xml:"time"`
	FileNotes          struct {
		EmptyFiles []string `json:"empty_file" xml:"empty_file"`
	} `json:"file_notes" xml:"file_notes"`
	ClamavDetails struct {
		ClamavVersion   string `json:"clamav_version" xml:"clamav_version"`
		ClamavDbVersion string `json:"clamav_db_version" xml:"clamav_db_version"`
	} `json:"clam_av_details" xml:"clam_av_details"`
}

//VulnerabilityResults represents the data collected from a vulnerability scan.  It includes
// information of the vulnerabilities seen.
type VulnerabilityResults struct {
	Vulnerabilities []struct {
		ID              int                             `json:"id" xml:"id"`
		ExternalID      string                          `json:"external_id" xml:"external_id"`
		SourceID        int                             `json:"source_id" xml:"source_id"`
		Title           string                          `json:"title" xml:"title"`
		Name            string                          `json:"name" xml:"name"`
		Org             string                          `json:"org" xml:"org"`
		Version         string                          `json:"version" xml:"version"`
		Up              interface{}                     `json:"up" xml:"up"`
		Edition         interface{}                     `json:"edition" xml:"edition"`
		Aliases         []string                        `json:"aliases" xml:"aliases"`
		CreatedAt       time.Time                       `json:"created_at" xml:"created_at"`
		UpdatedAt       time.Time                       `json:"updated_at" xml:"updated_at"`
		References      interface{}                     `json:"references" xml:"references"`
		Part            interface{}                     `json:"part" xml:"part"`
		Language        interface{}                     `json:"language" xml:"language"`
		Vulnerabilities []vulnerabilities.Vulnerability `json:"vulnerabilities" xml:"vulnerabilities"`
	} `json:"vulnerabilities" xml:"vulnerabilities"`
	Meta struct {
		VulnerabilityCount int `json:"vulnerability_count" xml:"vulnerability_count"`
	} `json:"meta" xml:"meta"`
}
<commit_msg>this pulls the inline structs for dependencies up for sdk exposure<commit_after>package scans

import (
	"encoding/json"
	"fmt"
	"strings"
	"time"

	"github.com/ion-channel/ionic/vulnerabilities"
)

// Results is an Ion Channel representation of the results from a
// scan summary.  It contains what type of results and the data pertaining to
// the results.
type Results struct {
	Type string      `json:"type" xml:"type"`
	Data interface{} `json:"data,omitempty" xml:"data,omitempty"`
}

type results struct {
	Type    string          `json:"type"`
	RawData json.RawMessage `json:"data"`
}

// UnmarshalJSON is a custom JSON unmarshaller implementation for the standard
// go json package to know how to properly interpret ScanSummaryResults from
// JSON.
func (r *Results) UnmarshalJSON(b []byte) error {
	var tr results
	err := json.Unmarshal(b, &tr)
	if err != nil {
		return err
	}

	r.Type = tr.Type

	switch strings.ToLower(tr.Type) {
	case "about_yml":
		var a AboutYMLResults
		err := json.Unmarshal(tr.RawData, &a)
		if err != nil {
			return fmt.Errorf("failed to unmarshall about yml results: %v", err)
		}

		r.Data = a
	case "community":
		var c []CommunityResults
		err := json.Unmarshal(tr.RawData, &c)
		if err != nil {
			return fmt.Errorf("failed to unmarshall community results: %v", err)
		}

		r.Data = c
	case "coverage", "external_coverage":
		var c CoverageResults
		err := json.Unmarshal(tr.RawData, &c)
		if err != nil {
			return fmt.Errorf("failed to unmarshall coverage results: %v", err)
		}

		r.Data = c
	case "dependency":
		var d DependencyResults
		err := json.Unmarshal(tr.RawData, &d)
		if err != nil {
			return fmt.Errorf("failed to unmarshall dependency results: %v", err)
		}

		r.Data = d
	case "ecosystems":
		var e EcosystemResults
		err := json.Unmarshal(tr.RawData, &e)
		if err != nil {
			return fmt.Errorf("failed to unmarshall ecosystems results: %v", err)
		}

		r.Data = e
	case "license":
		var l LicenseResults
		err := json.Unmarshal(tr.RawData, &l)
		if err != nil {
			return fmt.Errorf("failed to unmarshall license results: %v", err)
		}

		r.Data = l
	case "virus", "clamav":
		var v VirusResults
		err := json.Unmarshal(tr.RawData, &v)
		if err != nil {
			return fmt.Errorf("failed to unmarshall virus results: %v", err)
		}

		r.Data = v
	case "vulnerability":
		var v VulnerabilityResults
		err := json.Unmarshal(tr.RawData, &v)
		if err != nil {
			return fmt.Errorf("failed to unmarshall vulnerability results: %v", err)
		}

		r.Data = v
	case "external_vulnerability":
		var v ExternalVulnerabilitiesResults
		err := json.Unmarshal(tr.RawData, &v)
		if err != nil {
			return fmt.Errorf("failed to unmarshall external vulnerabilities results: %v", err)
		}

		r.Data = v
	case "difference":
		var v DifferenceResults
		err := json.Unmarshal(tr.RawData, &v)
		if err != nil {
			return fmt.Errorf("failed to unmarshall difference results: %v", err)
		}

		r.Data = v
	default:
		return fmt.Errorf("unsupported results type found: %v", tr.Type)
	}

	return nil
}

// AboutYMLResults represents the data collected from the AboutYML scan.  It
// includes a message and whether or not the About YML file found was valid or
// not.
type AboutYMLResults struct {
	Message string `json:"message" xml:"message"`
	Valid   bool   `json:"valid" xml:"valid"`
	Content string `json:"content" xml:"content"`
}

// CommunityResults represents the data collected from a community scan.  It
// represents all known data regarding the open community of a software project
type CommunityResults struct {
	Committers int    `json:"committers" xml:"committers"`
	Name       string `json:"name" xml:"name"`
	URL        string `json:"url" xml:"url"`
}

// CoverageResults represents the data collected from a code coverage scan.  It
// includes the value of the code coverage seen for the project.
type CoverageResults struct {
	Value float64 `json:"value" xml:"value"`
}

// Dependency represents data for an individual requirement resolution
type Dependency struct {
	LatestVersion string `json:"latest_version" xml:"latest_version"`
	Org           string `json:"org" xml:"org"`
	Name          string `json:"name" xml:"name"`
	Type          string `json:"type" xml:"type"`
	Package       string `json:"package" xml:"package"`
	Version       string `json:"version" xml:"version"`
	Scope         string `json:"scope" xml:"scope"`
}

// DependencyMeta represents data for a summary of all dependencies resolved
type DependencyMeta struct {
	FirstDegreeCount     int `json:"first_degree_count" xml:"first_degree_count"`
	NoVersionCount       int `json:"no_version_count" xml:"no_version_count"`
	TotalUniqueCount     int `json:"total_unique_count" xml:"total_unique_count"`
	UpdateAvailableCount int `json:"update_available_count" xml:"update_available_count"`
}

// DependencyResults represents the data collected from a dependency scan.  It
// includes a list of the dependencies seen and meta data counts about those
// dependencies seen.
type DependencyResults struct {
	Dependencies []Dependency   `json:"dependencies" xml:"dependencies"`
	Meta         DependencyMeta `json:"meta" xml:"meta"`
}

// DifferenceResults represents the checksum of a project.  It includes a checksum
// and flag indicating if there was a difference detected within that last 5 scans
type DifferenceResults struct {
	Checksum   string `json:"checksum" xml:"checksum"`
	Difference bool   `json:"difference" xml:"difference"`
}

// EcosystemResults represents the data collected from an ecosystems scan.  It
// include the name of the ecosystem and the number of lines seen for the given
// ecosystem.
type EcosystemResults struct {
	Ecosystems []struct {
		Ecosystem string `json:"ecosystem" xml:"ecosystem"`
		Lines     int    `json:"lines" xml:"lines"`
	} `json:"ecosystems" xml:"ecosystems"`
}

// ExternalVulnerabilitiesResults represents the data collected from an external
// vulnerability scan.  It includes the number of each vulnerability criticality
// seen within the project.
type ExternalVulnerabilitiesResults struct {
	Critical int `json:"critical" xml:"critical"`
	High     int `json:"high" xml:"high"`
	Medium   int `json:"medium" xml:"medium"`
	Low      int `json:"low" xml:"low"`
}

// LicenseResults represents the data collected from a license scan.  It
// includes the name and type of each license seen within the project.
type LicenseResults struct {
	License struct {
		Name string `json:"name" xml:"name"`
		Type []struct {
			Name string `json:"name" xml:"name"`
		} `json:"type" xml:"type"`
	} `json:"license" xml:"license"`
}

// VirusResults represents the data collected from a virus scan.  It includes
// information of the viruses seen and the virus scanner used.
type VirusResults struct {
	KnownViruses       int    `json:"known_viruses" xml:"known_viruses"`
	EngineVersion      string `json:"engine_version" xml:"engine_version"`
	ScannedDirectories int    `json:"scanned_directories" xml:"scanned_directories"`
	ScannedFiles       int    `json:"scanned_files" xml:"scanned_files"`
	InfectedFiles      int    `json:"infected_files" xml:"infected_files"`
	DataScanned        string `json:"data_scanned" xml:"data_scanned"`
	DataRead           string `json:"data_read" xml:"data_read"`
	Time               string `json:"time" xml:"time"`
	FileNotes          struct {
		EmptyFiles []string `json:"empty_file" xml:"empty_file"`
	} `json:"file_notes" xml:"file_notes"`
	ClamavDetails struct {
		ClamavVersion   string `json:"clamav_version" xml:"clamav_version"`
		ClamavDbVersion string `json:"clamav_db_version" xml:"clamav_db_version"`
	} `json:"clam_av_details" xml:"clam_av_details"`
}

//VulnerabilityResults represents the data collected from a vulnerability scan.  It includes
// information of the vulnerabilities seen.
type VulnerabilityResults struct {
	Vulnerabilities []struct {
		ID              int                             `json:"id" xml:"id"`
		ExternalID      string                          `json:"external_id" xml:"external_id"`
		SourceID        int                             `json:"source_id" xml:"source_id"`
		Title           string                          `json:"title" xml:"title"`
		Name            string                          `json:"name" xml:"name"`
		Org             string                          `json:"org" xml:"org"`
		Version         string                          `json:"version" xml:"version"`
		Up              interface{}                     `json:"up" xml:"up"`
		Edition         interface{}                     `json:"edition" xml:"edition"`
		Aliases         []string                        `json:"aliases" xml:"aliases"`
		CreatedAt       time.Time                       `json:"created_at" xml:"created_at"`
		UpdatedAt       time.Time                       `json:"updated_at" xml:"updated_at"`
		References      interface{}                     `json:"references" xml:"references"`
		Part            interface{}                     `json:"part" xml:"part"`
		Language        interface{}                     `json:"language" xml:"language"`
		Vulnerabilities []vulnerabilities.Vulnerability `json:"vulnerabilities" xml:"vulnerabilities"`
	} `json:"vulnerabilities" xml:"vulnerabilities"`
	Meta struct {
		VulnerabilityCount int `json:"vulnerability_count" xml:"vulnerability_count"`
	} `json:"meta" xml:"meta"`
}
<|endoftext|>


<commit_before>package scoring

import (
	"fmt"
	"math"
	"time"
)

// A point of reference Score.Update and Score.Relevance use to reference the
// current time. It is used in testing, so we always have the same current
// time. This is okay for this programs as it won't run for long.
var Now time.Time

// Represents a weight of a score and the age of it.
type Score struct {
	Weight int64
	Age    time.Time
}

// Update the weight and age of the current score.
func (s *Score) Update() {
	s.Weight++
	s.Age = Now
}

// Relevance of a score is the difference between the current time and when the
// score was last updated.
func (s *Score) Relevance() time.Duration {
	return Now.Sub(s.Age)
}

// Calculate the final score from the score weight and the age.
func (s *Score) Calculate() float64 {
	return float64(s.Weight) * math.Sqrt(float64(s.Relevance()))
}

// Calculate the final score from the score weight and the age.
func (s *Score) String() string {
	return fmt.Sprintf("{%s %s}", s.Weight, s.Age)
}

// Create a new score object with default weight of 1 and age set to now.
func NewScore() *Score {
	return &Score{1, Now}
}

func init() {
	Now = time.Now()
}
<commit_msg>Let the relevance grow even slower<commit_after>package scoring

import (
	"fmt"
	"math"
	"time"
)

// A point of reference Score.Update and Score.Relevance use to reference the
// current time. It is used in testing, so we always have the same current
// time. This is okay for this programs as it won't run for long.
var Now time.Time

// Represents a weight of a score and the age of it.
type Score struct {
	Weight int64
	Age    time.Time
}

// Update the weight and age of the current score.
func (s *Score) Update() {
	s.Weight++
	s.Age = Now
}

// Relevance of a score is the difference between the current time and when the
// score was last updated.
func (s *Score) Relevance() time.Duration {
	return Now.Sub(s.Age)
}

// Calculate the final score from the score weight and the age.
func (s *Score) Calculate() float64 {
	return float64(s.Weight) * math.Log(float64(s.Relevance()))
}

// Calculate the final score from the score weight and the age.
func (s *Score) String() string {
	return fmt.Sprintf("{%s %s}", s.Weight, s.Age)
}

// Create a new score object with default weight of 1 and age set to now.
func NewScore() *Score {
	return &Score{1, Now}
}

func init() {
	Now = time.Now()
}
<|endoftext|>


<commit_before>// Copyright 2012-2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package relation

import (
	"github.com/juju/juju/worker/uniter/hook"
)

// HookQueue exists to keep the package interface stable.
type HookQueue interface {
	HookSender
}

// NewAliveHookQueue returns a new HookQueue that aggregates the values
// obtained from the w watcher and sends into out the details about hooks that
// must be executed in the unit. It guarantees that the stream of hooks will
// respect the guarantees Juju makes about hook execution order. If any values
// have previously been received from w's Changes channel, the HookQueue's
// behaviour is undefined.
func NewAliveHookQueue(initial *State, out chan<- hook.Info, w RelationUnitsWatcher) HookQueue {
	source := NewLiveHookSource(initial, w)
	return NewHookSender(out, source)
}

// NewDyingHookQueue returns a new HookQueue that sends all hooks necessary
// to clean up the supplied initial relation hook state, while preserving the
// guarantees Juju makes about hook execution order.
func NewDyingHookQueue(initial *State, out chan<- hook.Info) HookQueue {
	source := NewDyingHookSource(initial)
	return NewHookSender(out, source)
}
<commit_msg>more docstrings<commit_after>// Copyright 2012-2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package relation

import (
	"github.com/juju/juju/worker/uniter/hook"
)

// HookQueue exists to keep the package interface stable.
type HookQueue interface {
	HookSender
}

// NewAliveHookQueue exists to keep the package interface stable; it wraps the
// result of NewLiveHookSource in a HookSender.
func NewAliveHookQueue(initial *State, out chan<- hook.Info, w RelationUnitsWatcher) HookQueue {
	source := NewLiveHookSource(initial, w)
	return NewHookSender(out, source)
}

// NewDyingHookQueue exists to keep the package interface stable; it wraps the
// result of NewDyingHookSource in a HookSender.
func NewDyingHookQueue(initial *State, out chan<- hook.Info) HookQueue {
	source := NewDyingHookSource(initial)
	return NewHookSender(out, source)
}
<|endoftext|>


<commit_before>package storage

import (
	"crypto/tls"
	"os"
	"sync"
	"time"

	"github.com/khlieng/dispatch/pkg/irc"
	"github.com/kjk/betterguid"
)

type User struct {
	ID       uint64
	IDBytes  []byte
	Username string

	store          Store
	messageLog     MessageStore
	messageIndex   MessageSearchProvider
	lastMessages   map[string]map[string]*Message
	clientSettings *ClientSettings
	lastIP         []byte
	certificate    *tls.Certificate
	lock           sync.Mutex
}

func NewUser(store Store) (*User, error) {
	user := &User{
		store:          store,
		clientSettings: DefaultClientSettings(),
		lastMessages:   map[string]map[string]*Message{},
	}

	err := store.SaveUser(user)
	if err != nil {
		return nil, err
	}

	err = os.MkdirAll(Path.User(user.Username), 0700)
	if err != nil {
		return nil, err
	}
	err = os.Mkdir(Path.Downloads(user.Username), 0700)
	if err != nil {
		return nil, err
	}

	user.messageLog, err = GetMessageStore(user)
	if err != nil {
		return nil, err
	}
	user.messageIndex, err = GetMessageSearchProvider(user)
	if err != nil {
		return nil, err
	}

	return user, nil
}

func LoadUsers(store Store) ([]*User, error) {
	users, err := store.Users()
	if err != nil {
		return nil, err
	}

	for _, user := range users {
		user.store = store
		user.messageLog, err = GetMessageStore(user)
		if err != nil {
			return nil, err
		}
		user.messageIndex, err = GetMessageSearchProvider(user)
		if err != nil {
			return nil, err
		}
		user.lastMessages = map[string]map[string]*Message{}
		user.loadCertificate()

		channels, err := user.Channels()
		if err != nil {
			return nil, err
		}

		for _, channel := range channels {
			messages, _, err := user.LastMessages(channel.Network, channel.Name, 1)
			if err == nil && len(messages) == 1 {
				user.lastMessages[channel.Network] = map[string]*Message{
					channel.Name: &messages[0],
				}
			}
		}
	}

	return users, nil
}

func (u *User) Remove() {
	u.store.DeleteUser(u)
	if u.messageLog != nil {
		u.messageLog.Close()
	}
	if u.messageIndex != nil {
		u.messageIndex.Close()
	}
	os.RemoveAll(Path.User(u.Username))
}

func (u *User) GetLastIP() []byte {
	u.lock.Lock()
	ip := u.lastIP
	u.lock.Unlock()
	return ip
}

func (u *User) SetLastIP(ip []byte) error {
	u.lock.Lock()
	u.lastIP = ip
	u.lock.Unlock()

	return u.store.SaveUser(u)
}

//easyjson:json
type ClientSettings struct {
	ColoredNicks bool
}

func DefaultClientSettings() *ClientSettings {
	return &ClientSettings{
		ColoredNicks: true,
	}
}

func (u *User) ClientSettings() *ClientSettings {
	u.lock.Lock()
	settings := *u.clientSettings
	u.lock.Unlock()
	return &settings
}

func (u *User) SetClientSettings(settings *ClientSettings) error {
	u.lock.Lock()
	u.clientSettings = settings
	u.lock.Unlock()

	return u.store.SaveUser(u)
}

func (u *User) UnmarshalClientSettingsJSON(b []byte) error {
	u.lock.Lock()
	err := u.clientSettings.UnmarshalJSON(b)
	u.lock.Unlock()

	if err != nil {
		return err
	}

	return u.store.SaveUser(u)
}

func (u *User) NewNetwork(template *Network, client *irc.Client) *Network {
	if template == nil {
		template = &Network{}
	}

	template.user = u
	template.client = client
	template.channels = map[string]*Channel{}
	template.lock = &sync.Mutex{}

	return template
}

func (u *User) Network(address string) (*Network, error) {
	return u.store.Network(u, address)
}

func (u *User) Networks() ([]*Network, error) {
	return u.store.Networks(u)
}

func (u *User) SaveNetwork(network *Network) error {
	return u.store.SaveNetwork(u, network)
}

func (u *User) RemoveNetwork(address string) error {
	return u.store.RemoveNetwork(u, address)
}

func (u *User) SetNick(nick, address string) error {
	network, err := u.Network(address)
	if err != nil {
		return err
	}
	network.Nick = nick
	return u.SaveNetwork(network)
}

func (u *User) SetNetworkName(name, address string) error {
	network, err := u.Network(address)
	if err != nil {
		return err
	}
	network.Name = name
	return u.SaveNetwork(network)
}

func (u *User) Channels() ([]*Channel, error) {
	return u.store.Channels(u)
}

func (u *User) SaveChannel(channel *Channel) error {
	return u.store.SaveChannel(u, channel)
}

func (u *User) RemoveChannel(network, channel string) error {
	return u.store.RemoveChannel(u, network, channel)
}

func (u *User) HasChannel(network, channel string) bool {
	return u.store.HasChannel(u, network, channel)
}

type Tab struct {
	Network string
	Name    string
}

func (u *User) OpenDMs() ([]Tab, error) {
	return u.store.OpenDMs(u)
}

func (u *User) AddOpenDM(network, nick string) error {
	return u.store.AddOpenDM(u, network, nick)
}

func (u *User) RemoveOpenDM(network, nick string) error {
	return u.store.RemoveOpenDM(u, network, nick)
}

type Message struct {
	ID      string  `json:"-" bleve:"-"`
	Network string  `json:"-" bleve:"server"`
	From    string  `bleve:"-"`
	To      string  `json:"-" bleve:"to"`
	Content string  `bleve:"content"`
	Time    int64   `bleve:"-"`
	Events  []Event `bleve:"-"`
}

func (m Message) Type() string {
	return "message"
}

func (u *User) LogMessage(msg *Message) error {
	if msg.Time == 0 {
		msg.Time = time.Now().Unix()
	}

	if msg.ID == "" {
		msg.ID = betterguid.New()
	}

	if msg.To == "" {
		msg.To = msg.From
	}

	u.setLastMessage(msg.Network, msg.To, msg)

	err := u.messageLog.LogMessage(msg)
	if err != nil {
		return err
	}
	return u.messageIndex.Index(msg.ID, msg)
}

type Event struct {
	Type   string
	Params []string
	Time   int64
}

func (u *User) LogEvent(network, name string, params []string, channels ...string) error {
	now := time.Now().Unix()
	event := Event{
		Type:   name,
		Params: params,
		Time:   now,
	}

	for _, channel := range channels {
		lastMessage := u.getLastMessage(network, channel)

		if lastMessage != nil && shouldCollapse(lastMessage, event) {
			lastMessage.Events = append(lastMessage.Events, event)
			u.setLastMessage(network, channel, lastMessage)

			err := u.messageLog.LogMessage(lastMessage)
			if err != nil {
				return err
			}
		} else {
			msg := &Message{
				ID:      betterguid.New(),
				Network: network,
				To:      channel,
				Time:    now,
				Events:  []Event{event},
			}
			u.setLastMessage(network, channel, msg)

			err := u.messageLog.LogMessage(msg)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

var collapsed = []string{"join", "part", "quit"}

func shouldCollapse(msg *Message, event Event) bool {
	matches := 0
	if len(msg.Events) > 0 {
		for _, collapseType := range collapsed {
			if msg.Events[0].Type == collapseType {
				matches++
			}
			if event.Type == collapseType {
				matches++
			}
		}
	}
	return matches == 2
}

func (u *User) getLastMessage(network, channel string) *Message {
	u.lock.Lock()
	defer u.lock.Unlock()

	if _, ok := u.lastMessages[network]; !ok {
		return nil
	}

	last := u.lastMessages[network][channel]
	if last != nil {
		msg := *last
		return &msg
	}
	return nil
}

func (u *User) setLastMessage(network, channel string, msg *Message) {
	u.lock.Lock()

	if _, ok := u.lastMessages[network]; !ok {
		u.lastMessages[network] = map[string]*Message{}
	}

	u.lastMessages[network][channel] = msg
	u.lock.Unlock()
}

func (u *User) Messages(network, channel string, count int, fromID string) ([]Message, bool, error) {
	return u.messageLog.Messages(network, channel, count, fromID)
}

func (u *User) LastMessages(network, channel string, count int) ([]Message, bool, error) {
	return u.Messages(network, channel, count, "")
}

func (u *User) SearchMessages(network, channel, q string) ([]Message, error) {
	ids, err := u.messageIndex.SearchMessages(network, channel, q)
	if err != nil {
		return nil, err
	}

	return u.messageLog.MessagesByID(network, channel, ids)
}
<commit_msg>Fix lastMessages map init in LoadUsers()<commit_after>package storage

import (
	"crypto/tls"
	"os"
	"sync"
	"time"

	"github.com/khlieng/dispatch/pkg/irc"
	"github.com/kjk/betterguid"
)

type User struct {
	ID       uint64
	IDBytes  []byte
	Username string

	store          Store
	messageLog     MessageStore
	messageIndex   MessageSearchProvider
	lastMessages   map[string]map[string]*Message
	clientSettings *ClientSettings
	lastIP         []byte
	certificate    *tls.Certificate
	lock           sync.Mutex
}

func NewUser(store Store) (*User, error) {
	user := &User{
		store:          store,
		clientSettings: DefaultClientSettings(),
		lastMessages:   map[string]map[string]*Message{},
	}

	err := store.SaveUser(user)
	if err != nil {
		return nil, err
	}

	err = os.MkdirAll(Path.User(user.Username), 0700)
	if err != nil {
		return nil, err
	}
	err = os.Mkdir(Path.Downloads(user.Username), 0700)
	if err != nil {
		return nil, err
	}

	user.messageLog, err = GetMessageStore(user)
	if err != nil {
		return nil, err
	}
	user.messageIndex, err = GetMessageSearchProvider(user)
	if err != nil {
		return nil, err
	}

	return user, nil
}

func LoadUsers(store Store) ([]*User, error) {
	users, err := store.Users()
	if err != nil {
		return nil, err
	}

	for _, user := range users {
		user.store = store
		user.messageLog, err = GetMessageStore(user)
		if err != nil {
			return nil, err
		}
		user.messageIndex, err = GetMessageSearchProvider(user)
		if err != nil {
			return nil, err
		}
		user.lastMessages = map[string]map[string]*Message{}
		user.loadCertificate()

		channels, err := user.Channels()
		if err != nil {
			return nil, err
		}

		for _, channel := range channels {
			messages, _, err := user.LastMessages(channel.Network, channel.Name, 1)
			if err == nil && len(messages) == 1 {
				if _, ok := user.lastMessages[channel.Network]; !ok {
					user.lastMessages[channel.Network] = map[string]*Message{}
				}

				user.lastMessages[channel.Network][channel.Name] = &messages[0]
			}
		}
	}

	return users, nil
}

func (u *User) Remove() {
	u.store.DeleteUser(u)
	if u.messageLog != nil {
		u.messageLog.Close()
	}
	if u.messageIndex != nil {
		u.messageIndex.Close()
	}
	os.RemoveAll(Path.User(u.Username))
}

func (u *User) GetLastIP() []byte {
	u.lock.Lock()
	ip := u.lastIP
	u.lock.Unlock()
	return ip
}

func (u *User) SetLastIP(ip []byte) error {
	u.lock.Lock()
	u.lastIP = ip
	u.lock.Unlock()

	return u.store.SaveUser(u)
}

//easyjson:json
type ClientSettings struct {
	ColoredNicks bool
}

func DefaultClientSettings() *ClientSettings {
	return &ClientSettings{
		ColoredNicks: true,
	}
}

func (u *User) ClientSettings() *ClientSettings {
	u.lock.Lock()
	settings := *u.clientSettings
	u.lock.Unlock()
	return &settings
}

func (u *User) SetClientSettings(settings *ClientSettings) error {
	u.lock.Lock()
	u.clientSettings = settings
	u.lock.Unlock()

	return u.store.SaveUser(u)
}

func (u *User) UnmarshalClientSettingsJSON(b []byte) error {
	u.lock.Lock()
	err := u.clientSettings.UnmarshalJSON(b)
	u.lock.Unlock()

	if err != nil {
		return err
	}

	return u.store.SaveUser(u)
}

func (u *User) NewNetwork(template *Network, client *irc.Client) *Network {
	if template == nil {
		template = &Network{}
	}

	template.user = u
	template.client = client
	template.channels = map[string]*Channel{}
	template.lock = &sync.Mutex{}

	return template
}

func (u *User) Network(address string) (*Network, error) {
	return u.store.Network(u, address)
}

func (u *User) Networks() ([]*Network, error) {
	return u.store.Networks(u)
}

func (u *User) SaveNetwork(network *Network) error {
	return u.store.SaveNetwork(u, network)
}

func (u *User) RemoveNetwork(address string) error {
	return u.store.RemoveNetwork(u, address)
}

func (u *User) SetNick(nick, address string) error {
	network, err := u.Network(address)
	if err != nil {
		return err
	}
	network.Nick = nick
	return u.SaveNetwork(network)
}

func (u *User) SetNetworkName(name, address string) error {
	network, err := u.Network(address)
	if err != nil {
		return err
	}
	network.Name = name
	return u.SaveNetwork(network)
}

func (u *User) Channels() ([]*Channel, error) {
	return u.store.Channels(u)
}

func (u *User) SaveChannel(channel *Channel) error {
	return u.store.SaveChannel(u, channel)
}

func (u *User) RemoveChannel(network, channel string) error {
	return u.store.RemoveChannel(u, network, channel)
}

func (u *User) HasChannel(network, channel string) bool {
	return u.store.HasChannel(u, network, channel)
}

type Tab struct {
	Network string
	Name    string
}

func (u *User) OpenDMs() ([]Tab, error) {
	return u.store.OpenDMs(u)
}

func (u *User) AddOpenDM(network, nick string) error {
	return u.store.AddOpenDM(u, network, nick)
}

func (u *User) RemoveOpenDM(network, nick string) error {
	return u.store.RemoveOpenDM(u, network, nick)
}

type Message struct {
	ID      string  `json:"-" bleve:"-"`
	Network string  `json:"-" bleve:"server"`
	From    string  `bleve:"-"`
	To      string  `json:"-" bleve:"to"`
	Content string  `bleve:"content"`
	Time    int64   `bleve:"-"`
	Events  []Event `bleve:"-"`
}

func (m Message) Type() string {
	return "message"
}

func (u *User) LogMessage(msg *Message) error {
	if msg.Time == 0 {
		msg.Time = time.Now().Unix()
	}

	if msg.ID == "" {
		msg.ID = betterguid.New()
	}

	if msg.To == "" {
		msg.To = msg.From
	}

	u.setLastMessage(msg.Network, msg.To, msg)

	err := u.messageLog.LogMessage(msg)
	if err != nil {
		return err
	}
	return u.messageIndex.Index(msg.ID, msg)
}

type Event struct {
	Type   string
	Params []string
	Time   int64
}

func (u *User) LogEvent(network, name string, params []string, channels ...string) error {
	now := time.Now().Unix()
	event := Event{
		Type:   name,
		Params: params,
		Time:   now,
	}

	for _, channel := range channels {
		lastMessage := u.getLastMessage(network, channel)

		if lastMessage != nil && shouldCollapse(lastMessage, event) {
			lastMessage.Events = append(lastMessage.Events, event)
			u.setLastMessage(network, channel, lastMessage)

			err := u.messageLog.LogMessage(lastMessage)
			if err != nil {
				return err
			}
		} else {
			msg := &Message{
				ID:      betterguid.New(),
				Network: network,
				To:      channel,
				Time:    now,
				Events:  []Event{event},
			}
			u.setLastMessage(network, channel, msg)

			err := u.messageLog.LogMessage(msg)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

var collapsed = []string{"join", "part", "quit"}

func shouldCollapse(msg *Message, event Event) bool {
	matches := 0
	if len(msg.Events) > 0 {
		for _, collapseType := range collapsed {
			if msg.Events[0].Type == collapseType {
				matches++
			}
			if event.Type == collapseType {
				matches++
			}
		}
	}
	return matches == 2
}

func (u *User) getLastMessage(network, channel string) *Message {
	u.lock.Lock()
	defer u.lock.Unlock()

	if _, ok := u.lastMessages[network]; !ok {
		return nil
	}

	last := u.lastMessages[network][channel]
	if last != nil {
		msg := *last
		return &msg
	}
	return nil
}

func (u *User) setLastMessage(network, channel string, msg *Message) {
	u.lock.Lock()

	if _, ok := u.lastMessages[network]; !ok {
		u.lastMessages[network] = map[string]*Message{}
	}

	u.lastMessages[network][channel] = msg
	u.lock.Unlock()
}

func (u *User) Messages(network, channel string, count int, fromID string) ([]Message, bool, error) {
	return u.messageLog.Messages(network, channel, count, fromID)
}

func (u *User) LastMessages(network, channel string, count int) ([]Message, bool, error) {
	return u.Messages(network, channel, count, "")
}

func (u *User) SearchMessages(network, channel, q string) ([]Message, error) {
	ids, err := u.messageIndex.SearchMessages(network, channel, q)
	if err != nil {
		return nil, err
	}

	return u.messageLog.MessagesByID(network, channel, ids)
}
<|endoftext|>


<commit_before>package executordispatch

import (
	"path/filepath"

	"polydawn.net/repeatr/def"
	"polydawn.net/repeatr/executor"
	"polydawn.net/repeatr/executor/chroot"
	"polydawn.net/repeatr/executor/nsinit"
	"polydawn.net/repeatr/executor/null"
)

// TODO: This should not require a global string -> class map :|
// Should attempt to reflect-find, trying main package name first.
// Will make simpler to use extended transports, etc.

func Get(desire string) executor.Executor {
	var executor executor.Executor

	switch desire {
	case "null":
		executor = &null.Executor{}
	case "nsinit":
		executor = &nsinit.Executor{}
	case "chroot":
		executor = &chroot.Executor{}
	default:
		panic(def.ValidationError.New("No such executor %s", desire))
	}

	// Set the base path to operate from
	executor.Configure(filepath.Join(def.Base(), "executor", desire))

	return executor
}
<commit_msg>Add runc to usable executor options!<commit_after>package executordispatch

import (
	"path/filepath"

	"polydawn.net/repeatr/def"
	"polydawn.net/repeatr/executor"
	"polydawn.net/repeatr/executor/chroot"
	"polydawn.net/repeatr/executor/nsinit"
	"polydawn.net/repeatr/executor/null"
	"polydawn.net/repeatr/executor/runc"
)

// TODO: This should not require a global string -> class map :|
// Should attempt to reflect-find, trying main package name first.
// Will make simpler to use extended transports, etc.

func Get(desire string) executor.Executor {
	var executor executor.Executor

	switch desire {
	case "null":
		executor = &null.Executor{}
	case "nsinit":
		executor = &nsinit.Executor{}
	case "chroot":
		executor = &chroot.Executor{}
	case "runc":
		executor = &runc.Executor{}
	default:
		panic(def.ValidationError.New("No such executor %s", desire))
	}

	// Set the base path to operate from
	executor.Configure(filepath.Join(def.Base(), "executor", desire))

	return executor
}
<|endoftext|>


<commit_before>package runner

import (
	"context"
	"fmt"
	"log"
	"sync"
	"time"

	"github.com/grafana/metrictank/cmd/mt-simulate-lock-congestion/metricname"
	"github.com/grafana/metrictank/cmd/mt-simulate-lock-congestion/query"

	"github.com/grafana/metrictank/idx"
	"github.com/grafana/metrictank/idx/memory"
	"github.com/raintank/schema"
	"golang.org/x/sync/errgroup"
)

// TestRun represents one test run, including all the structures used for the test
type TestRun struct {
	index                 idx.MetricIndex
	metricNameGenerator   metricname.NameGenerator
	queryPatternGenerator query.QueryGenerator
	stats                 runStats
	addsPerSec            uint32 // how many adds per second we want to execute. 0 means unlimited, as many as possible
	addThreads            uint32 // number of concurrent add threads
	addSampleFactor       uint32 // how often to print info about a completed add
	addDelay              uint32 // once the benchmark starts the querying threads start, if addDelay is >0 then the adding into the index is delayed by the given number of seconds
	initialIndexSize      uint32 // prepopulate the index with the defined number of entries before starting the actual test run
	queriesPerSec         uint32 // how many queries per second we want to execute. 0 means unlimited, as many as possible
	querySampleFactor     uint32 // how often to print info about a completed query
	totalQueryCount       uint32 // the total number of queries we will start
	runDuration           time.Duration
}

const orgID = 1

// NewTestRun Instantiates a new test run
func NewTestRun(nameGenerator metricname.NameGenerator, queryGenerator query.QueryGenerator, addDelay, addsPerSec, addThreads, addSampleFactor, initialIndexSize, queriesPerSec, querySampleFactor uint32, runDuration time.Duration) *TestRun {
	totalQueryCount := (queriesPerSec + addDelay) * uint32(runDuration.Seconds())

	runner := TestRun{
		stats:                 newRunStats(totalQueryCount),
		totalQueryCount:       totalQueryCount,
		index:                 memory.New(),
		metricNameGenerator:   nameGenerator,
		queryPatternGenerator: queryGenerator,
		addsPerSec:            addsPerSec,
		addThreads:            addThreads,
		addSampleFactor:       addSampleFactor,
		addDelay:              addDelay,
		initialIndexSize:      initialIndexSize,
		queriesPerSec:         queriesPerSec,
		querySampleFactor:     querySampleFactor,
		runDuration:           runDuration,
	}

	return &runner
}

// Run executes the run
func (t *TestRun) Run() {
	mainCtx, cancel := context.WithCancel(context.Background())

	log.Printf("Starting the metric name generator")
	t.metricNameGenerator.Start(mainCtx, t.addThreads)

	log.Printf("Starting the query generator")
	t.queryPatternGenerator.Start()

	log.Printf("Prepopulating the index with %d entries", t.initialIndexSize)
	t.prepopulateIndex()

	// wait group to wait until all routines have been started
	startedWg := sync.WaitGroup{}
	workerThreads, workerCtx := errgroup.WithContext(mainCtx)

	startedWg.Add(1)
	workerThreads.Go(t.queryRoutine(workerCtx, &startedWg))

	<-time.After(time.Second * time.Duration(t.addDelay))

	startedWg.Add(int(t.addThreads))
	for i := uint32(0); i < t.addThreads; i++ {
		workerThreads.Go(t.addRoutine(workerCtx, &startedWg, i))
	}
	startedWg.Wait()
	log.Printf("Benchmark has started")

	// as soon as all routines have started, we start the timer
	<-time.After(t.runDuration)
	cancel()
	workerThreads.Wait()
}

// PrintStats writes all the statistics in human readable format into stdout
func (t *TestRun) PrintStats() {
	t.stats.Print(uint32(t.runDuration.Seconds()))
}

func (t *TestRun) prepopulateIndex() {
	for i := uint32(0); i < t.initialIndexSize; i++ {
		md := schema.MetricData{OrgId: orgID, Interval: 1, Value: 2, Time: int64(time.Now().Unix()), Mtype: "gauge"}
		md.Name = t.metricNameGenerator.GetNewMetricName()
		md.SetId()
		key, err := schema.MKeyFromString(md.Id)
		if err != nil {
			log.Fatalf("Unexpected error when generating MKey from ID string: %s", err)
		}

		t.index.AddOrUpdate(key, &md, int32(i%t.addThreads))
	}
}

func (t *TestRun) addRoutine(ctx context.Context, startedWg *sync.WaitGroup, routineID uint32) func() error {
	partitionID := int32(routineID)

	addEntryToIndex := func(time int) error {
		md := schema.MetricData{OrgId: orgID, Interval: 1, Value: 2, Time: int64(time), Mtype: "gauge"}
		md.Name = t.metricNameGenerator.GetNewMetricName()
		md.SetId()
		key, err := schema.MKeyFromString(md.Id)
		if err != nil {
			return fmt.Errorf("Unexpected error when generating MKey from ID string: %s", err)
		}

		t.index.AddOrUpdate(key, &md, partitionID)
		adds := t.stats.incAddsCompleted()
		if adds%t.addSampleFactor == 0 {
			log.Printf("Sample: added metric name to index %s", md.Name)
		}

		return nil
	}

	if t.addsPerSec > 0 {
		interval := time.Duration(int64(1000000000) * int64(t.addThreads) / int64(t.addsPerSec))

		return func() error {
			ticker := time.NewTicker(interval)
			go func() {
				<-ctx.Done()
				ticker.Stop()
			}()

			startedWg.Done()
			for {
				select {
				case <-ctx.Done():
					return nil
				case tick := <-ticker.C:
					if err := addEntryToIndex(tick.Second()); err != nil {
						return err
					}
				}
			}
		}
	}

	return func() error {
		startedWg.Done()

		for i := 0; ; i++ {
			select {
			case <-ctx.Done():
				return nil
			default:
				if err := addEntryToIndex(i); err != nil {
					return err
				}
			}
		}
	}
}

func (t *TestRun) queryRoutine(ctx context.Context, startedWg *sync.WaitGroup) func() error {
	return func() error {
		ticker := time.NewTicker(time.Duration(int64(float64(1000000000) / float64(t.queriesPerSec))))
		startedWg.Done()
		for {
			select {
			case <-ctx.Done():
				return nil
			case <-ticker.C:
				go t.runQuery()
			}
		}
	}
}

func (t *TestRun) runQuery() {
	pattern := t.queryPatternGenerator.GetPattern()
	pre := time.Now()

	queryStartedID := t.stats.incQueriesStarted()
	if queryStartedID > t.totalQueryCount {
		return
	}

	res, err := t.index.Find(orgID, pattern, 0)
	if err != nil {
		log.Printf("Warning: Query failed with error: %s", err)
	}

	queryCompletedID := t.stats.incQueriesCompleted()
	t.stats.addQueryTime(queryCompletedID, time.Now().Sub(pre))
	if queryCompletedID%t.querySampleFactor == 0 {
		log.Printf("Sample: query got %d results for pattern %s", len(res), pattern)
	}
}
<commit_msg>use input handler to update index<commit_after>package runner

import (
	"context"
	"log"
	"sync"
	"time"

	"github.com/grafana/metrictank/cmd/mt-simulate-lock-congestion/metricname"
	"github.com/grafana/metrictank/cmd/mt-simulate-lock-congestion/query"
	"github.com/grafana/metrictank/conf"
	"github.com/grafana/metrictank/input"
	"github.com/grafana/metrictank/mdata"

	"github.com/grafana/metrictank/idx"
	"github.com/grafana/metrictank/idx/memory"
	"github.com/raintank/schema"
	"golang.org/x/sync/errgroup"
)

// TestRun represents one test run, including all the structures used for the test
type TestRun struct {
	index                 idx.MetricIndex
	handler               input.Handler
	metricNameGenerator   metricname.NameGenerator
	queryPatternGenerator query.QueryGenerator
	stats                 runStats
	addsPerSec            uint32 // how many adds per second we want to execute. 0 means unlimited, as many as possible
	addThreads            uint32 // number of concurrent add threads
	addSampleFactor       uint32 // how often to print info about a completed add
	addDelay              uint32 // once the benchmark starts the querying threads start, if addDelay is >0 then the adding into the index is delayed by the given number of seconds
	initialIndexSize      uint32 // prepopulate the index with the defined number of entries before starting the actual test run
	queriesPerSec         uint32 // how many queries per second we want to execute. 0 means unlimited, as many as possible
	querySampleFactor     uint32 // how often to print info about a completed query
	totalQueryCount       uint32 // the total number of queries we will start
	runDuration           time.Duration
}

const orgID = 1

// NewTestRun Instantiates a new test run
func NewTestRun(nameGenerator metricname.NameGenerator, queryGenerator query.QueryGenerator, addDelay, addsPerSec, addThreads, addSampleFactor, initialIndexSize, queriesPerSec, querySampleFactor uint32, runDuration time.Duration) *TestRun {
	totalQueryCount := (queriesPerSec + addDelay) * uint32(runDuration.Seconds())

	index := memory.New()
	// initializing with a `nil` store, that's a bit risky but good enough for the moment
	mdata.Schemas = conf.NewSchemas(nil)
	metrics := mdata.NewAggMetrics(nil, nil, false, 3600, 7200, 3600)
	handler := input.NewDefaultHandler(metrics, index, "lock-congestion-simulator")

	runner := TestRun{
		index:                 index,
		handler:               handler,
		stats:                 newRunStats(totalQueryCount),
		totalQueryCount:       totalQueryCount,
		metricNameGenerator:   nameGenerator,
		queryPatternGenerator: queryGenerator,
		addsPerSec:            addsPerSec,
		addThreads:            addThreads,
		addSampleFactor:       addSampleFactor,
		addDelay:              addDelay,
		initialIndexSize:      initialIndexSize,
		queriesPerSec:         queriesPerSec,
		querySampleFactor:     querySampleFactor,
		runDuration:           runDuration,
	}

	return &runner
}

// Run executes the run
func (t *TestRun) Run() {
	mainCtx, cancel := context.WithCancel(context.Background())

	log.Printf("Starting the metric name generator")
	t.metricNameGenerator.Start(mainCtx, t.addThreads)

	log.Printf("Starting the query generator")
	t.queryPatternGenerator.Start()

	log.Printf("Prepopulating the index with %d entries", t.initialIndexSize)
	t.prepopulateIndex()

	// wait group to wait until all routines have been started
	startedWg := sync.WaitGroup{}
	workerThreads, workerCtx := errgroup.WithContext(mainCtx)

	startedWg.Add(1)
	workerThreads.Go(t.queryRoutine(workerCtx, &startedWg))

	<-time.After(time.Second * time.Duration(t.addDelay))

	startedWg.Add(int(t.addThreads))
	for i := uint32(0); i < t.addThreads; i++ {
		workerThreads.Go(t.addRoutine(workerCtx, &startedWg, i))
	}
	startedWg.Wait()
	log.Printf("Benchmark has started")

	// as soon as all routines have started, we start the timer
	<-time.After(t.runDuration)
	cancel()
	workerThreads.Wait()
}

// PrintStats writes all the statistics in human readable format into stdout
func (t *TestRun) PrintStats() {
	t.stats.Print(uint32(t.runDuration.Seconds()))
}

func (t *TestRun) prepopulateIndex() {
	for i := uint32(0); i < t.initialIndexSize; i++ {
		md := schema.MetricData{OrgId: orgID, Interval: 1, Value: 2, Time: int64(time.Now().Unix()), Mtype: "gauge"}
		md.Name = t.metricNameGenerator.GetNewMetricName()
		md.SetId()
		key, err := schema.MKeyFromString(md.Id)
		if err != nil {
			log.Fatalf("Unexpected error when generating MKey from ID string: %s", err)
		}

		t.index.AddOrUpdate(key, &md, int32(i%t.addThreads))
	}
}

func (t *TestRun) addRoutine(ctx context.Context, startedWg *sync.WaitGroup, routineID uint32) func() error {
	partitionID := int32(routineID)

	addEntryToIndex := func(time int) error {
		md := schema.MetricData{OrgId: orgID, Interval: 1, Value: 2, Time: int64(time), Mtype: "gauge"}
		md.Name = t.metricNameGenerator.GetNewMetricName()
		md.SetId()
		// key, err := schema.MKeyFromString(md.Id)
		// if err != nil {
		// 	return fmt.Errorf("Unexpected error when generating MKey from ID string: %s", err)
		// }

		//t.index.AddOrUpdate(key, &md, partitionID)
		t.handler.ProcessMetricData(&md, partitionID)
		adds := t.stats.incAddsCompleted()
		if adds%t.addSampleFactor == 0 {
			log.Printf("Sample: added metric name to index %s", md.Name)
		}

		return nil
	}

	if t.addsPerSec > 0 {
		interval := time.Duration(int64(1000000000) * int64(t.addThreads) / int64(t.addsPerSec))

		return func() error {
			ticker := time.NewTicker(interval)
			go func() {
				<-ctx.Done()
				ticker.Stop()
			}()

			startedWg.Done()
			for {
				select {
				case <-ctx.Done():
					return nil
				case tick := <-ticker.C:
					if err := addEntryToIndex(tick.Second()); err != nil {
						return err
					}
				}
			}
		}
	}

	return func() error {
		startedWg.Done()

		for i := 0; ; i++ {
			select {
			case <-ctx.Done():
				return nil
			default:
				if err := addEntryToIndex(int(time.Now().Unix())); err != nil {
					return err
				}
			}
		}
	}
}

func (t *TestRun) queryRoutine(ctx context.Context, startedWg *sync.WaitGroup) func() error {
	return func() error {
		ticker := time.NewTicker(time.Duration(int64(float64(1000000000) / float64(t.queriesPerSec))))
		startedWg.Done()
		for {
			select {
			case <-ctx.Done():
				return nil
			case <-ticker.C:
				go t.runQuery()
			}
		}
	}
}

func (t *TestRun) runQuery() {
	pattern := t.queryPatternGenerator.GetPattern()
	pre := time.Now()

	queryStartedID := t.stats.incQueriesStarted()
	if queryStartedID > t.totalQueryCount {
		return
	}

	res, err := t.index.Find(orgID, pattern, 0)
	if err != nil {
		log.Printf("Warning: Query failed with error: %s", err)
	}

	queryCompletedID := t.stats.incQueriesCompleted()
	t.stats.addQueryTime(queryCompletedID, time.Now().Sub(pre))
	if queryCompletedID%t.querySampleFactor == 0 {
		log.Printf("Sample: query got %d results for pattern %s", len(res), pattern)
	}
}
<|endoftext|>


<commit_before>package signal

import (
    "os"
    S "os/signal"
)

// return true if need to break
type Callback func() bool

type Handler struct {
    schan chan os.Signal
    cb map[os.Signal]Callback
}

func NewHandler() (sh *Handler) {
    sh = &Handler{make(chan os.Signal, 1), make(map[os.Signal]Callback, 5)}
    S.Notify(sh.schan, os.Interrupt, os.Kill)
    return
}

func (sh *Handler)Bind(s os.Signal, cb Callback) {
    sh.cb[s] = cb
}

func (sh *Handler)Loop() os.Signal {
    for {
        s := <-sh.schan
        f, ok := sh.cb[s]
        if ok {
            f()
            return s
        }
    }
    return nil
}
<commit_msg>make sure it is a none-buffered channel<commit_after>package signal

import (
    "os"
    S "os/signal"
)

// return true if need to break
type Callback func() bool

type Handler struct {
    schan chan os.Signal
    cb map[os.Signal]Callback
}

func NewHandler() (sh *Handler) {
    sh = &Handler{make(chan os.Signal), make(map[os.Signal]Callback, 5)}
    S.Notify(sh.schan, os.Interrupt, os.Kill)
    return
}

func (sh *Handler)Bind(s os.Signal, cb Callback) {
    sh.cb[s] = cb
}

func (sh *Handler)Loop() os.Signal {
    for {
        s := <-sh.schan
        f, ok := sh.cb[s]
        if ok {
            f()
            return s
        }
    }
    return nil
}
<|endoftext|>


<commit_before>package common

import "github.com/couchbase/indexing/secondary/dcp"
import "errors"
import "fmt"
import "time"
import "net"
import "strings"
import "sync"

var (
	ErrInvalidNodeId       = errors.New("Invalid NodeId")
	ErrInvalidService      = errors.New("Invalid service")
	ErrNodeNotBucketMember = errors.New("Node is not a member of bucket")
)

// Helper object for fetching cluster information
// Can be used by services running on a cluster node to connect with
// local management service for obtaining cluster information.
// Info cache can be updated by using Refresh() method.
type ClusterInfoCache struct {
	sync.Mutex
	url       string
	poolName  string
	logPrefix string
	retries   int

	client          couchbase.Client
	pool            couchbase.Pool
	nodes           []couchbase.Node
	nodesvs         []couchbase.NodeServices
	poolsvsCh       chan couchbase.PoolServices
	poolsvsIsActive bool
	poolsvsErr      error
}

type NodeId int

func NewClusterInfoCache(cluster string, pool string) *ClusterInfoCache {
	if !strings.HasPrefix(cluster, "http://") {
		cluster = "http://" + cluster
	}
	c := &ClusterInfoCache{
		url:             cluster,
		poolName:        pool,
		poolsvsCh:       make(chan couchbase.PoolServices),
		poolsvsIsActive: false,
		retries:         0,
	}

	return c
}

func (c *ClusterInfoCache) SetLogPrefix(p string) {
	c.logPrefix = p
}

func (c *ClusterInfoCache) SetMaxRetries(r int) {
	c.retries = r
}

func (c *ClusterInfoCache) Fetch() error {

	fn := func(r int, err error) error {
		if r > 0 {
			Infof("%vError occured during cluster info update (%v) .. Retrying(%d)",
				c.logPrefix, err, r)
		}

		c.client, err = couchbase.Connect(c.url)
		if err != nil {
			return err
		}

		c.pool, err = c.client.GetPool(c.poolName)
		if err != nil {
			return err
		}

		for _, n := range c.pool.Nodes {
			if n.ClusterMembership == "active" {
				c.nodes = append(c.nodes, n)
			}
		}

		found := false
		for _, node := range c.nodes {
			if node.ThisNode {
				found = true
			}
		}

		if !found {
			return errors.New("Current node's cluster membership is not active")
		}

		var poolServs couchbase.PoolServices
		poolServs, err = c.client.GetPoolServices(c.poolName)
		if err != nil {
			return err
		}
		c.nodesvs = poolServs.NodesExt

		// Streaming nodeServices API background callback setup
		// If Fetch() is called for the first time, nodeServices callback is setup
		// Otherwise if streaming API handler is in error state, that means handler
		// is not present anymore and we need to setup a new handler.
		if c.poolsvsIsActive == false || c.poolsvsErr != nil {
			err = c.client.NodeServicesCallback(c.poolName, c.nodeServicesCallback)
			if err != nil {
				return err
			}
			c.poolsvsIsActive = true
			c.poolsvsErr = nil
		}

		return nil
	}

	rh := NewRetryHelper(c.retries, time.Second, 1, fn)
	return rh.Run()
}

func (c ClusterInfoCache) GetNodesByServiceType(srvc string) (nids []NodeId) {
	for i, svs := range c.nodesvs {
		if _, ok := svs.Services[srvc]; ok {
			nids = append(nids, NodeId(i))
		}
	}

	return
}

func (c *ClusterInfoCache) nodeServicesCallback(ps couchbase.PoolServices, err error) bool {
	if err != nil {
		c.poolsvsErr = err
		close(c.poolsvsCh)
		return false
	}

	c.poolsvsCh <- ps
	return true
}

func (c *ClusterInfoCache) WaitAndUpdateServices() error {
	if c.poolsvsErr != nil {
		return c.poolsvsErr
	}

	ps := <-c.poolsvsCh
	if c.poolsvsErr == nil {
		c.nodesvs = ps.NodesExt
	}

	return c.poolsvsErr
}

func (c ClusterInfoCache) GetNodesByBucket(bucket string) (nids []NodeId, err error) {
	b, berr := c.pool.GetBucket(bucket)
	if berr != nil {
		err = berr
		return
	}
	defer b.Close()

	for i, _ := range c.nodes {
		nid := NodeId(i)
		if _, ok := c.findVBServerIndex(b, nid); ok {
			nids = append(nids, nid)
		}
	}

	return
}

func (c ClusterInfoCache) GetCurrentNode() NodeId {
	for i, node := range c.nodes {
		if node.ThisNode {
			return NodeId(i)
		}
	}
	// TODO: can we avoid this panic ?
	panic("Current node is not in active membership")
}

func (c ClusterInfoCache) GetServiceAddress(nid NodeId, srvc string) (addr string, err error) {
	var port int
	var ok bool

	if int(nid) >= len(c.nodesvs) {
		err = ErrInvalidNodeId
		return
	}

	node := c.nodesvs[nid]
	if port, ok = node.Services[srvc]; !ok {
		err = ErrInvalidService
		return
	}

	addr = net.JoinHostPort(node.Hostname, fmt.Sprint(port))
	return
}

func (c ClusterInfoCache) GetVBuckets(nid NodeId, bucket string) (vbs []uint32, err error) {
	b, berr := c.pool.GetBucket(bucket)
	if berr != nil {
		err = berr
		return
	}
	defer b.Close()

	idx, ok := c.findVBServerIndex(b, nid)
	if !ok {
		err = ErrNodeNotBucketMember
		return
	}

	vbmap := b.VBServerMap()

	for vb, idxs := range vbmap.VBucketMap {
		if idxs[0] == idx {
			vbs = append(vbs, uint32(vb))
		}
	}

	return
}

func (c ClusterInfoCache) findVBServerIndex(b *couchbase.Bucket, nid NodeId) (int, bool) {
	bnodes := b.Nodes()

	for idx, n := range bnodes {
		if c.sameNode(n, c.nodes[nid]) {
			return idx, true
		}
	}

	return 0, false
}

func (c ClusterInfoCache) sameNode(n1 couchbase.Node, n2 couchbase.Node) bool {
	return n1.Hostname == n2.Hostname
}
<commit_msg>cluster_info: Replace with client connect hostname for current node address<commit_after>package common

import "github.com/couchbase/indexing/secondary/dcp"
import "errors"
import "fmt"
import "time"
import "net"
import "net/url"
import "strings"
import "sync"

var (
	ErrInvalidNodeId       = errors.New("Invalid NodeId")
	ErrInvalidService      = errors.New("Invalid service")
	ErrNodeNotBucketMember = errors.New("Node is not a member of bucket")
)

// Helper object for fetching cluster information
// Can be used by services running on a cluster node to connect with
// local management service for obtaining cluster information.
// Info cache can be updated by using Refresh() method.
type ClusterInfoCache struct {
	sync.Mutex
	url       string
	poolName  string
	logPrefix string
	retries   int

	client          couchbase.Client
	pool            couchbase.Pool
	nodes           []couchbase.Node
	nodesvs         []couchbase.NodeServices
	poolsvsCh       chan couchbase.PoolServices
	poolsvsIsActive bool
	poolsvsErr      error
}

type NodeId int

func NewClusterInfoCache(cluster string, pool string) *ClusterInfoCache {
	if !strings.HasPrefix(cluster, "http://") {
		cluster = "http://" + cluster
	}
	c := &ClusterInfoCache{
		url:             cluster,
		poolName:        pool,
		poolsvsCh:       make(chan couchbase.PoolServices),
		poolsvsIsActive: false,
		retries:         0,
	}

	return c
}

func (c *ClusterInfoCache) SetLogPrefix(p string) {
	c.logPrefix = p
}

func (c *ClusterInfoCache) SetMaxRetries(r int) {
	c.retries = r
}

func (c *ClusterInfoCache) Fetch() error {

	fn := func(r int, err error) error {
		if r > 0 {
			Infof("%vError occured during cluster info update (%v) .. Retrying(%d)",
				c.logPrefix, err, r)
		}

		c.client, err = couchbase.Connect(c.url)
		if err != nil {
			return err
		}

		c.pool, err = c.client.GetPool(c.poolName)
		if err != nil {
			return err
		}

		for _, n := range c.pool.Nodes {
			if n.ClusterMembership == "active" {
				c.nodes = append(c.nodes, n)
			}
		}

		found := false
		for _, node := range c.nodes {
			if node.ThisNode {
				found = true
			}
		}

		if !found {
			return errors.New("Current node's cluster membership is not active")
		}

		var poolServs couchbase.PoolServices
		poolServs, err = c.client.GetPoolServices(c.poolName)
		if err != nil {
			return err
		}
		c.nodesvs = poolServs.NodesExt

		// Streaming nodeServices API background callback setup
		// If Fetch() is called for the first time, nodeServices callback is setup
		// Otherwise if streaming API handler is in error state, that means handler
		// is not present anymore and we need to setup a new handler.
		if c.poolsvsIsActive == false || c.poolsvsErr != nil {
			err = c.client.NodeServicesCallback(c.poolName, c.nodeServicesCallback)
			if err != nil {
				return err
			}
			c.poolsvsIsActive = true
			c.poolsvsErr = nil
		}

		return nil
	}

	rh := NewRetryHelper(c.retries, time.Second, 1, fn)
	return rh.Run()
}

func (c ClusterInfoCache) GetNodesByServiceType(srvc string) (nids []NodeId) {
	for i, svs := range c.nodesvs {
		if _, ok := svs.Services[srvc]; ok {
			nids = append(nids, NodeId(i))
		}
	}

	return
}

func (c *ClusterInfoCache) nodeServicesCallback(ps couchbase.PoolServices, err error) bool {
	if err != nil {
		c.poolsvsErr = err
		close(c.poolsvsCh)
		return false
	}

	c.poolsvsCh <- ps
	return true
}

func (c *ClusterInfoCache) WaitAndUpdateServices() error {
	if c.poolsvsErr != nil {
		return c.poolsvsErr
	}

	ps := <-c.poolsvsCh
	if c.poolsvsErr == nil {
		c.nodesvs = ps.NodesExt
	}

	return c.poolsvsErr
}

func (c ClusterInfoCache) GetNodesByBucket(bucket string) (nids []NodeId, err error) {
	b, berr := c.pool.GetBucket(bucket)
	if berr != nil {
		err = berr
		return
	}
	defer b.Close()

	for i, _ := range c.nodes {
		nid := NodeId(i)
		if _, ok := c.findVBServerIndex(b, nid); ok {
			nids = append(nids, nid)
		}
	}

	return
}

func (c ClusterInfoCache) GetCurrentNode() NodeId {
	for i, node := range c.nodes {
		if node.ThisNode {
			return NodeId(i)
		}
	}
	// TODO: can we avoid this panic ?
	panic("Current node is not in active membership")
}

func (c ClusterInfoCache) GetServiceAddress(nid NodeId, srvc string) (addr string, err error) {
	var port int
	var ok bool

	if int(nid) >= len(c.nodesvs) {
		err = ErrInvalidNodeId
		return
	}

	node := c.nodesvs[nid]
	if port, ok = node.Services[srvc]; !ok {
		err = ErrInvalidService
		return
	}

	// For current node, hostname might be empty
	// Insert hostname used to connect to the cluster
	cUrl, err := url.Parse(c.url)
	if err != nil {
		return "", errors.New("Unable to parse cluster url - " + err.Error())
	}
	h, _, _ := net.SplitHostPort(cUrl.Host)
	if node.Hostname == "" {
		node.Hostname = h
	}

	addr = net.JoinHostPort(node.Hostname, fmt.Sprint(port))
	return
}

func (c ClusterInfoCache) GetVBuckets(nid NodeId, bucket string) (vbs []uint32, err error) {
	b, berr := c.pool.GetBucket(bucket)
	if berr != nil {
		err = berr
		return
	}
	defer b.Close()

	idx, ok := c.findVBServerIndex(b, nid)
	if !ok {
		err = ErrNodeNotBucketMember
		return
	}

	vbmap := b.VBServerMap()

	for vb, idxs := range vbmap.VBucketMap {
		if idxs[0] == idx {
			vbs = append(vbs, uint32(vb))
		}
	}

	return
}

func (c ClusterInfoCache) findVBServerIndex(b *couchbase.Bucket, nid NodeId) (int, bool) {
	bnodes := b.Nodes()

	for idx, n := range bnodes {
		if c.sameNode(n, c.nodes[nid]) {
			return idx, true
		}
	}

	return 0, false
}

func (c ClusterInfoCache) sameNode(n1 couchbase.Node, n2 couchbase.Node) bool {
	return n1.Hostname == n2.Hostname
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package aws

// AccessKey represents an AWS access key, including its ID and associated
// secret.
//
// To manage your access keys, go here:
//
//     https://portal.aws.amazon.com/gp/aws/securityCredentials
//
type AccessKey struct {
	// The key's ID.
	//
	// For example:
	//
	//     AKIAIOSFODNN7EXAMPLE
	//
	Id     string

	// The key's associated secret.
	//
	// For example:
	//
	//     wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
	//
	Secret string
}
<commit_msg>Fixed formatting.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package aws

// AccessKey represents an AWS access key, including its ID and associated
// secret.
//
// To manage your access keys, go here:
//
//     https://portal.aws.amazon.com/gp/aws/securityCredentials
//
type AccessKey struct {
	// The key's ID.
	//
	// For example:
	//
	//     AKIAIOSFODNN7EXAMPLE
	//
	Id string

	// The key's associated secret.
	//
	// For example:
	//
	//     wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
	//
	Secret string
}
<|endoftext|>


<commit_before>// Copyright 2013-2016 Aerospike, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package aerospike

import (
	"math"
	"reflect"
	"strings"
	"sync"
	"time"
)

const (
	aerospikeTag = "as"
	keyTag       = "key"
)

func valueToInterface(f reflect.Value, clusterSupportsFloat bool) interface{} {
	// get to the core value
	for f.Kind() == reflect.Ptr {
		if f.IsNil() {
			return nil
		}
		f = reflect.Indirect(f)
	}

	switch f.Kind() {
	case reflect.Uint64:
		return int64(f.Uint())
	case reflect.Float64, reflect.Float32:
		// support floats through integer encoding if
		// server doesn't support floats
		if clusterSupportsFloat {
			return f.Float()
		} else {
			return int(math.Float64bits(f.Float()))
		}
	case reflect.Struct:
		if f.Type().PkgPath() == "time" && f.Type().Name() == "Time" {
			return f.Interface().(time.Time).UTC().UnixNano()
		}
		return structToMap(f, clusterSupportsFloat)
	case reflect.Bool:
		if f.Bool() == true {
			return int64(1)
		}
		return int64(0)
	case reflect.Map:
		if f.IsNil() {
			return nil
		}

		newMap := make(map[interface{}]interface{}, f.Len())
		for _, mk := range f.MapKeys() {
			newMap[valueToInterface(mk, clusterSupportsFloat)] = valueToInterface(f.MapIndex(mk), clusterSupportsFloat)
		}

		return newMap
	case reflect.Slice, reflect.Array:
		if f.Kind() == reflect.Slice && f.IsNil() {
			return nil
		}

		// convert to primitives recursively
		newSlice := make([]interface{}, f.Len(), f.Cap())
		for i := 0; i < len(newSlice); i++ {
			newSlice[i] = valueToInterface(f.Index(i), clusterSupportsFloat)
		}

		return newSlice
	case reflect.Interface:
		if f.IsNil() {
			return nil
		}
		return f.Interface()
	default:
		return f.Interface()
	}
}

func fieldAlias(f reflect.StructField) string {
	alias := f.Tag.Get(aerospikeTag)
	if alias != "" {
		alias = strings.Trim(alias, " ")

		// if tag is -, the field should not be persisted
		if alias == "-" {
			return ""
		}
		return alias
	}
	return f.Name
}

func structToMap(s reflect.Value, clusterSupportsFloat bool) map[string]interface{} {
	if !s.IsValid() {
		return nil
	}

	// map tags
	cacheObjectTags(s)

	typeOfT := s.Type()
	numFields := s.NumField()

	var binMap map[string]interface{}
	for i := 0; i < numFields; i++ {
		// skip unexported fields
		if typeOfT.Field(i).PkgPath != "" {
			continue
		}

		binValue := valueToInterface(s.Field(i), clusterSupportsFloat)

		if binMap == nil {
			binMap = make(map[string]interface{}, numFields)
		}

		alias := fieldAlias(typeOfT.Field(i))
		if alias == "" {
			continue
		}

		binMap[alias] = binValue
	}

	return binMap
}

func marshal(v interface{}, clusterSupportsFloat bool) []*Bin {
	s := reflect.Indirect(reflect.ValueOf(v).Elem())

	// map tags
	cacheObjectTags(s)

	numFields := s.NumField()
	bins := binPool.Get(numFields).([]*Bin)

	binCount := 0
	n := structToMap(s, clusterSupportsFloat)
	for k, v := range n {
		bins[binCount].Name = k

		bins[binCount].Value = NewValue(v)
		binCount++
	}

	return bins[:binCount]
}

type SyncMap struct {
	objectMappings map[string]map[string]string
	objectFields   map[string][]string
	mutex          sync.RWMutex
}

func (sm *SyncMap) setMapping(obj reflect.Value, mapping map[string]string, fields []string) {
	objType := obj.Type().Name()
	sm.mutex.Lock()
	sm.objectMappings[objType] = mapping
	sm.objectFields[objType] = fields
	sm.mutex.Unlock()
}

func (sm *SyncMap) mappingExists(obj reflect.Value) bool {
	objType := obj.Type().Name()
	sm.mutex.RLock()
	_, exists := sm.objectMappings[objType]
	sm.mutex.RUnlock()
	return exists
}

func (sm *SyncMap) getMapping(obj reflect.Value) map[string]string {
	if !obj.IsValid() {
		return nil
	}
	objType := obj.Type().Name()
	sm.mutex.RLock()
	mapping := sm.objectMappings[objType]
	sm.mutex.RUnlock()
	return mapping
}

func (sm *SyncMap) getFields(obj reflect.Value) []string {
	objType := obj.Type().Name()
	sm.mutex.RLock()
	fields := sm.objectFields[objType]
	sm.mutex.RUnlock()
	return fields
}

var objectMappings = &SyncMap{objectMappings: map[string]map[string]string{}, objectFields: map[string][]string{}}

func cacheObjectTags(obj reflect.Value) {
	// exit if already processed
	if objectMappings.mappingExists(obj) {
		return
	}

	for obj.Kind() == reflect.Ptr {
		if obj.IsNil() {
			return
		}
		obj = reflect.Indirect(obj)
	}

	mapping := map[string]string{}
	fields := []string{}

	typeOfT := obj.Type()
	numFields := obj.NumField()
	for i := 0; i < numFields; i++ {
		f := typeOfT.Field(i)
		// skip unexported fields
		if f.PkgPath != "" {
			continue
		}

		tag := strings.Trim(f.Tag.Get(aerospikeTag), " ")
		if tag != "-" {
			if tag != "" {
				mapping[tag] = f.Name
				fields = append(fields, tag)
			} else {
				fields = append(fields, f.Name)
			}
		}
	}

	objectMappings.setMapping(obj, mapping, fields)
}
<commit_msg>marshal: fixed object mapping cache for anonymous structs (issue 114)<commit_after>// Copyright 2013-2016 Aerospike, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package aerospike

import (
	"math"
	"reflect"
	"strings"
	"sync"
	"time"
)

const (
	aerospikeTag = "as"
	keyTag       = "key"
)

func valueToInterface(f reflect.Value, clusterSupportsFloat bool) interface{} {
	// get to the core value
	for f.Kind() == reflect.Ptr {
		if f.IsNil() {
			return nil
		}
		f = reflect.Indirect(f)
	}

	switch f.Kind() {
	case reflect.Uint64:
		return int64(f.Uint())
	case reflect.Float64, reflect.Float32:
		// support floats through integer encoding if
		// server doesn't support floats
		if clusterSupportsFloat {
			return f.Float()
		} else {
			return int(math.Float64bits(f.Float()))
		}
	case reflect.Struct:
		if f.Type().PkgPath() == "time" && f.Type().Name() == "Time" {
			return f.Interface().(time.Time).UTC().UnixNano()
		}
		return structToMap(f, clusterSupportsFloat)
	case reflect.Bool:
		if f.Bool() == true {
			return int64(1)
		}
		return int64(0)
	case reflect.Map:
		if f.IsNil() {
			return nil
		}

		newMap := make(map[interface{}]interface{}, f.Len())
		for _, mk := range f.MapKeys() {
			newMap[valueToInterface(mk, clusterSupportsFloat)] = valueToInterface(f.MapIndex(mk), clusterSupportsFloat)
		}

		return newMap
	case reflect.Slice, reflect.Array:
		if f.Kind() == reflect.Slice && f.IsNil() {
			return nil
		}

		// convert to primitives recursively
		newSlice := make([]interface{}, f.Len(), f.Cap())
		for i := 0; i < len(newSlice); i++ {
			newSlice[i] = valueToInterface(f.Index(i), clusterSupportsFloat)
		}

		return newSlice
	case reflect.Interface:
		if f.IsNil() {
			return nil
		}
		return f.Interface()
	default:
		return f.Interface()
	}
}

func fieldAlias(f reflect.StructField) string {
	alias := f.Tag.Get(aerospikeTag)
	if alias != "" {
		alias = strings.Trim(alias, " ")

		// if tag is -, the field should not be persisted
		if alias == "-" {
			return ""
		}
		return alias
	}
	return f.Name
}

func structToMap(s reflect.Value, clusterSupportsFloat bool) map[string]interface{} {
	if !s.IsValid() {
		return nil
	}

	// map tags
	cacheObjectTags(s)

	typeOfT := s.Type()
	numFields := s.NumField()

	var binMap map[string]interface{}
	for i := 0; i < numFields; i++ {
		// skip unexported fields
		if typeOfT.Field(i).PkgPath != "" {
			continue
		}

		binValue := valueToInterface(s.Field(i), clusterSupportsFloat)

		if binMap == nil {
			binMap = make(map[string]interface{}, numFields)
		}

		alias := fieldAlias(typeOfT.Field(i))
		if alias == "" {
			continue
		}

		binMap[alias] = binValue
	}

	return binMap
}

func marshal(v interface{}, clusterSupportsFloat bool) []*Bin {
	s := reflect.Indirect(reflect.ValueOf(v).Elem())

	// map tags
	cacheObjectTags(s)

	numFields := s.NumField()
	bins := binPool.Get(numFields).([]*Bin)

	binCount := 0
	n := structToMap(s, clusterSupportsFloat)
	for k, v := range n {
		bins[binCount].Name = k

		bins[binCount].Value = NewValue(v)
		binCount++
	}

	return bins[:binCount]
}

type SyncMap struct {
	objectMappings map[reflect.Type]map[string]string
	objectFields   map[reflect.Type][]string
	mutex          sync.RWMutex
}

func (sm *SyncMap) setMapping(obj reflect.Value, mapping map[string]string, fields []string) {
	objType := obj.Type()
	sm.mutex.Lock()
	sm.objectMappings[objType] = mapping
	sm.objectFields[objType] = fields
	sm.mutex.Unlock()
}

func (sm *SyncMap) mappingExists(obj reflect.Value) bool {
	objType := obj.Type()
	sm.mutex.RLock()
	_, exists := sm.objectMappings[objType]
	sm.mutex.RUnlock()
	return exists
}

func (sm *SyncMap) getMapping(obj reflect.Value) map[string]string {
	if !obj.IsValid() {
		return nil
	}
	objType := obj.Type()
	sm.mutex.RLock()
	mapping := sm.objectMappings[objType]
	sm.mutex.RUnlock()
	return mapping
}

func (sm *SyncMap) getFields(obj reflect.Value) []string {
	objType := obj.Type()
	sm.mutex.RLock()
	fields := sm.objectFields[objType]
	sm.mutex.RUnlock()
	return fields
}

var objectMappings = &SyncMap{objectMappings: map[reflect.Type]map[string]string{}, objectFields: map[reflect.Type][]string{}}

func cacheObjectTags(obj reflect.Value) {
	// exit if already processed
	if objectMappings.mappingExists(obj) {
		return
	}

	for obj.Kind() == reflect.Ptr {
		if obj.IsNil() {
			return
		}
		obj = reflect.Indirect(obj)
	}

	mapping := map[string]string{}
	fields := []string{}

	typeOfT := obj.Type()
	numFields := obj.NumField()
	for i := 0; i < numFields; i++ {
		f := typeOfT.Field(i)
		// skip unexported fields
		if f.PkgPath != "" {
			continue
		}

		tag := strings.Trim(f.Tag.Get(aerospikeTag), " ")
		if tag != "-" {
			if tag != "" {
				mapping[tag] = f.Name
				fields = append(fields, tag)
			} else {
				fields = append(fields, f.Name)
			}
		}
	}

	objectMappings.setMapping(obj, mapping, fields)
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cgotest

/*
#include <unistd.h>

unsigned int sleep(unsigned int seconds);

extern void BackgroundSleep(int);
void twoSleep(int);
*/
import "C"

import (
	"runtime"
	"testing"
	"time"
)

var sleepDone = make(chan bool)

func parallelSleep(n int) {
	C.twoSleep(C.int(n))
	<-sleepDone
}

//export BackgroundSleep
func BackgroundSleep(n int) {
	go func() {
		C.sleep(C.uint(n))
		sleepDone <- true
	}()
}

// wasteCPU starts a background goroutine to waste CPU
// to cause the power management to raise the CPU frequency.
// On ARM this has the side effect of making sleep more accurate.
func wasteCPU() chan struct{} {
	done := make(chan struct{})
	go func() {
		for {
			select {
			case <-done:
				return
			default:
			}
		}
	}()
	// pause for a short amount of time to allow the
	// power management to recognise load has risen.
	<-time.After(300 * time.Millisecond)
	return done
}

func testParallelSleep(t *testing.T) {
	defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(2))
	defer close(wasteCPU())

	sleepSec := 1
	start := time.Now()
	parallelSleep(sleepSec)
	dt := time.Since(start)
	t.Logf("sleep(%d) slept for %v", sleepSec, dt)
	// bug used to run sleeps in serial, producing a 2*sleepSec-second delay.
	if dt >= time.Duration(sleepSec)*1300*time.Millisecond {
		t.Fatalf("parallel %d-second sleeps slept for %f seconds", sleepSec, dt.Seconds())
	}
}
<commit_msg>misc/cgo/test: do not run wasteCPU during TestParallelSleep unless on arm<commit_after>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cgotest

/*
#include <unistd.h>

unsigned int sleep(unsigned int seconds);

extern void BackgroundSleep(int);
void twoSleep(int);
*/
import "C"

import (
	"runtime"
	"testing"
	"time"
)

var sleepDone = make(chan bool)

func parallelSleep(n int) {
	C.twoSleep(C.int(n))
	<-sleepDone
}

//export BackgroundSleep
func BackgroundSleep(n int) {
	go func() {
		C.sleep(C.uint(n))
		sleepDone <- true
	}()
}

// wasteCPU starts a background goroutine to waste CPU
// to cause the power management to raise the CPU frequency.
// On ARM this has the side effect of making sleep more accurate.
func wasteCPU() chan struct{} {
	done := make(chan struct{})
	go func() {
		for {
			select {
			case <-done:
				return
			default:
			}
		}
	}()
	// pause for a short amount of time to allow the
	// power management to recognise load has risen.
	<-time.After(300 * time.Millisecond)
	return done
}

func testParallelSleep(t *testing.T) {
	if runtime.GOARCH == "arm" {
		// on ARM, the 1.3s deadline is frequently missed,
		// and burning cpu seems to help
		defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(2))
		defer close(wasteCPU())
	}

	sleepSec := 1
	start := time.Now()
	parallelSleep(sleepSec)
	dt := time.Since(start)
	t.Logf("sleep(%d) slept for %v", sleepSec, dt)
	// bug used to run sleeps in serial, producing a 2*sleepSec-second delay.
	if dt >= time.Duration(sleepSec)*1300*time.Millisecond {
		t.Fatalf("parallel %d-second sleeps slept for %f seconds", sleepSec, dt.Seconds())
	}
}
<|endoftext|>


<commit_before>package discovery

import (
	"net"
	"strconv"
	"strings"

	etcd "github.com/coreos/etcd/clientv3"
	"github.com/golang/protobuf/proto"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
)

var netDial = net.Dial

/*
NewSimpleClient determines the correct connection endpoints matching the
specified path and attempts to connect to them. If multiple destinations
are linked to the destination (e.g. through a service subdirectory),
a connection is attempted to each backend in random order. In case of
connection errors the target is skipped and the next one is attempted.

The connections returned are regular connections, if they are interrupted
they will not be reestablished.
*/
func NewSimpleClient(ctx context.Context, client etcd.KV, path string) (
	net.Conn, error) {
	var configs []*ExportedServiceRecord
	var configPerms []int
	var getResponse *etcd.GetResponse
	var realPath, slashPath string
	var conn net.Conn
	var err error

	if path[0] == '/' {
		realPath = path
	} else {
		realPath = "/ns/service/" + realPath
	}

	getResponse, err = client.Get(ctx, realPath, etcd.WithPrefix())
	if err != nil {
		return nil, err
	}

	if realPath[len(realPath)-1] == '/' {
		slashPath = realPath
	} else {
		slashPath = realPath + "/"
	}

	for _, kv := range getResponse.Kvs {
		if string(kv.Key) == realPath || strings.HasPrefix(string(kv.Key), slashPath) {
			var config = new(ExportedServiceRecord)
			err = proto.Unmarshal(kv.Value, config)
			if err == nil {
				configs = append(configs, config)
			}
		}
	}

	if len(configs) == 0 {
		if err == nil {
			err = grpc.Errorf(codes.Unavailable, "No services were found at %s",
				realPath)
		}

		return nil, err
	}

	// Try to connect to different ports at random.
	configPerms = make([]int, len(configs))
	for _, i := range configPerms {
		var config = configs[i]
		var dest = net.JoinHostPort(
			config.Address, strconv.Itoa(int(config.Port)))

		conn, err = netDial(config.Protocol, dest)
		if err == nil {
			return conn, err
		}
	}

	return nil, err
}
<commit_msg>Fix a few bugs in simple_client.go found by the unit test.<commit_after>package discovery

import (
	"math/rand"
	"net"
	"strconv"
	"strings"

	etcd "github.com/coreos/etcd/clientv3"
	"github.com/golang/protobuf/proto"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
)

var netDial = net.Dial

/*
NewSimpleClient determines the correct connection endpoints matching the
specified path and attempts to connect to them. If multiple destinations
are linked to the destination (e.g. through a service subdirectory),
a connection is attempted to each backend in random order. In case of
connection errors the target is skipped and the next one is attempted.

The connections returned are regular connections, if they are interrupted
they will not be reestablished.
*/
func NewSimpleClient(ctx context.Context, client etcd.KV, path string) (
	net.Conn, error) {
	var configs []*ExportedServiceRecord
	var configPerms []int
	var getResponse *etcd.GetResponse
	var realPath, slashPath string
	var conn net.Conn
	var err error

	if path[0] == '/' {
		realPath = path
	} else {
		realPath = "/ns/service/" + path
	}

	getResponse, err = client.Get(ctx, realPath, etcd.WithPrefix())
	if err != nil {
		return nil, err
	}

	if realPath[len(realPath)-1] == '/' {
		slashPath = realPath
	} else {
		slashPath = realPath + "/"
	}

	for _, kv := range getResponse.Kvs {
		if string(kv.Key) == realPath || strings.HasPrefix(string(kv.Key), slashPath) {
			var config = new(ExportedServiceRecord)
			err = proto.Unmarshal(kv.Value, config)
			if err == nil {
				configs = append(configs, config)
			}
		}
	}

	if len(configs) == 0 {
		if err == nil {
			err = grpc.Errorf(codes.Unavailable, "No services were found at %s",
				realPath)
		}

		return nil, err
	}

	// Try to connect to different ports at random.
	configPerms = rand.Perm(len(configs))
	for _, i := range configPerms {
		var config = configs[i]
		var dest = net.JoinHostPort(
			config.Address, strconv.Itoa(int(config.Port)))

		conn, err = netDial(config.Protocol, dest)
		if err == nil {
			return conn, err
		}
	}

	return nil, err
}
<|endoftext|>