<commit_before>package registry

import (
	"bytes"
	"io/ioutil"
	"net/http"

	manifestV1 "github.com/docker/distribution/manifest/schema1"
	manifestV2 "github.com/docker/distribution/manifest/schema2"
	digest "github.com/opencontainers/go-digest"
)

func (registry *Registry) Manifest(repository, reference string) (*manifestV1.SignedManifest, error) {
	url := registry.url("/v2/%s/manifests/%s", repository, reference)
	registry.Logf("registry.manifest.get url=%s repository=%s reference=%s", url, repository, reference)

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}

	req.Header.Set("Accept", manifestV1.MediaTypeManifest)
	resp, err := registry.Client.Do(req)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	signedManifest := &manifestV1.SignedManifest{}
	err = signedManifest.UnmarshalJSON(body)
	if err != nil {
		return nil, err
	}

	return signedManifest, nil
}

func (registry *Registry) ManifestV2(repository, reference string) (*manifestV2.DeserializedManifest, error) {
	url := registry.url("/v2/%s/manifests/%s", repository, reference)
	registry.Logf("registry.manifest.get url=%s repository=%s reference=%s", url, repository, reference)

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}

	req.Header.Set("Accept", manifestV2.MediaTypeManifest)
	resp, err := registry.Client.Do(req)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	deserialized := &manifestV2.DeserializedManifest{}
	err = deserialized.UnmarshalJSON(body)
	if err != nil {
		return nil, err
	}
	return deserialized, nil
}

func (registry *Registry) ManifestDigest(repository, reference string) (digest.Digest, error) {
	url := registry.url("/v2/%s/manifests/%s", repository, reference)
	registry.Logf("registry.manifest.head url=%s repository=%s reference=%s", url, repository, reference)

	req, err := http.NewRequest("HEAD", url, nil)
	if err != nil {
		return "", err
	}

	req.Header.Set("Accept", manifestV2.MediaTypeManifest)
	resp, err := registry.Client.Do(req)
	if resp != nil {
		defer resp.Body.Close()
	}
	if err != nil {
		return "", err
	}
	return digest.Digest(resp.Header.Get("Docker-Content-Digest")), nil
}

func (registry *Registry) DeleteManifest(repository string, digest digest.Digest) error {
	url := registry.url("/v2/%s/manifests/%s", repository, digest)
	registry.Logf("registry.manifest.delete url=%s repository=%s reference=%s", url, repository, digest)

	req, err := http.NewRequest("DELETE", url, nil)
	if err != nil {
		return err
	}
	req.Header.Set("Accept", manifestV2.MediaTypeManifest)
	resp, err := registry.Client.Do(req)
	if resp != nil {
		defer resp.Body.Close()
	}
	if err != nil {
		return err
	}
	return nil
}

func (registry *Registry) PutManifest(repository, reference string, signedManifest *manifestV1.SignedManifest) error {
	url := registry.url("/v2/%s/manifests/%s", repository, reference)
	registry.Logf("registry.manifest.put url=%s repository=%s reference=%s", url, repository, reference)

	body, err := signedManifest.MarshalJSON()
	if err != nil {
		return err
	}

	buffer := bytes.NewBuffer(body)
	req, err := http.NewRequest("PUT", url, buffer)
	if err != nil {
		return err
	}

	req.Header.Set("Content-Type", manifestV1.MediaTypeManifest)
	resp, err := registry.Client.Do(req)
	if resp != nil {
		defer resp.Body.Close()
	}
	return err
}
<commit_msg>Added manifest metadata v2 helper function<commit_after>package registry

import (
	"io/ioutil"
	"net/http"

	manifestV1 "github.com/docker/distribution/manifest/schema1"
	manifestV2 "github.com/docker/distribution/manifest/schema2"
	digest "github.com/opencontainers/go-digest"
)

func (registry *Registry) ManifestV2(repository, reference string) (*manifestV2.DeserializedManifest, error) {
	url := registry.url("/v2/%s/manifests/%s", repository, reference)
	registry.Logf("registry.manifest.get url=%s repository=%s reference=%s", url, repository, reference)

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}

	req.Header.Set("Accept", manifestV2.MediaTypeManifest)
	resp, err := registry.Client.Do(req)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	deserialized := &manifestV2.DeserializedManifest{}
	err = deserialized.UnmarshalJSON(body)
	if err != nil {
		return nil, err
	}
	return deserialized, nil
}

func (registry *Registry) ManifestDigest(repository, reference string) (digest.Digest, error) {
	url := registry.url("/v2/%s/manifests/%s", repository, reference)
	registry.Logf("registry.manifest.head url=%s repository=%s reference=%s", url, repository, reference)

	req, err := http.NewRequest("HEAD", url, nil)
	if err != nil {
		return "", err
	}

	req.Header.Set("Accept", manifestV2.MediaTypeManifest)
	resp, err := registry.Client.Do(req)
	if resp != nil {
		defer resp.Body.Close()
	}
	if err != nil {
		return "", err
	}
	return digest.Digest(resp.Header.Get("Docker-Content-Digest")), nil
}

func (registry *Registry) DeleteManifest(repository string, digest digest.Digest) error {
	url := registry.url("/v2/%s/manifests/%s", repository, digest)
	registry.Logf("registry.manifest.delete url=%s repository=%s reference=%s", url, repository, digest)

	req, err := http.NewRequest("DELETE", url, nil)
	if err != nil {
		return err
	}
	req.Header.Set("Accept", manifestV2.MediaTypeManifest)
	resp, err := registry.Client.Do(req)
	if resp != nil {
		defer resp.Body.Close()
	}
	if err != nil {
		return err
	}
	return nil
}

func (registry *Registry) PutManifest(repository, reference string) error {
	url := registry.url("/v2/%s/manifests/%s", repository, reference)
	registry.Logf("registry.manifest.put url=%s repository=%s reference=%s", url, repository, reference)

	req, err := http.NewRequest("PUT", url, nil)
	if err != nil {
		return err
	}

	req.Header.Set("Content-Type", manifestV2.MediaTypeManifest)
	resp, err := registry.Client.Do(req)
	if resp != nil {
		defer resp.Body.Close()
	}
	return err
}

func (registry *Registry) ManifestMetadata(repository string, digest digest.Digest) ([]byte, error) {
	url := registry.url("/v2/%s/blobs/%s", repository, digest.String())
	registry.Logf("registry.layer.check url=%s repository=%s digest=%s", url, repository, digest)

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}

	req.Header.Set("Content-Type", manifestV1.MediaTypeManifest)
	resp, err := registry.Client.Do(req)
	if resp != nil {
		defer resp.Body.Close()
	}
	if err != nil {
		return nil, err
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	return body, nil
}
<|endoftext|>


<commit_before>package scans

import (
	"encoding/json"
	"testing"

	. "github.com/franela/goblin"
	. "github.com/onsi/gomega"
)

func TestScanResults(t *testing.T) {
	g := Goblin(t)
	RegisterFailHandler(func(m string, _ ...int) { g.Fail(m) })

	g.Describe("Scan Results", func() {
		g.It("should unmarshal a scan results with about yml data", func() {
			var r Results
			err := json.Unmarshal([]byte(SampleValidScanResultsAboutYML), &r)

			Expect(err).To(BeNil())
			Expect(r.Type).To(Equal("about_yml"))

			a, ok := r.Data.(AboutYMLResults)
			Expect(ok).To(Equal(true))
			Expect(a.Content).To(Equal("some content"))
		})

		g.It("should unmarshal a scan results with community data", func() {
			var r Results
			err := json.Unmarshal([]byte(SampleValidScanResultsCommunity), &r)

			Expect(err).To(BeNil())
			Expect(r.Type).To(Equal("community"))

			a, ok := r.Data.([]CommunityResults)
			Expect(ok).To(Equal(true))
			Expect(len(a)).To(Equal(1))
			Expect(a[0].Committers).To(Equal(7))
			Expect(a[0].Name).To(Equal("ion-channel/ion-connect"))
			Expect(a[0].URL).To(Equal("https://github.com/ion-channel/ion-connect"))
		})

		g.It("should unmarshal a scan results with coverage data", func() {
			var r Results
			err := json.Unmarshal([]byte(SampleValidScanResultsCoverage), &r)

			Expect(err).To(BeNil())
			Expect(r.Type).To(Equal("external_coverage"))

			c, ok := r.Data.(CoverageResults)
			Expect(ok).To(Equal(true))
			Expect(c.Value).To(Equal(42.0))
		})

		g.It("should unmarshal a scan results with dependency data", func() {
			var r Results
			err := json.Unmarshal([]byte(SampleValidScanResultsDependency), &r)

			Expect(err).To(BeNil())
			Expect(r.Type).To(Equal("dependency"))

			d, ok := r.Data.(DependencyResults)
			Expect(ok).To(Equal(true))
			Expect(len(d.Dependencies)).To(Equal(7))
			Expect(d.Meta.FirstDegreeCount).To(Equal(3))
			Expect(d.Meta.NoVersionCount).To(Equal(0))
			Expect(d.Meta.TotalUniqueCount).To(Equal(7))
			Expect(d.Meta.UpdateAvailableCount).To(Equal(2))
		})

		g.It("should unmarshal a scan results with difference data", func() {
			var r Results
			err := json.Unmarshal([]byte(SampleValidScanResultsDifference), &r)

			Expect(err).To(BeNil())
			Expect(r.Type).To(Equal("difference"))

			d, ok := r.Data.(DifferenceResults)
			Expect(ok).To(Equal(true))
			Expect(d.Checksum).To(Equal("checksumishere"))
			Expect(d.Difference).To(BeTrue())
		})

		g.It("should unmarshal a scan results with ecosystem data", func() {
			var r Results
			err := json.Unmarshal([]byte(SampleValidScanResultsEcosystems), &r)

			Expect(err).To(BeNil())
			Expect(r.Type).To(Equal("ecosystems"))

			e, ok := r.Data.(EcosystemResults)
			Expect(ok).To(Equal(true))
			Expect(len(e.Ecosystems)).To(Equal(3))
		})

		g.It("should unmarshal a scan results with external vulnerabilities scan data", func() {
			var r Results
			err := json.Unmarshal([]byte(SampleValidExternalVulnerabilities), &r)

			Expect(err).To(BeNil())
			Expect(r.Type).To(Equal("external_vulnerability"))

			e, ok := r.Data.(ExternalVulnerabilitiesResults)
			Expect(ok).To(Equal(true))
			Expect(e.Critical).To(Equal(1))
			Expect(e.High).To(Equal(0))
			Expect(e.Medium).To(Equal(1))
			Expect(e.Low).To(Equal(0))
		})

		g.It("should unmarshal a scan results with license data", func() {
			var r Results
			err := json.Unmarshal([]byte(SampleValidScanResultsLicense), &r)

			Expect(err).To(BeNil())
			Expect(r.Type).To(Equal("license"))

			l, ok := r.Data.(LicenseResults)
			Expect(ok).To(Equal(true))
			Expect(l.License.Name).To(Equal("Not found"))
		})

		g.It("should unmarshal a scan results with virus data", func() {
			var r Results
			err := json.Unmarshal([]byte(SampleValidScanResultsVirus), &r)

			Expect(err).To(BeNil())
			Expect(r.Type).To(Equal("virus"))

			v, ok := r.Data.(VirusResults)
			Expect(ok).To(Equal(true))
			Expect(v.KnownViruses).To(Equal(10))
		})

		g.It("should unmarshal a scan results with vulnerability data", func() {
			var r Results
			err := json.Unmarshal([]byte(SampleValidScanResultsVulnerability), &r)

			Expect(err).To(BeNil())
			Expect(r.Type).To(Equal("vulnerability"))

			v, ok := r.Data.(VulnerabilityResults)
			Expect(ok).To(Equal(true))
			Expect(v.Meta.VulnerabilityCount).To(Equal(1))
		})

		g.It("should return an error for an invalid results type", func() {
			var r Results
			err := json.Unmarshal([]byte(SampleInvalidResults), &r)

			Expect(err).NotTo(BeNil())
			Expect(err.Error()).To(ContainSubstring("invalid results type"))
		})
	})
}

const (
	SampleValidScanResultsAboutYML      = `{"type":"about_yml", "data":{"message": "foo message", "valid": true, "content": "some content"}}`
	SampleValidScanResultsCommunity     = `{"type":"community", "data":[{"committers":7,"name":"ion-channel/ion-connect","url":"https://github.com/ion-channel/ion-connect"}]}`
	SampleValidScanResultsCoverage      = `{"type":"external_coverage", "data":{"value":42.0}}`
	SampleValidScanResultsDependency    = `{"type":"dependency","data":{"dependencies":[{"latest_version":"2.0","org":"net.sourceforge.javacsv","name":"javacsv","type":"maven","package":"jar","version":"2.0","scope":"compile"},{"latest_version":"4.12","org":"junit","name":"junit","type":"maven","package":"jar","version":"4.11","scope":"test"},{"latest_version":"1.4-atlassian-1","org":"org.hamcrest","name":"hamcrest-core","type":"maven","package":"jar","version":"1.3","scope":"test"},{"latest_version":"4.5.2","org":"org.apache.httpcomponents","name":"httpclient","type":"maven","package":"jar","version":"4.3.4","scope":"compile"},{"latest_version":"4.4.5","org":"org.apache.httpcomponents","name":"httpcore","type":"maven","package":"jar","version":"4.3.2","scope":"compile"},{"latest_version":"99.0-does-not-exist","org":"commons-logging","name":"commons-logging","type":"maven","package":"jar","version":"1.1.3","scope":"compile"},{"latest_version":"20041127.091804","org":"commons-codec","name":"commons-codec","type":"maven","package":"jar","version":"1.6","scope":"compile"}],"meta":{"first_degree_count":3,"no_version_count":0,"total_unique_count":7,"update_available_count":2}}}`
	SampleValidScanResultsEcosystems    = `{"type":"ecosystems","data":{"ecosystems":[{"ecosystem":"Java","lines":2430},{"ecosystem":"Makefile","lines":210},{"ecosystem":"Ruby","lines":666}]}}`
	SampleValidScanResultsLicense       = `{"type":"license","data":{"license":{"name":"Not found","type":[]}}}`
	SampleValidScanResultsVirus         = `{"type":"virus","data":{"known_viruses":10,"engine_version":"","scanned_directories":1,"scanned_files":2,"infected_files":1,"data_scanned":"some cool data was scanned","data_read":"we read some data","time":"10PM","file_notes": {"empty_file":["file1","file2","file3"]},"clam_av_details":{"clamav_version":"1.0.0","clamav_db_version":"1.1.0"}}}`
	SampleValidScanResultsVulnerability = `{"type":"vulnerability","data":{"vulnerabilities":[{"id":316274974,"name":"hadoop","org":"apache","version":"2.8.0","up":null,"edition":null,"aliases":null,"created_at":"2017-02-13T20:02:32.785Z","updated_at":"2017-02-13T20:02:32.785Z","title":null,"references":null,"part":null,"language":null,"source_id":1,"external_id":"cpe:/a:apache:hadoop:2.8.0","vulnerabilities":[{"id":92596,"external_id":"CVE-2017-7669","title":"CVE-2017-7669","summary":"In Apache Hadoop 2.8.0, 3.0.0-alpha1, and 3.0.0-alpha2, the LinuxContainerExecutor runs docker commands as root with insufficient input validation. When the docker feature is enabled, authenticated users can run commands as root.","score":"8.5","score_version":"2.0","score_system":"CVSS","score_details":{"cvssv2":{"vectorString":"(AV:N/AC:M/Au:S/C:C/I:C/A:C)","accessVector":"NETWORK","accessComplexity":"MEDIUM","authentication":"SINGLE","confidentialityImpact":"COMPLETE","integrityImpact":"COMPLETE","availabilityImpact":"COMPLETE","baseScore":8.5},"cvssv3":{"vectorString":"AV:N/AC:H/PR:L/UI:N/S:U/C:H/I:H/A:H","attackVector":"NETWORK","attackComplexity":"HIGH","privilegesRequired":"LOW","userInteraction":"NONE","scope":"UNCHANGED","confidentialityImpact":"HIGH","integrityImpact":"HIGH","availabilityImpact":"HIGH","baseScore":7.5,"baseSeverity":"HIGH"}},"vector":"NETWORK","access_complexity":"MEDIUM","vulnerability_authentication":"SINGLE","confidentiality_impact":"COMPLETE","integrity_impact":"COMPLETE","availability_impact":"COMPLETE","vulnerability_source":null,"assessment_check":null,"scanner":null,"recommendation":"","references":[{"type":"UNKNOWN","source":"","url":"http://www.securityfocus.com/bid/98795","text":"http://www.securityfocus.com/bid/98795"},{"type":"UNKNOWN","source":"","url":"https://mail-archives.apache.org/mod_mbox/hadoop-user/201706.mbox/%3C4A2FDA56-491B-4C2A-915F-C9D4A4BDB92A%40apache.org%3E","text":"https://mail-archives.apache.org/mod_mbox/hadoop-user/201706.mbox/%3C4A2FDA56-491B-4C2A-915F-C9D4A4BDB92A%40apache.org%3E"}],"modified_at":"2017-06-09T16:21:00.000Z","published_at":"2017-06-05T01:29:00.000Z","created_at":"2017-07-12T23:07:35.491Z","updated_at":"2017-07-12T23:07:35.491Z","source_id":1}]}],"meta":{"vulnerability_count":1}}}`
	SampleInvalidResults                = `{"type":"fooresult", "data":"I pitty the foo"}`
	SampleValidScanResultsDifference    = `{"data": {"checksum": "checksumishere","difference": true},"type": "difference"}`
	SampleValidExternalVulnerabilities  = `{"type":"external_vulnerability","data":{"critical":1,"high":0,"medium":1,"low": 0}}`
)
<commit_msg>Fix ze test.<commit_after>package scans

import (
	"encoding/json"
	"testing"

	. "github.com/franela/goblin"
	. "github.com/onsi/gomega"
)

func TestScanResults(t *testing.T) {
	g := Goblin(t)
	RegisterFailHandler(func(m string, _ ...int) { g.Fail(m) })

	g.Describe("Scan Results", func() {
		g.It("should unmarshal a scan results with about yml data", func() {
			var r Results
			err := json.Unmarshal([]byte(SampleValidScanResultsAboutYML), &r)

			Expect(err).To(BeNil())
			Expect(r.Type).To(Equal("about_yml"))

			a, ok := r.Data.(AboutYMLResults)
			Expect(ok).To(Equal(true))
			Expect(a.Content).To(Equal("some content"))
		})

		g.It("should unmarshal a scan results with community data", func() {
			var r Results
			err := json.Unmarshal([]byte(SampleValidScanResultsCommunity), &r)

			Expect(err).To(BeNil())
			Expect(r.Type).To(Equal("community"))

			a, ok := r.Data.([]CommunityResults)
			Expect(ok).To(Equal(true))
			Expect(len(a)).To(Equal(1))
			Expect(a[0].Committers).To(Equal(7))
			Expect(a[0].Name).To(Equal("ion-channel/ion-connect"))
			Expect(a[0].URL).To(Equal("https://github.com/ion-channel/ion-connect"))
		})

		g.It("should unmarshal a scan results with coverage data", func() {
			var r Results
			err := json.Unmarshal([]byte(SampleValidScanResultsCoverage), &r)

			Expect(err).To(BeNil())
			Expect(r.Type).To(Equal("external_coverage"))

			c, ok := r.Data.(CoverageResults)
			Expect(ok).To(Equal(true))
			Expect(c.Value).To(Equal(42.0))
		})

		g.It("should unmarshal a scan results with dependency data", func() {
			var r Results
			err := json.Unmarshal([]byte(SampleValidScanResultsDependency), &r)

			Expect(err).To(BeNil())
			Expect(r.Type).To(Equal("dependency"))

			d, ok := r.Data.(DependencyResults)
			Expect(ok).To(Equal(true))
			Expect(len(d.Dependencies)).To(Equal(7))
			Expect(d.Meta.FirstDegreeCount).To(Equal(3))
			Expect(d.Meta.NoVersionCount).To(Equal(0))
			Expect(d.Meta.TotalUniqueCount).To(Equal(7))
			Expect(d.Meta.UpdateAvailableCount).To(Equal(2))
		})

		g.It("should unmarshal a scan results with difference data", func() {
			var r Results
			err := json.Unmarshal([]byte(SampleValidScanResultsDifference), &r)

			Expect(err).To(BeNil())
			Expect(r.Type).To(Equal("difference"))

			d, ok := r.Data.(DifferenceResults)
			Expect(ok).To(Equal(true))
			Expect(d.Checksum).To(Equal("checksumishere"))
			Expect(d.Difference).To(BeTrue())
		})

		g.It("should unmarshal a scan results with ecosystem data", func() {
			var r Results
			err := json.Unmarshal([]byte(SampleValidScanResultsEcosystems), &r)

			Expect(err).To(BeNil())
			Expect(r.Type).To(Equal("ecosystems"))

			e, ok := r.Data.(EcosystemResults)
			Expect(ok).To(Equal(true))
			Expect(len(e.Ecosystems)).To(Equal(3))
		})

		g.It("should unmarshal a scan results with external vulnerabilities scan data", func() {
			var r Results
			err := json.Unmarshal([]byte(SampleValidExternalVulnerabilities), &r)

			Expect(err).To(BeNil())
			Expect(r.Type).To(Equal("external_vulnerability"))

			e, ok := r.Data.(ExternalVulnerabilitiesResults)
			Expect(ok).To(Equal(true))
			Expect(e.Critical).To(Equal(1))
			Expect(e.High).To(Equal(0))
			Expect(e.Medium).To(Equal(1))
			Expect(e.Low).To(Equal(0))
		})

		g.It("should unmarshal a scan results with license data", func() {
			var r Results
			err := json.Unmarshal([]byte(SampleValidScanResultsLicense), &r)

			Expect(err).To(BeNil())
			Expect(r.Type).To(Equal("license"))

			l, ok := r.Data.(LicenseResults)
			Expect(ok).To(Equal(true))
			Expect(l.License.Name).To(Equal("Not found"))
		})

		g.It("should unmarshal a scan results with virus data", func() {
			var r Results
			err := json.Unmarshal([]byte(SampleValidScanResultsVirus), &r)

			Expect(err).To(BeNil())
			Expect(r.Type).To(Equal("virus"))

			v, ok := r.Data.(VirusResults)
			Expect(ok).To(Equal(true))
			Expect(v.KnownViruses).To(Equal(10))
		})

		g.It("should unmarshal a scan results with vulnerability data", func() {
			var r Results
			err := json.Unmarshal([]byte(SampleValidScanResultsVulnerability), &r)

			Expect(err).To(BeNil())
			Expect(r.Type).To(Equal("vulnerability"))

			v, ok := r.Data.(VulnerabilityResults)
			Expect(ok).To(Equal(true))
			Expect(v.Meta.VulnerabilityCount).To(Equal(1))
		})

		g.It("should return an error for an invalid results type", func() {
			var r Results
			err := json.Unmarshal([]byte(SampleInvalidResults), &r)

			Expect(err).NotTo(BeNil())
			Expect(err.Error()).To(ContainSubstring("unsupported results type found:"))
		})
	})
}

const (
	SampleValidScanResultsAboutYML      = `{"type":"about_yml", "data":{"message": "foo message", "valid": true, "content": "some content"}}`
	SampleValidScanResultsCommunity     = `{"type":"community", "data":[{"committers":7,"name":"ion-channel/ion-connect","url":"https://github.com/ion-channel/ion-connect"}]}`
	SampleValidScanResultsCoverage      = `{"type":"external_coverage", "data":{"value":42.0}}`
	SampleValidScanResultsDependency    = `{"type":"dependency","data":{"dependencies":[{"latest_version":"2.0","org":"net.sourceforge.javacsv","name":"javacsv","type":"maven","package":"jar","version":"2.0","scope":"compile"},{"latest_version":"4.12","org":"junit","name":"junit","type":"maven","package":"jar","version":"4.11","scope":"test"},{"latest_version":"1.4-atlassian-1","org":"org.hamcrest","name":"hamcrest-core","type":"maven","package":"jar","version":"1.3","scope":"test"},{"latest_version":"4.5.2","org":"org.apache.httpcomponents","name":"httpclient","type":"maven","package":"jar","version":"4.3.4","scope":"compile"},{"latest_version":"4.4.5","org":"org.apache.httpcomponents","name":"httpcore","type":"maven","package":"jar","version":"4.3.2","scope":"compile"},{"latest_version":"99.0-does-not-exist","org":"commons-logging","name":"commons-logging","type":"maven","package":"jar","version":"1.1.3","scope":"compile"},{"latest_version":"20041127.091804","org":"commons-codec","name":"commons-codec","type":"maven","package":"jar","version":"1.6","scope":"compile"}],"meta":{"first_degree_count":3,"no_version_count":0,"total_unique_count":7,"update_available_count":2}}}`
	SampleValidScanResultsEcosystems    = `{"type":"ecosystems","data":{"ecosystems":[{"ecosystem":"Java","lines":2430},{"ecosystem":"Makefile","lines":210},{"ecosystem":"Ruby","lines":666}]}}`
	SampleValidScanResultsLicense       = `{"type":"license","data":{"license":{"name":"Not found","type":[]}}}`
	SampleValidScanResultsVirus         = `{"type":"virus","data":{"known_viruses":10,"engine_version":"","scanned_directories":1,"scanned_files":2,"infected_files":1,"data_scanned":"some cool data was scanned","data_read":"we read some data","time":"10PM","file_notes": {"empty_file":["file1","file2","file3"]},"clam_av_details":{"clamav_version":"1.0.0","clamav_db_version":"1.1.0"}}}`
	SampleValidScanResultsVulnerability = `{"type":"vulnerability","data":{"vulnerabilities":[{"id":316274974,"name":"hadoop","org":"apache","version":"2.8.0","up":null,"edition":null,"aliases":null,"created_at":"2017-02-13T20:02:32.785Z","updated_at":"2017-02-13T20:02:32.785Z","title":null,"references":null,"part":null,"language":null,"source_id":1,"external_id":"cpe:/a:apache:hadoop:2.8.0","vulnerabilities":[{"id":92596,"external_id":"CVE-2017-7669","title":"CVE-2017-7669","summary":"In Apache Hadoop 2.8.0, 3.0.0-alpha1, and 3.0.0-alpha2, the LinuxContainerExecutor runs docker commands as root with insufficient input validation. When the docker feature is enabled, authenticated users can run commands as root.","score":"8.5","score_version":"2.0","score_system":"CVSS","score_details":{"cvssv2":{"vectorString":"(AV:N/AC:M/Au:S/C:C/I:C/A:C)","accessVector":"NETWORK","accessComplexity":"MEDIUM","authentication":"SINGLE","confidentialityImpact":"COMPLETE","integrityImpact":"COMPLETE","availabilityImpact":"COMPLETE","baseScore":8.5},"cvssv3":{"vectorString":"AV:N/AC:H/PR:L/UI:N/S:U/C:H/I:H/A:H","attackVector":"NETWORK","attackComplexity":"HIGH","privilegesRequired":"LOW","userInteraction":"NONE","scope":"UNCHANGED","confidentialityImpact":"HIGH","integrityImpact":"HIGH","availabilityImpact":"HIGH","baseScore":7.5,"baseSeverity":"HIGH"}},"vector":"NETWORK","access_complexity":"MEDIUM","vulnerability_authentication":"SINGLE","confidentiality_impact":"COMPLETE","integrity_impact":"COMPLETE","availability_impact":"COMPLETE","vulnerability_source":null,"assessment_check":null,"scanner":null,"recommendation":"","references":[{"type":"UNKNOWN","source":"","url":"http://www.securityfocus.com/bid/98795","text":"http://www.securityfocus.com/bid/98795"},{"type":"UNKNOWN","source":"","url":"https://mail-archives.apache.org/mod_mbox/hadoop-user/201706.mbox/%3C4A2FDA56-491B-4C2A-915F-C9D4A4BDB92A%40apache.org%3E","text":"https://mail-archives.apache.org/mod_mbox/hadoop-user/201706.mbox/%3C4A2FDA56-491B-4C2A-915F-C9D4A4BDB92A%40apache.org%3E"}],"modified_at":"2017-06-09T16:21:00.000Z","published_at":"2017-06-05T01:29:00.000Z","created_at":"2017-07-12T23:07:35.491Z","updated_at":"2017-07-12T23:07:35.491Z","source_id":1}]}],"meta":{"vulnerability_count":1}}}`
	SampleInvalidResults                = `{"type":"fooresult", "data":"I pitty the foo"}`
	SampleValidScanResultsDifference    = `{"data": {"checksum": "checksumishere","difference": true},"type": "difference"}`
	SampleValidExternalVulnerabilities  = `{"type":"external_vulnerability","data":{"critical":1,"high":0,"medium":1,"low": 0}}`
)
<|endoftext|>


<commit_before>package caa

// CoverArtInfo is the unmarshaled representation of a JSON file in the Cover Art Archive.
// See https://musicbrainz.org/doc/Cover_Art_Archive/API#Cover_Art_Archive_Metadata for an example.
type CoverArtInfo struct {
	Images  []CoverArtImageInfo
	Release string
}

// CoverArtImageInfo is the unmarshaled representation of a single images metadata in a CAA JSON file.
// See https://musicbrainz.org/doc/Cover_Art_Archive/API#Cover_Art_Archive_Metadata for an example.
type CoverArtImageInfo struct {
	Types      []string
	Front      bool
	Back       bool
	Comment    string
	Thumbnails thumbnailMap
	Approved   bool
	Edit       int
}

// CoverArtImage is a wrapper around an image from the CAA, containing its binary data and mimetype information.
type CoverArtImage struct {
	Mimetype string
	Data     []byte
}

type thumbnailMap map[string]string
<commit_msg>Forgot the most important part - the link to the image<commit_after>package caa

// CoverArtInfo is the unmarshaled representation of a JSON file in the Cover Art Archive.
// See https://musicbrainz.org/doc/Cover_Art_Archive/API#Cover_Art_Archive_Metadata for an example.
type CoverArtInfo struct {
	Images  []CoverArtImageInfo
	Release string
}

// CoverArtImageInfo is the unmarshaled representation of a single images metadata in a CAA JSON file.
// See https://musicbrainz.org/doc/Cover_Art_Archive/API#Cover_Art_Archive_Metadata for an example.
type CoverArtImageInfo struct {
	Types      []string
	Front      bool
	Back       bool
	Comment    string
	Image      string
	Thumbnails thumbnailMap
	Approved   bool
	Edit       int
}

// CoverArtImage is a wrapper around an image from the CAA, containing its binary data and mimetype information.
type CoverArtImage struct {
	Mimetype string
	Data     []byte
}

type thumbnailMap map[string]string
<|endoftext|>


<commit_before>// Copyright ©2015 The corridor Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package corridor

import (
	"github.com/gonum/matrix/mat64"
	"github.com/satori/go.uuid"
)

// parameters are comprised of fixed input avlues that are
// unique to the problem specification that are referenced
// by the algorithm at various stage of the solution process
type Parameters struct {
	SrcSubs []int
	DstSubs []int
	RndCoef float64
	PopSize int
	SelFrac float64
	SelProb float64
	MutaCnt int
	MutaFrc float64
	EvoSize int
	ConSize int
}

// domains are comprised of boolean arrays which indicate the
// feasible locations for the search algorithm
type Domain struct {
	Rows   int
	Cols   int
	Matrix *mat64.Dense
	BndCnt int
}

// objectives are comprised of matrices which use location
// indices to key to floating point fitness values within the
// search domain
type Objective struct {
	Id     int
	Matrix *mat64.Dense
}

// multiObjective objects are comprised of a channel of individual
// independent objectives that are used for the evaluation of
// chromosome and population level fitness values
type MultiObjective struct {
	ObjectiveCount int
	Objectives     []*Objective
}

// a basis solution is comprised of the subscript indices forming
// the euclidean shortest path connecting the source to the dest
type Basis struct {
	Matrix *mat64.Dense
	Subs   [][]int
	MaxLen int
}

// chromosomess are comprised of genes which are distinct row column
// indices to some spatially reference search domain.
type Chromosome struct {
	Id               uuid.UUID
	Subs             [][]int
	Fitness          [][]float64
	TotalFitness     []float64
	AggregateFitness float64
}

// populations are comprised of a fixed number of chromosomes.
// this number corresponds to the populationSize.
type Population struct {
	Id                   int
	Chromosomes          chan *Chromosome
	MeanFitness          []float64
	AggregateMeanFitness float64
}

// evolutions are comprised of a stochastic number of populations.
// this number is determined by the convergence rate of the
// algorithm.
type Evolution struct {
	Populations     chan *Population
	FitnessGradient []float64
}
<commit_msg>Updated comment blocks for types.go<commit_after>/* Copyright ©2015 The corridor Authors. All rights reserved.
Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file. */

package corridor

import (
	"github.com/gonum/matrix/mat64"
	"github.com/satori/go.uuid"
)

/* parameters are comprised of fixed input avlues that are
unique to the problem specification that are referenced
by the algorithm at various stage of the solution process */
type Parameters struct {
	SrcSubs []int
	DstSubs []int
	RndCoef float64
	PopSize int
	SelFrac float64
	SelProb float64
	MutaCnt int
	MutaFrc float64
	EvoSize int
	ConSize int
}

/* domains are comprised of boolean arrays which indicate the
feasible locations for the search algorithm */
type Domain struct {
	Rows   int
	Cols   int
	Matrix *mat64.Dense
	BndCnt int
}

/* objectives are comprised of matrices which use location
indices to key to floating point fitness values within the
search domain */
type Objective struct {
	Id     int
	Matrix *mat64.Dense
}

/* multiObjective objects are comprised of a channel of individual
independent objectives that are used for the evaluation of
chromosome and population level fitness values */
type MultiObjective struct {
	ObjectiveCount int
	Objectives     []*Objective
}

/* a basis solution is comprised of the subscript indices forming
the euclidean shortest path connecting the source to the dest */
type Basis struct {
	Matrix *mat64.Dense
	Subs   [][]int
	MaxLen int
}

/* chromosomess are comprised of genes which are distinct row column
indices to some spatially reference search domain */
type Chromosome struct {
	Id               uuid.UUID
	Subs             [][]int
	Fitness          [][]float64
	TotalFitness     []float64
	AggregateFitness float64
}

/* populations are comprised of a fixed number of chromosomes.
this number corresponds to the populationSize. */
type Population struct {
	Id                   int
	Chromosomes          chan *Chromosome
	MeanFitness          []float64
	AggregateMeanFitness float64
}

/* evolutions are comprised of a stochastic number of populations.
this number is determined by the convergence rate of the
algorithm */
type Evolution struct {
	Populations     chan *Population
	FitnessGradient []float64
}
<|endoftext|>


<commit_before>package parse

import (
	"encoding/json"
	"fmt"
	"time"
)

// An interface for custom Parse types. Contains a single method:
//
// ClassName() - returns a string containing the class name as it appears in your
// Parse database.
//
// Implement this interface if your class name does not match your struct
// name. If this class is not implemented, the name of the struct will
// be used when interacting with the Parse API
type iClassName interface {
	ClassName() string
}

// An interface for custom Parse types to override the endpoint used for querying.
//
// Contains a single method:
//
// Endpoint() - returns the endpoint to use when querying the Parse REST API.
//
// If this method is not implented, the endpoint is constructed as follows:
//
// /classes/{ClassName} - where {ClassName} is the name of the struct or the value returned by the ClassName
// method if implemented
type iParseEp interface {
	Endpoint() string
}

// A base type containing fields common to all Parse types
//
// Embed this struct in custom types
type Base struct {
	Id        string `parse:"objectId"`
	CreatedAt *time.Time
	UpdatedAt *time.Time
	Acl       *ACL
	Extra     map[string]interface{}
}

// Represents the built-in Parse "User" class
type User struct {
	Base
}

func (u *User) ClassName() string {
	return "_User"
}

func (u *User) Endpoint() string {
	return "users"
}

// Represents the built-in Parse "Installation" class
type Installation struct {
	Base
}

func (i *Installation) ClassName() string {
	return "_Installation"
}

func (i *Installation) Endpoint() string {
	return "installations"
}

type ACL struct {
}

type GeoPoint struct {
}

type File struct {
}

type Pointer struct {
	Id string
	ClassName string
}

func (p Pointer) MarshalJSON() ([]byte, error) {
	return json.Marshal(&struct {
		Type string `json:"__type"`
		ClassName string `json:"className"`
		Id string `json:"objectId"`
	}{
		"Pointer",
		p.ClassName,
		p.Id,
	})
}

type Date time.Time

func (d Date) MarshalJSON() ([]byte, error) {
	return json.Marshal(&struct {
		Type string `json:"__type"`
		Iso  string `json:"iso"`
	}{
		"Date",
		time.Time(d).Format(time.RFC3339),
	})
}

func (d *Date) UnmarshalJSON(b []byte) error {
	s := struct {
		Type string `json:"__type"`
		Iso  string `json:"iso"`
	}{}
	err := json.Unmarshal(b, &s)
	if err != nil {
		return err
	}
	if s.Type != "Date" {
		return fmt.Errorf("cannot unmarshal type %s to type Date", s.Type)
	}

	t, err := time.Parse(s.Iso, time.RFC3339Nano)
	if err != nil {
		return err
	}

	*d = Date(t)
	return nil
}
<commit_msg>update geopoint type<commit_after>package parse

import (
	"encoding/json"
	"fmt"
	"time"
)

// An interface for custom Parse types. Contains a single method:
//
// ClassName() - returns a string containing the class name as it appears in your
// Parse database.
//
// Implement this interface if your class name does not match your struct
// name. If this class is not implemented, the name of the struct will
// be used when interacting with the Parse API
type iClassName interface {
	ClassName() string
}

// An interface for custom Parse types to override the endpoint used for querying.
//
// Contains a single method:
//
// Endpoint() - returns the endpoint to use when querying the Parse REST API.
//
// If this method is not implented, the endpoint is constructed as follows:
//
// /classes/{ClassName} - where {ClassName} is the name of the struct or the value returned by the ClassName
// method if implemented
type iParseEp interface {
	Endpoint() string
}

// A base type containing fields common to all Parse types
//
// Embed this struct in custom types
type Base struct {
	Id        string `parse:"objectId"`
	CreatedAt *time.Time
	UpdatedAt *time.Time
	Acl       *ACL
	Extra     map[string]interface{}
}

// Represents the built-in Parse "User" class
type User struct {
	Base
}

func (u *User) ClassName() string {
	return "_User"
}

func (u *User) Endpoint() string {
	return "users"
}

// Represents the built-in Parse "Installation" class
type Installation struct {
	Base
}

func (i *Installation) ClassName() string {
	return "_Installation"
}

func (i *Installation) Endpoint() string {
	return "installations"
}

type ACL struct {
}

type GeoPoint struct {
	Latitude float64
	Longitude float64
}

func (g GeoPoint) MarshalJSON() ([]byte, error) {
	return json.Marshal(&struct {
		Type string `json:"__type"`
		Latitude float64 `json:"latitude"`
		Longitude float64 `json:"longitude"`
	}{
		"GeoPoint",
		g.Latitude,
		g.Longitude,
	})
}

func (g *GeoPoint) UnmarshalJSON(b []byte) error {
	s := struct {
		Type string `json:"__type"`
		Latitude float64 `json:"latitude"`
		Longitude float64 `json:"longitude"`
	}{}
	err := json.Unmarshal(b, &s)
	if err != nil {
		return err
	}
	if s.Type != "GeoPoint" {
		return fmt.Errorf("cannot unmarshal type %s to type GeoPoint", s.Type)
	}

	g.Latitude = s.Latitude
	g.Longitude = s.Longitude
	return nil
}

type File struct {
}

type Pointer struct {
	Id string
	ClassName string
}

func (p Pointer) MarshalJSON() ([]byte, error) {
	return json.Marshal(&struct {
		Type string `json:"__type"`
		ClassName string `json:"className"`
		Id string `json:"objectId"`
	}{
		"Pointer",
		p.ClassName,
		p.Id,
	})
}

type Date time.Time

func (d Date) MarshalJSON() ([]byte, error) {
	return json.Marshal(&struct {
		Type string `json:"__type"`
		Iso  string `json:"iso"`
	}{
		"Date",
		time.Time(d).Format(time.RFC3339),
	})
}

func (d *Date) UnmarshalJSON(b []byte) error {
	s := struct {
		Type string `json:"__type"`
		Iso  string `json:"iso"`
	}{}
	err := json.Unmarshal(b, &s)
	if err != nil {
		return err
	}
	if s.Type != "Date" {
		return fmt.Errorf("cannot unmarshal type %s to type Date", s.Type)
	}

	t, err := time.Parse(s.Iso, time.RFC3339Nano)
	if err != nil {
		return err
	}

	*d = Date(t)
	return nil
}
<|endoftext|>


<commit_before>package libcontainer

import (
	"errors"

	"github.com/syndtr/gocapability/capability"
)

var (
	ErrUnkownNamespace  = errors.New("Unknown namespace")
	ErrUnkownCapability = errors.New("Unknown capability")
	ErrUnsupported      = errors.New("Unsupported method")
)

type Mounts []Mount

func (s Mounts) OfType(t string) Mounts {
	out := Mounts{}
	for _, m := range s {
		if m.Type == t {
			out = append(out, m)
		}
	}
	return out
}

type Mount struct {
	Type        string `json:"type,omitempty"`
	Source      string `json:"source,omitempty"`      // Source path, in the host namespace
	Destination string `json:"destination,omitempty"` // Destination path, in the container
	Writable    bool   `json:"writable,omitempty"`
	Private     bool   `json:"private,omitempty"`
}

// namespaceList is used to convert the libcontainer types
// into the names of the files located in /proc/<pid>/ns/* for
// each namespace
var (
	namespaceList = Namespaces{}

	capabilityList = Capabilities{
		{Key: "SETPCAP", Value: capability.CAP_SETPCAP},
		{Key: "SYS_MODULE", Value: capability.CAP_SYS_MODULE},
		{Key: "SYS_RAWIO", Value: capability.CAP_SYS_RAWIO},
		{Key: "SYS_PACCT", Value: capability.CAP_SYS_PACCT},
		{Key: "SYS_ADMIN", Value: capability.CAP_SYS_ADMIN},
		{Key: "SYS_NICE", Value: capability.CAP_SYS_NICE},
		{Key: "SYS_RESOURCE", Value: capability.CAP_SYS_RESOURCE},
		{Key: "SYS_TIME", Value: capability.CAP_SYS_TIME},
		{Key: "SYS_TTY_CONFIG", Value: capability.CAP_SYS_TTY_CONFIG},
		{Key: "MKNOD", Value: capability.CAP_MKNOD},
		{Key: "AUDIT_WRITE", Value: capability.CAP_AUDIT_WRITE},
		{Key: "AUDIT_CONTROL", Value: capability.CAP_AUDIT_CONTROL},
		{Key: "MAC_OVERRIDE", Value: capability.CAP_MAC_OVERRIDE},
		{Key: "MAC_ADMIN", Value: capability.CAP_MAC_ADMIN},
		{Key: "NET_ADMIN", Value: capability.CAP_NET_ADMIN},
		{Key: "SYSLOG", Value: capability.CAP_SYSLOG},
		{Key: "SETUID", Value: capability.CAP_SETUID},
		{Key: "SETGID", Value: capability.CAP_SETGID},
		{Key: "CHOWN", Value: capability.CAP_CHOWN},
		{Key: "NET_RAW", Value: capability.CAP_NET_RAW},
		{Key: "DAC_OVERRIDE", Value: capability.CAP_DAC_OVERRIDE},
		{Key: "FOWNER", Value: capability.CAP_FOWNER},
	}
)

type (
	Namespace struct {
		Key   string `json:"key,omitempty"`
		Value int    `json:"value,omitempty"`
		File  string `json:"file,omitempty"`
	}
	Namespaces []*Namespace
)

func (ns *Namespace) String() string {
	return ns.Key
}

func GetNamespace(key string) *Namespace {
	for _, ns := range namespaceList {
		if ns.Key == key {
			cpy := *ns
			return &cpy
		}
	}
	return nil
}

// Contains returns true if the specified Namespace is
// in the slice
func (n Namespaces) Contains(ns string) bool {
	return n.Get(ns) != nil
}

func (n Namespaces) Get(ns string) *Namespace {
	for _, nsp := range n {
		if nsp != nil && nsp.Key == ns {
			return nsp
		}
	}
	return nil
}

type (
	Capability struct {
		Key   string         `json:"key,omitempty"`
		Value capability.Cap `json:"value,omitempty"`
	}
	Capabilities []*Capability
)

func (c *Capability) String() string {
	return c.Key
}

func GetCapability(key string) *Capability {
	for _, capp := range capabilityList {
		if capp.Key == key {
			cpy := *capp
			return &cpy
		}
	}
	return nil
}

func GetAllCapabilities() []string {
	output := make([]string, len(capabilityList))
	for i, capability := range capabilityList {
		output[i] = capability.String()
	}
	return output
}

// Contains returns true if the specified Capability is
// in the slice
func (c Capabilities) Contains(capp string) bool {
	return c.Get(capp) != nil
}

func (c Capabilities) Get(capp string) *Capability {
	for _, cap := range c {
		if cap.Key == capp {
			return cap
		}
	}
	return nil
}
<commit_msg>Add the rest of the caps so that they are retained in privilged mode<commit_after>package libcontainer

import (
	"errors"

	"github.com/syndtr/gocapability/capability"
)

var (
	ErrUnkownNamespace  = errors.New("Unknown namespace")
	ErrUnkownCapability = errors.New("Unknown capability")
	ErrUnsupported      = errors.New("Unsupported method")
)

type Mounts []Mount

func (s Mounts) OfType(t string) Mounts {
	out := Mounts{}
	for _, m := range s {
		if m.Type == t {
			out = append(out, m)
		}
	}
	return out
}

type Mount struct {
	Type        string `json:"type,omitempty"`
	Source      string `json:"source,omitempty"`      // Source path, in the host namespace
	Destination string `json:"destination,omitempty"` // Destination path, in the container
	Writable    bool   `json:"writable,omitempty"`
	Private     bool   `json:"private,omitempty"`
}

// namespaceList is used to convert the libcontainer types
// into the names of the files located in /proc/<pid>/ns/* for
// each namespace
var (
	namespaceList = Namespaces{}

	capabilityList = Capabilities{
		{Key: "SETPCAP", Value: capability.CAP_SETPCAP},
		{Key: "SYS_MODULE", Value: capability.CAP_SYS_MODULE},
		{Key: "SYS_RAWIO", Value: capability.CAP_SYS_RAWIO},
		{Key: "SYS_PACCT", Value: capability.CAP_SYS_PACCT},
		{Key: "SYS_ADMIN", Value: capability.CAP_SYS_ADMIN},
		{Key: "SYS_NICE", Value: capability.CAP_SYS_NICE},
		{Key: "SYS_RESOURCE", Value: capability.CAP_SYS_RESOURCE},
		{Key: "SYS_TIME", Value: capability.CAP_SYS_TIME},
		{Key: "SYS_TTY_CONFIG", Value: capability.CAP_SYS_TTY_CONFIG},
		{Key: "MKNOD", Value: capability.CAP_MKNOD},
		{Key: "AUDIT_WRITE", Value: capability.CAP_AUDIT_WRITE},
		{Key: "AUDIT_CONTROL", Value: capability.CAP_AUDIT_CONTROL},
		{Key: "MAC_OVERRIDE", Value: capability.CAP_MAC_OVERRIDE},
		{Key: "MAC_ADMIN", Value: capability.CAP_MAC_ADMIN},
		{Key: "NET_ADMIN", Value: capability.CAP_NET_ADMIN},
		{Key: "SYSLOG", Value: capability.CAP_SYSLOG},
		{Key: "SETUID", Value: capability.CAP_SETUID},
		{Key: "SETGID", Value: capability.CAP_SETGID},
		{Key: "CHOWN", Value: capability.CAP_CHOWN},
		{Key: "NET_RAW", Value: capability.CAP_NET_RAW},
		{Key: "DAC_OVERRIDE", Value: capability.CAP_DAC_OVERRIDE},
		{Key: "FOWNER", Value: capability.CAP_FOWNER},
		{Key: "DAC_READ_SEARCH", Value: capability.CAP_DAC_READ_SEARCH},
		{Key: "FSETID", Value: capability.CAP_FSETID},
		{Key: "KILL", Value: capability.CAP_KILL},
		{Key: "SETGID", Value: capability.CAP_SETGID},
		{Key: "SETUID", Value: capability.CAP_SETUID},
		{Key: "LINUX_IMMUTABLE", Value: capability.CAP_LINUX_IMMUTABLE},
		{Key: "NET_BIND_SERVICE", Value: capability.CAP_NET_BIND_SERVICE},
		{Key: "NET_BROADCAST", Value: capability.CAP_NET_BROADCAST},
		{Key: "IPC_LOCK", Value: capability.CAP_IPC_LOCK},
		{Key: "IPC_OWNER", Value: capability.CAP_IPC_OWNER},
		{Key: "SYS_CHROOT", Value: capability.CAP_SYS_CHROOT},
		{Key: "SYS_PTRACE", Value: capability.CAP_SYS_PTRACE},
		{Key: "SYS_BOOT", Value: capability.CAP_SYS_BOOT},
		{Key: "LEASE", Value: capability.CAP_LEASE},
		{Key: "SETFCAP", Value: capability.CAP_SETFCAP},
		{Key: "WAKE_ALARM", Value: capability.CAP_WAKE_ALARM},
		{Key: "BLOCK_SUSPEND", Value: capability.CAP_BLOCK_SUSPEND},
	}
)

type (
	Namespace struct {
		Key   string `json:"key,omitempty"`
		Value int    `json:"value,omitempty"`
		File  string `json:"file,omitempty"`
	}
	Namespaces []*Namespace
)

func (ns *Namespace) String() string {
	return ns.Key
}

func GetNamespace(key string) *Namespace {
	for _, ns := range namespaceList {
		if ns.Key == key {
			cpy := *ns
			return &cpy
		}
	}
	return nil
}

// Contains returns true if the specified Namespace is
// in the slice
func (n Namespaces) Contains(ns string) bool {
	return n.Get(ns) != nil
}

func (n Namespaces) Get(ns string) *Namespace {
	for _, nsp := range n {
		if nsp != nil && nsp.Key == ns {
			return nsp
		}
	}
	return nil
}

type (
	Capability struct {
		Key   string         `json:"key,omitempty"`
		Value capability.Cap `json:"value,omitempty"`
	}
	Capabilities []*Capability
)

func (c *Capability) String() string {
	return c.Key
}

func GetCapability(key string) *Capability {
	for _, capp := range capabilityList {
		if capp.Key == key {
			cpy := *capp
			return &cpy
		}
	}
	return nil
}

func GetAllCapabilities() []string {
	output := make([]string, len(capabilityList))
	for i, capability := range capabilityList {
		output[i] = capability.String()
	}
	return output
}

// Contains returns true if the specified Capability is
// in the slice
func (c Capabilities) Contains(capp string) bool {
	return c.Get(capp) != nil
}

func (c Capabilities) Get(capp string) *Capability {
	for _, cap := range c {
		if cap.Key == capp {
			return cap
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package msgpack

import (
	"reflect"
	"sync"
)

var errorType = reflect.TypeOf((*error)(nil)).Elem()

var customEncoderType = reflect.TypeOf((*CustomEncoder)(nil)).Elem()
var customDecoderType = reflect.TypeOf((*CustomDecoder)(nil)).Elem()

var marshalerType = reflect.TypeOf((*Marshaler)(nil)).Elem()
var unmarshalerType = reflect.TypeOf((*Unmarshaler)(nil)).Elem()

type encoderFunc func(*Encoder, reflect.Value) error
type decoderFunc func(*Decoder, reflect.Value) error

var typEncMap = make(map[reflect.Type]encoderFunc)
var typDecMap = make(map[reflect.Type]decoderFunc)

// Register registers encoder and decoder functions for a value.
// This is low level API and in most cases you should prefer implementing
// Marshaler/CustomEncoder and Unmarshaler/CustomDecoder interfaces.
func Register(value interface{}, enc encoderFunc, dec decoderFunc) {
	typ := reflect.TypeOf(value)
	if enc != nil {
		typEncMap[typ] = enc
	}
	if dec != nil {
		typDecMap[typ] = dec
	}
}

//------------------------------------------------------------------------------

var structs = newStructCache(false)
var jsonStructs = newStructCache(true)

type structCache struct {
	mu sync.RWMutex
	m  map[reflect.Type]*fields

	useJSONTag bool
}

func newStructCache(useJSONTag bool) *structCache {
	return &structCache{
		m: make(map[reflect.Type]*fields),

		useJSONTag: useJSONTag,
	}
}

func (m *structCache) Fields(typ reflect.Type) *fields {
	m.mu.RLock()
	fs, ok := m.m[typ]
	m.mu.RUnlock()
	if ok {
		return fs
	}

	m.mu.Lock()
	fs, ok = m.m[typ]
	if !ok {
		fs = getFields(typ, m.useJSONTag)
		m.m[typ] = fs
	}
	m.mu.Unlock()

	return fs
}

//------------------------------------------------------------------------------

type field struct {
	name      string
	index     []int
	omitEmpty bool
	encoder   encoderFunc
	decoder   decoderFunc
}

func (f *field) value(v reflect.Value) reflect.Value {
	return fieldByIndex(v, f.index)
}

func (f *field) Omit(strct reflect.Value) bool {
	return f.omitEmpty && isEmptyValue(f.value(strct))
}

func (f *field) EncodeValue(e *Encoder, strct reflect.Value) error {
	return f.encoder(e, f.value(strct))
}

func (f *field) DecodeValue(d *Decoder, strct reflect.Value) error {
	return f.decoder(d, f.value(strct))
}

//------------------------------------------------------------------------------

type fields struct {
	Table   map[string]*field
	List    []*field
	AsArray bool

	hasOmitEmpty bool
}

func newFields(numField int) *fields {
	return &fields{
		Table: make(map[string]*field, numField),
		List:  make([]*field, 0, numField),
	}
}

func (fs *fields) Add(field *field) {
	fs.Table[field.name] = field
	fs.List = append(fs.List, field)
	if field.omitEmpty {
		fs.hasOmitEmpty = true
	}
}

func (fs *fields) OmitEmpty(strct reflect.Value) []*field {
	if !fs.hasOmitEmpty {
		return fs.List
	}

	fields := make([]*field, 0, len(fs.List))
	for _, f := range fs.List {
		if !f.Omit(strct) {
			fields = append(fields, f)
		}
	}
	return fields
}

func getFields(typ reflect.Type, useJSONTag bool) *fields {
	numField := typ.NumField()
	fs := newFields(numField)

	var omitEmpty bool
	for i := 0; i < numField; i++ {
		f := typ.Field(i)

		tag := f.Tag.Get("msgpack")
		if useJSONTag && tag == "" {
			tag = f.Tag.Get("json")
		}

		name, opt := parseTag(tag)
		if name == "-" {
			continue
		}

		if f.Name == "_msgpack" {
			if opt.Contains("asArray") {
				fs.AsArray = true
			}
			if opt.Contains("omitempty") {
				omitEmpty = true
			}
		}

		if f.PkgPath != "" && !f.Anonymous {
			continue
		}

		field := &field{
			name:      name,
			index:     f.Index,
			omitEmpty: omitEmpty || opt.Contains("omitempty"),
			encoder:   getEncoder(f.Type),
			decoder:   getDecoder(f.Type),
		}

		if field.name == "" {
			field.name = f.Name
		}

		if f.Anonymous {
			if opt.Contains("inline") {
				inlineFields(fs, f.Type, field, useJSONTag)
				continue
			}
			if autoinlineFields(fs, f.Type, field, useJSONTag) {
				continue
			}
		}

		fs.Add(field)
	}
	return fs
}

var encodeStructValuePtr uintptr
var decodeStructValuePtr uintptr

func init() {
	encodeStructValuePtr = reflect.ValueOf(encodeStructValue).Pointer()
	decodeStructValuePtr = reflect.ValueOf(decodeStructValue).Pointer()
}

func inlineFields(fs *fields, typ reflect.Type, f *field, useJSONTag bool) {
	inlinedFields := getFields(typ, useJSONTag).List
	for _, field := range inlinedFields {
		if _, ok := fs.Table[field.name]; ok {
			// Don't inline shadowed fields.
			continue
		}
		field.index = append(f.index, field.index...)
		fs.Add(field)
	}
}

func autoinlineFields(fs *fields, typ reflect.Type, f *field, useJSONTag bool) bool {
	var encoder encoderFunc
	var decoder decoderFunc

	if typ.Kind() == reflect.Struct {
		encoder = f.encoder
		decoder = f.decoder
	} else {
		for typ.Kind() == reflect.Ptr {
			typ = typ.Elem()
			encoder = getEncoder(typ)
			decoder = getDecoder(typ)
		}
		if typ.Kind() != reflect.Struct {
			return false
		}
	}

	if reflect.ValueOf(encoder).Pointer() != encodeStructValuePtr {
		return false
	}
	if reflect.ValueOf(decoder).Pointer() != decodeStructValuePtr {
		return false
	}

	inlinedFields := getFields(typ, useJSONTag).List
	for _, field := range inlinedFields {
		if _, ok := fs.Table[field.name]; ok {
			// Don't inline shadowed fields.
			return false
		}
	}

	for _, field := range inlinedFields {
		field.index = append(f.index, field.index...)
		fs.Add(field)
	}
	return true
}

func isEmptyValue(v reflect.Value) bool {
	switch v.Kind() {
	case reflect.Array, reflect.Map, reflect.Slice, reflect.String:
		return v.Len() == 0
	case reflect.Bool:
		return !v.Bool()
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		return v.Int() == 0
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
		return v.Uint() == 0
	case reflect.Float32, reflect.Float64:
		return v.Float() == 0
	case reflect.Interface, reflect.Ptr:
		return v.IsNil()
	}
	return false
}

func fieldByIndex(v reflect.Value, index []int) reflect.Value {
	if len(index) == 1 {
		return v.Field(index[0])
	}
	for i, x := range index {
		if i > 0 {
			var ok bool
			v, ok = indirectNew(v)
			if !ok {
				return v
			}
		}
		v = v.Field(x)
	}
	return v
}

func indirectNew(v reflect.Value) (reflect.Value, bool) {
	if v.Kind() == reflect.Ptr {
		if v.IsNil() {
			if !v.CanSet() {
				return v, false
			}
			elemType := v.Type().Elem()
			if elemType.Kind() != reflect.Struct {
				return v, false
			}
			v.Set(reflect.New(elemType))
		}
		v = v.Elem()
	}
	return v, true
}
<commit_msg>Allow decoding inlined structs<commit_after>package msgpack

import (
	"reflect"
	"sync"
)

var errorType = reflect.TypeOf((*error)(nil)).Elem()

var customEncoderType = reflect.TypeOf((*CustomEncoder)(nil)).Elem()
var customDecoderType = reflect.TypeOf((*CustomDecoder)(nil)).Elem()

var marshalerType = reflect.TypeOf((*Marshaler)(nil)).Elem()
var unmarshalerType = reflect.TypeOf((*Unmarshaler)(nil)).Elem()

type encoderFunc func(*Encoder, reflect.Value) error
type decoderFunc func(*Decoder, reflect.Value) error

var typEncMap = make(map[reflect.Type]encoderFunc)
var typDecMap = make(map[reflect.Type]decoderFunc)

// Register registers encoder and decoder functions for a value.
// This is low level API and in most cases you should prefer implementing
// Marshaler/CustomEncoder and Unmarshaler/CustomDecoder interfaces.
func Register(value interface{}, enc encoderFunc, dec decoderFunc) {
	typ := reflect.TypeOf(value)
	if enc != nil {
		typEncMap[typ] = enc
	}
	if dec != nil {
		typDecMap[typ] = dec
	}
}

//------------------------------------------------------------------------------

var structs = newStructCache(false)
var jsonStructs = newStructCache(true)

type structCache struct {
	mu sync.RWMutex
	m  map[reflect.Type]*fields

	useJSONTag bool
}

func newStructCache(useJSONTag bool) *structCache {
	return &structCache{
		m: make(map[reflect.Type]*fields),

		useJSONTag: useJSONTag,
	}
}

func (m *structCache) Fields(typ reflect.Type) *fields {
	m.mu.RLock()
	fs, ok := m.m[typ]
	m.mu.RUnlock()
	if ok {
		return fs
	}

	m.mu.Lock()
	fs, ok = m.m[typ]
	if !ok {
		fs = getFields(typ, m.useJSONTag)
		m.m[typ] = fs
	}
	m.mu.Unlock()

	return fs
}

//------------------------------------------------------------------------------

type field struct {
	name      string
	index     []int
	omitEmpty bool
	encoder   encoderFunc
	decoder   decoderFunc
}

func (f *field) value(v reflect.Value) reflect.Value {
	return fieldByIndex(v, f.index)
}

func (f *field) Omit(strct reflect.Value) bool {
	return f.omitEmpty && isEmptyValue(f.value(strct))
}

func (f *field) EncodeValue(e *Encoder, strct reflect.Value) error {
	return f.encoder(e, f.value(strct))
}

func (f *field) DecodeValue(d *Decoder, strct reflect.Value) error {
	return f.decoder(d, f.value(strct))
}

//------------------------------------------------------------------------------

type fields struct {
	Table   map[string]*field
	List    []*field
	AsArray bool

	hasOmitEmpty bool
}

func newFields(numField int) *fields {
	return &fields{
		Table: make(map[string]*field, numField),
		List:  make([]*field, 0, numField),
	}
}

func (fs *fields) Add(field *field) {
	fs.Table[field.name] = field
	fs.List = append(fs.List, field)
	if field.omitEmpty {
		fs.hasOmitEmpty = true
	}
}

func (fs *fields) OmitEmpty(strct reflect.Value) []*field {
	if !fs.hasOmitEmpty {
		return fs.List
	}

	fields := make([]*field, 0, len(fs.List))
	for _, f := range fs.List {
		if !f.Omit(strct) {
			fields = append(fields, f)
		}
	}
	return fields
}

func getFields(typ reflect.Type, useJSONTag bool) *fields {
	numField := typ.NumField()
	fs := newFields(numField)

	var omitEmpty bool
	for i := 0; i < numField; i++ {
		f := typ.Field(i)

		tag := f.Tag.Get("msgpack")
		if useJSONTag && tag == "" {
			tag = f.Tag.Get("json")
		}

		name, opt := parseTag(tag)
		if name == "-" {
			continue
		}

		if f.Name == "_msgpack" {
			if opt.Contains("asArray") {
				fs.AsArray = true
			}
			if opt.Contains("omitempty") {
				omitEmpty = true
			}
		}

		if f.PkgPath != "" && !f.Anonymous {
			continue
		}

		field := &field{
			name:      name,
			index:     f.Index,
			omitEmpty: omitEmpty || opt.Contains("omitempty"),
			encoder:   getEncoder(f.Type),
			decoder:   getDecoder(f.Type),
		}

		if field.name == "" {
			field.name = f.Name
		}

		if f.Anonymous && !opt.Contains("noinline") {
			inline := opt.Contains("inline")
			if inline {
				inlineFields(fs, f.Type, field, useJSONTag)
			} else {
				inline = autoinlineFields(fs, f.Type, field, useJSONTag)
			}
			if inline {
				fs.Table[field.name] = field
				continue
			}
		}

		fs.Add(field)
	}
	return fs
}

var encodeStructValuePtr uintptr
var decodeStructValuePtr uintptr

func init() {
	encodeStructValuePtr = reflect.ValueOf(encodeStructValue).Pointer()
	decodeStructValuePtr = reflect.ValueOf(decodeStructValue).Pointer()
}

func inlineFields(fs *fields, typ reflect.Type, f *field, useJSONTag bool) {
	inlinedFields := getFields(typ, useJSONTag).List
	for _, field := range inlinedFields {
		if _, ok := fs.Table[field.name]; ok {
			// Don't inline shadowed fields.
			continue
		}
		field.index = append(f.index, field.index...)
		fs.Add(field)
	}
}

func autoinlineFields(fs *fields, typ reflect.Type, f *field, useJSONTag bool) bool {
	var encoder encoderFunc
	var decoder decoderFunc

	if typ.Kind() == reflect.Struct {
		encoder = f.encoder
		decoder = f.decoder
	} else {
		for typ.Kind() == reflect.Ptr {
			typ = typ.Elem()
			encoder = getEncoder(typ)
			decoder = getDecoder(typ)
		}
		if typ.Kind() != reflect.Struct {
			return false
		}
	}

	if reflect.ValueOf(encoder).Pointer() != encodeStructValuePtr {
		return false
	}
	if reflect.ValueOf(decoder).Pointer() != decodeStructValuePtr {
		return false
	}

	inlinedFields := getFields(typ, useJSONTag).List
	for _, field := range inlinedFields {
		if _, ok := fs.Table[field.name]; ok {
			// Don't auto inline if there are shadowed fields.
			return false
		}
	}

	for _, field := range inlinedFields {
		field.index = append(f.index, field.index...)
		fs.Add(field)
	}
	return true
}

func isEmptyValue(v reflect.Value) bool {
	switch v.Kind() {
	case reflect.Array, reflect.Map, reflect.Slice, reflect.String:
		return v.Len() == 0
	case reflect.Bool:
		return !v.Bool()
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		return v.Int() == 0
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
		return v.Uint() == 0
	case reflect.Float32, reflect.Float64:
		return v.Float() == 0
	case reflect.Interface, reflect.Ptr:
		return v.IsNil()
	}
	return false
}

func fieldByIndex(v reflect.Value, index []int) reflect.Value {
	if len(index) == 1 {
		return v.Field(index[0])
	}
	for i, x := range index {
		if i > 0 {
			var ok bool
			v, ok = indirectNew(v)
			if !ok {
				return v
			}
		}
		v = v.Field(x)
	}
	return v
}

func indirectNew(v reflect.Value) (reflect.Value, bool) {
	if v.Kind() == reflect.Ptr {
		if v.IsNil() {
			if !v.CanSet() {
				return v, false
			}
			elemType := v.Type().Elem()
			if elemType.Kind() != reflect.Struct {
				return v, false
			}
			v.Set(reflect.New(elemType))
		}
		v = v.Elem()
	}
	return v, true
}
<|endoftext|>


<commit_before>package cluster

import (
	"encoding/binary"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"sync"
	"time"

	"github.com/golang/protobuf/proto"
)

// Transport is the interface the network layer must provide.
type Transport interface {
	net.Listener

	// Dial is used to create a connection to a service listening
	// on an address.
	Dial(address string, timeout time.Duration) (net.Conn, error)
}

// Service provides information about the node and cluster.
type Service struct {
	tn      Transport // Network layer this service uses
	addr    net.Addr  // Address on which this service is listening
	timeout time.Duration

	mu      sync.RWMutex
	https   bool   // Serving HTTPS?
	apiAddr string // host:port this node serves the HTTP API.

	wg sync.WaitGroup

	logger *log.Logger
}

// NewService returns a new instance of the cluster service
func NewService(tn Transport) *Service {
	return &Service{
		tn:      tn,
		addr:    tn.Addr(),
		timeout: 10 * time.Second,
		logger:  log.New(os.Stderr, "[cluster] ", log.LstdFlags),
	}
}

// Open opens the Service.
func (s *Service) Open() error {
	s.wg.Add(1)
	go s.serve()
	s.logger.Println("service listening on", s.tn.Addr())
	return nil
}

// Close closes the service.
func (s *Service) Close() error {
	s.tn.Close()
	s.wg.Wait()
	return nil
}

// Addr returns the address the service is listening on.
func (s *Service) Addr() string {
	return s.addr.String()
}

func (s *Service) EnableHTTPS(b bool) {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.https = b
}

func (s *Service) SetAPIAddr(addr string) {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.apiAddr = addr
}

func (s *Service) GetAPIAddr() string {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.apiAddr
}

func (s *Service) GetNodeAPIAddr(nodeAddr string) (string, error) {
	conn, err := s.tn.Dial(nodeAddr, s.timeout)
	if err != nil {
		return "", err
	}
	defer conn.Close()

	// Send the request
	c := &Command{
		Type: Command_COMMAND_TYPE_GET_NODE_API_URL,
	}
	p, err := proto.Marshal(c)
	if err != nil {
		conn.Close()
	}

	// Write length of Protobuf, the Protobuf
	b := make([]byte, 4)
	binary.LittleEndian.PutUint16(b[0:], uint16(len(p)))

	conn.Write(b)
	conn.Write(p)

	b, err = ioutil.ReadAll(conn)
	if err != nil {
		return "", err
	}

	a := &Address{}
	err = proto.Unmarshal(b, a)
	if err != nil {
		return "", err
	}

	return a.Url, nil
}

func (s *Service) serve() error {
	defer s.wg.Done()
	for {
		conn, err := s.tn.Accept()
		if err != nil {
			return err
		}

		go s.handleConn(conn)
	}
}

func (s *Service) handleConn(conn net.Conn) {
	b := make([]byte, 4)
	_, err := io.ReadFull(conn, b)
	if err != nil {
		conn.Close() // Only way to signal an error.
	}
	sz := binary.LittleEndian.Uint16(b[0:])

	b = make([]byte, sz)
	_, err = io.ReadFull(conn, b)
	if err != nil {
		conn.Close() // Only way to signal an error.
	}

	c := &Command{}
	err = proto.Unmarshal(b, c)
	if err != nil {
		conn.Close()
	}

	switch c.Type {
	case Command_COMMAND_TYPE_GET_NODE_API_URL:
		s.mu.RLock()
		defer s.mu.RUnlock()

		a := &Address{}
		scheme := "http"
		if s.https {
			scheme = "https"
		}
		a.Url = fmt.Sprintf("%s://%s", scheme, s.apiAddr)

		b, err = proto.Marshal(a)
		if err != nil {
			conn.Close()
		}
		conn.Write(b)
		conn.Close()
	}
}
<commit_msg>Defer conn close<commit_after>package cluster

import (
	"encoding/binary"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"sync"
	"time"

	"github.com/golang/protobuf/proto"
)

// Transport is the interface the network layer must provide.
type Transport interface {
	net.Listener

	// Dial is used to create a connection to a service listening
	// on an address.
	Dial(address string, timeout time.Duration) (net.Conn, error)
}

// Service provides information about the node and cluster.
type Service struct {
	tn      Transport // Network layer this service uses
	addr    net.Addr  // Address on which this service is listening
	timeout time.Duration

	mu      sync.RWMutex
	https   bool   // Serving HTTPS?
	apiAddr string // host:port this node serves the HTTP API.

	wg sync.WaitGroup

	logger *log.Logger
}

// NewService returns a new instance of the cluster service
func NewService(tn Transport) *Service {
	return &Service{
		tn:      tn,
		addr:    tn.Addr(),
		timeout: 10 * time.Second,
		logger:  log.New(os.Stderr, "[cluster] ", log.LstdFlags),
	}
}

// Open opens the Service.
func (s *Service) Open() error {
	s.wg.Add(1)
	go s.serve()
	s.logger.Println("service listening on", s.tn.Addr())
	return nil
}

// Close closes the service.
func (s *Service) Close() error {
	s.tn.Close()
	s.wg.Wait()
	return nil
}

// Addr returns the address the service is listening on.
func (s *Service) Addr() string {
	return s.addr.String()
}

func (s *Service) EnableHTTPS(b bool) {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.https = b
}

func (s *Service) SetAPIAddr(addr string) {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.apiAddr = addr
}

func (s *Service) GetAPIAddr() string {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.apiAddr
}

func (s *Service) GetNodeAPIAddr(nodeAddr string) (string, error) {
	conn, err := s.tn.Dial(nodeAddr, s.timeout)
	if err != nil {
		return "", err
	}
	defer conn.Close()

	// Send the request
	c := &Command{
		Type: Command_COMMAND_TYPE_GET_NODE_API_URL,
	}
	p, err := proto.Marshal(c)
	if err != nil {
		conn.Close()
	}

	// Write length of Protobuf, the Protobuf
	b := make([]byte, 4)
	binary.LittleEndian.PutUint16(b[0:], uint16(len(p)))

	conn.Write(b)
	conn.Write(p)

	b, err = ioutil.ReadAll(conn)
	if err != nil {
		return "", err
	}

	a := &Address{}
	err = proto.Unmarshal(b, a)
	if err != nil {
		return "", err
	}

	return a.Url, nil
}

func (s *Service) serve() error {
	defer s.wg.Done()
	for {
		conn, err := s.tn.Accept()
		if err != nil {
			return err
		}

		go s.handleConn(conn)
	}
}

func (s *Service) handleConn(conn net.Conn) {
	defer conn.Close()

	b := make([]byte, 4)
	_, err := io.ReadFull(conn, b)
	if err != nil {
		return
	}
	sz := binary.LittleEndian.Uint16(b[0:])

	b = make([]byte, sz)
	_, err = io.ReadFull(conn, b)
	if err != nil {
		return
	}

	c := &Command{}
	err = proto.Unmarshal(b, c)
	if err != nil {
		conn.Close()
	}

	switch c.Type {
	case Command_COMMAND_TYPE_GET_NODE_API_URL:
		s.mu.RLock()
		defer s.mu.RUnlock()

		a := &Address{}
		scheme := "http"
		if s.https {
			scheme = "https"
		}
		a.Url = fmt.Sprintf("%s://%s", scheme, s.apiAddr)

		b, err = proto.Marshal(a)
		if err != nil {
			conn.Close()
		}
		conn.Write(b)
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package command

import (
	"context"
	"encoding/binary"
	"fmt"
	"math"
	"math/rand"
	"os"
	"strconv"
	"sync"
	"time"

	v3 "github.com/coreos/etcd/clientv3"
	"github.com/coreos/etcd/pkg/report"

	"github.com/spf13/cobra"
	"golang.org/x/time/rate"
	"gopkg.in/cheggaaa/pb.v1"
)

var (
	checkPerfLoad        string
	checkPerfPrefix      string
	checkDatascaleLoad   string
	checkDatascalePrefix string
	autoCompact          bool
	autoDefrag           bool
)

type checkPerfCfg struct {
	limit    int
	clients  int
	duration int
}

var checkPerfCfgMap = map[string]checkPerfCfg{
	// TODO: support read limit
	"s": {
		limit:    150,
		clients:  50,
		duration: 60,
	},
	"m": {
		limit:    1000,
		clients:  200,
		duration: 60,
	},
	"l": {
		limit:    8000,
		clients:  500,
		duration: 60,
	},
	"xl": {
		limit:    15000,
		clients:  1000,
		duration: 60,
	},
}

type checkDatascaleCfg struct {
	limit   int
	kvSize  int
	clients int
}

var checkDatascaleCfgMap = map[string]checkDatascaleCfg{
	"s": {
		limit:   10000,
		kvSize:  1024,
		clients: 50,
	},
	"m": {
		limit:   100000,
		kvSize:  1024,
		clients: 200,
	},
	"l": {
		limit:   1000000,
		kvSize:  1024,
		clients: 500,
	},
	"xl": {
		// xl tries to hit the upper bound aggressively which is 3 versions of 1M objects (3M in total)
		limit:   30000000,
		kvSize:  1024,
		clients: 1000,
	},
}

// NewCheckCommand returns the cobra command for "check".
func NewCheckCommand() *cobra.Command {
	cc := &cobra.Command{
		Use:   "check <subcommand>",
		Short: "commands for checking properties of the etcd cluster",
	}

	cc.AddCommand(NewCheckPerfCommand())
	cc.AddCommand(NewCheckDatascaleCommand())

	return cc
}

// NewCheckPerfCommand returns the cobra command for "check perf".
func NewCheckPerfCommand() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "perf [options]",
		Short: "Check the performance of the etcd cluster",
		Run:   newCheckPerfCommand,
	}

	// TODO: support customized configuration
	cmd.Flags().StringVar(&checkPerfLoad, "load", "s", "The performance check's workload model. Accepted workloads: s(small), m(medium), l(large), xl(xLarge)")
	cmd.Flags().StringVar(&checkPerfPrefix, "prefix", "/etcdctl-check-perf/", "The prefix for writing the performance check's keys.")
	cmd.Flags().BoolVar(&autoCompact, "auto-compact", false, "Compact storage with last revision after test is finished.")
	cmd.Flags().BoolVar(&autoDefrag, "auto-defrag", false, "Defragment storage after test is finished.")

	return cmd
}

// newCheckPerfCommand executes the "check perf" command.
func newCheckPerfCommand(cmd *cobra.Command, args []string) {
	var checkPerfAlias = map[string]string{
		"s": "s", "small": "s",
		"m": "m", "medium": "m",
		"l": "l", "large": "l",
		"xl": "xl", "xLarge": "xl",
	}

	model, ok := checkPerfAlias[checkPerfLoad]
	if !ok {
		ExitWithError(ExitBadFeature, fmt.Errorf("unknown load option %v", checkPerfLoad))
	}
	cfg := checkPerfCfgMap[model]

	requests := make(chan v3.Op, cfg.clients)
	limit := rate.NewLimiter(rate.Limit(cfg.limit), 1)

	cc := clientConfigFromCmd(cmd)
	clients := make([]*v3.Client, cfg.clients)
	for i := 0; i < cfg.clients; i++ {
		clients[i] = cc.mustClient()
	}

	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(cfg.duration)*time.Second)
	resp, err := clients[0].Get(ctx, checkPerfPrefix, v3.WithPrefix(), v3.WithLimit(1))
	cancel()
	if err != nil {
		ExitWithError(ExitError, err)
	}
	if len(resp.Kvs) > 0 {
		ExitWithError(ExitInvalidInput, fmt.Errorf("prefix %q has keys. Delete with etcdctl del --prefix %s first.", checkPerfPrefix, checkPerfPrefix))
	}

	ksize, vsize := 256, 1024
	k, v := make([]byte, ksize), string(make([]byte, vsize))

	bar := pb.New(cfg.duration)
	bar.Format("Bom !")
	bar.Start()

	r := report.NewReport("%4.4f")
	var wg sync.WaitGroup

	wg.Add(len(clients))
	for i := range clients {
		go func(c *v3.Client) {
			defer wg.Done()
			for op := range requests {
				st := time.Now()
				_, derr := c.Do(context.Background(), op)
				r.Results() <- report.Result{Err: derr, Start: st, End: time.Now()}
			}
		}(clients[i])
	}

	go func() {
		cctx, ccancel := context.WithTimeout(context.Background(), time.Duration(cfg.duration)*time.Second)
		defer ccancel()
		for limit.Wait(cctx) == nil {
			binary.PutVarint(k, int64(rand.Int63n(math.MaxInt64)))
			requests <- v3.OpPut(checkPerfPrefix+string(k), v)
		}
		close(requests)
	}()

	go func() {
		for i := 0; i < cfg.duration; i++ {
			time.Sleep(time.Second)
			bar.Add(1)
		}
		bar.Finish()
	}()

	sc := r.Stats()
	wg.Wait()
	close(r.Results())

	s := <-sc

	ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second)
	dresp, err := clients[0].Delete(ctx, checkPerfPrefix, v3.WithPrefix())
	cancel()
	if err != nil {
		ExitWithError(ExitError, err)
	}

	if autoCompact {
		compact(clients[0], dresp.Header.Revision)
	}

	if autoDefrag {
		for _, ep := range clients[0].Endpoints() {
			defrag(clients[0], ep)
		}
	}

	ok = true
	if len(s.ErrorDist) != 0 {
		fmt.Println("FAIL: too many errors")
		for k, v := range s.ErrorDist {
			fmt.Printf("FAIL: ERROR(%v) -> %d\n", k, v)
		}
		ok = false
	}

	if s.RPS/float64(cfg.limit) <= 0.9 {
		fmt.Printf("FAIL: Throughput too low: %d writes/s\n", int(s.RPS)+1)
		ok = false
	} else {
		fmt.Printf("PASS: Throughput is %d writes/s\n", int(s.RPS)+1)
	}
	if s.Slowest > 0.5 { // slowest request > 500ms
		fmt.Printf("Slowest request took too long: %fs\n", s.Slowest)
		ok = false
	} else {
		fmt.Printf("PASS: Slowest request took %fs\n", s.Slowest)
	}
	if s.Stddev > 0.1 { // stddev > 100ms
		fmt.Printf("Stddev too high: %fs\n", s.Stddev)
		ok = false
	} else {
		fmt.Printf("PASS: Stddev is %fs\n", s.Stddev)
	}

	if ok {
		fmt.Println("PASS")
	} else {
		fmt.Println("FAIL")
		os.Exit(ExitError)
	}
}

// NewCheckDatascaleCommand returns the cobra command for "check datascale".
func NewCheckDatascaleCommand() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "datascale [options]",
		Short: "Check the memory usage of holding data for diferent workloads on a given server endpoint.",
		Long:  "If no endpoint is provided, localhost will be used. If multiple endpoints are provided, first endpoint will be used.",
		Run:   newCheckDatascaleCommand,
	}

	cmd.Flags().StringVar(&checkDatascaleLoad, "load", "s", "The datascale check's workload model. Accepted workloads: s(small), m(medium), l(large), xl(xLarge)")
	cmd.Flags().StringVar(&checkDatascalePrefix, "prefix", "/etcdctl-check-datascale/", "The prefix for writing the datascale check's keys.")
	cmd.Flags().BoolVar(&autoCompact, "auto-compact", false, "Compact storage with last revision after test is finished.")
	cmd.Flags().BoolVar(&autoDefrag, "auto-defrag", false, "Defragment storage after test is finished.")

	return cmd
}

// newCheckDatascaleCommand executes the "check datascale" command.
func newCheckDatascaleCommand(cmd *cobra.Command, args []string) {
	var checkDatascaleAlias = map[string]string{
		"s": "s", "small": "s",
		"m": "m", "medium": "m",
		"l": "l", "large": "l",
		"xl": "xl", "xLarge": "xl",
	}

	model, ok := checkDatascaleAlias[checkDatascaleLoad]
	if !ok {
		ExitWithError(ExitBadFeature, fmt.Errorf("unknown load option %v", checkDatascaleLoad))
	}
	cfg := checkDatascaleCfgMap[model]

	requests := make(chan v3.Op, cfg.clients)

	cc := clientConfigFromCmd(cmd)
	clients := make([]*v3.Client, cfg.clients)
	for i := 0; i < cfg.clients; i++ {
		clients[i] = cc.mustClient()
	}

	// get endpoints
	eps, errEndpoints := endpointsFromCmd(cmd)
	if errEndpoints != nil {
		ExitWithError(ExitError, errEndpoints)
	}

	ctx, cancel := context.WithCancel(context.Background())
	resp, err := clients[0].Get(ctx, checkDatascalePrefix, v3.WithPrefix(), v3.WithLimit(1))
	cancel()
	if err != nil {
		ExitWithError(ExitError, err)
	}
	if len(resp.Kvs) > 0 {
		ExitWithError(ExitInvalidInput, fmt.Errorf("prefix %q has keys. Delete with etcdctl del --prefix %s first.", checkDatascalePrefix, checkDatascalePrefix))
	}

	ksize, vsize := 512, 512
	k, v := make([]byte, ksize), string(make([]byte, vsize))

	r := report.NewReport("%4.4f")
	var wg sync.WaitGroup
	wg.Add(len(clients))

	// get the process_resident_memory_bytes and process_virtual_memory_bytes before the put operations
	bytesBefore := endpointMemoryMetrics(eps[0])
	if bytesBefore == 0 {
		fmt.Println("FAIL: Could not read process_resident_memory_bytes before the put operations.")
		os.Exit(ExitError)
	}

	fmt.Println(fmt.Sprintf("Start data scale check for work load [%v key-value pairs, %v bytes per key-value, %v concurrent clients].", cfg.limit, cfg.kvSize, cfg.clients))
	for i := range clients {
		go func(c *v3.Client) {
			defer wg.Done()
			for op := range requests {
				st := time.Now()
				_, derr := c.Do(context.Background(), op)
				r.Results() <- report.Result{Err: derr, Start: st, End: time.Now()}
			}
		}(clients[i])
	}

	go func() {
		for i := 0; i < cfg.limit; i++ {
			binary.PutVarint(k, int64(rand.Int63n(math.MaxInt64)))
			requests <- v3.OpPut(checkDatascalePrefix+string(k), v)
		}
		close(requests)
	}()

	sc := r.Stats()
	wg.Wait()
	close(r.Results())
	s := <-sc

	// get the process_resident_memory_bytes after the put operations
	bytesAfter := endpointMemoryMetrics(eps[0])
	if bytesAfter == 0 {
		fmt.Println("FAIL: Could not read process_resident_memory_bytes after the put operations.")
		os.Exit(ExitError)
	}

	// delete the created kv pairs
	ctx, cancel = context.WithCancel(context.Background())
	dresp, derr := clients[0].Delete(ctx, checkDatascalePrefix, v3.WithPrefix())
	defer cancel()
	if derr != nil {
		ExitWithError(ExitError, derr)
	}

	if autoCompact {
		compact(clients[0], dresp.Header.Revision)
	}

	if autoDefrag {
		for _, ep := range clients[0].Endpoints() {
			defrag(clients[0], ep)
		}
	}

	if bytesAfter == 0 {
		fmt.Println("FAIL: Could not read process_resident_memory_bytes after the put operations.")
		os.Exit(ExitError)
	}

	bytesUsed := bytesAfter - bytesBefore
	mbUsed := bytesUsed / (1024 * 1024)

	if len(s.ErrorDist) != 0 {
		fmt.Println("FAIL: too many errors")
		for k, v := range s.ErrorDist {
			fmt.Printf("FAIL: ERROR(%v) -> %d\n", k, v)
		}
		os.Exit(ExitError)
	} else {
		fmt.Println(fmt.Sprintf("PASS: Approximate system memory used : %v MB.", strconv.FormatFloat(float64(mbUsed), 'f', 2, 64)))
	}
}
<commit_msg>ctlv3: fix typo in check command<commit_after>// Copyright 2017 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package command

import (
	"context"
	"encoding/binary"
	"fmt"
	"math"
	"math/rand"
	"os"
	"strconv"
	"sync"
	"time"

	v3 "github.com/coreos/etcd/clientv3"
	"github.com/coreos/etcd/pkg/report"

	"github.com/spf13/cobra"
	"golang.org/x/time/rate"
	"gopkg.in/cheggaaa/pb.v1"
)

var (
	checkPerfLoad        string
	checkPerfPrefix      string
	checkDatascaleLoad   string
	checkDatascalePrefix string
	autoCompact          bool
	autoDefrag           bool
)

type checkPerfCfg struct {
	limit    int
	clients  int
	duration int
}

var checkPerfCfgMap = map[string]checkPerfCfg{
	// TODO: support read limit
	"s": {
		limit:    150,
		clients:  50,
		duration: 60,
	},
	"m": {
		limit:    1000,
		clients:  200,
		duration: 60,
	},
	"l": {
		limit:    8000,
		clients:  500,
		duration: 60,
	},
	"xl": {
		limit:    15000,
		clients:  1000,
		duration: 60,
	},
}

type checkDatascaleCfg struct {
	limit   int
	kvSize  int
	clients int
}

var checkDatascaleCfgMap = map[string]checkDatascaleCfg{
	"s": {
		limit:   10000,
		kvSize:  1024,
		clients: 50,
	},
	"m": {
		limit:   100000,
		kvSize:  1024,
		clients: 200,
	},
	"l": {
		limit:   1000000,
		kvSize:  1024,
		clients: 500,
	},
	"xl": {
		// xl tries to hit the upper bound aggressively which is 3 versions of 1M objects (3M in total)
		limit:   30000000,
		kvSize:  1024,
		clients: 1000,
	},
}

// NewCheckCommand returns the cobra command for "check".
func NewCheckCommand() *cobra.Command {
	cc := &cobra.Command{
		Use:   "check <subcommand>",
		Short: "commands for checking properties of the etcd cluster",
	}

	cc.AddCommand(NewCheckPerfCommand())
	cc.AddCommand(NewCheckDatascaleCommand())

	return cc
}

// NewCheckPerfCommand returns the cobra command for "check perf".
func NewCheckPerfCommand() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "perf [options]",
		Short: "Check the performance of the etcd cluster",
		Run:   newCheckPerfCommand,
	}

	// TODO: support customized configuration
	cmd.Flags().StringVar(&checkPerfLoad, "load", "s", "The performance check's workload model. Accepted workloads: s(small), m(medium), l(large), xl(xLarge)")
	cmd.Flags().StringVar(&checkPerfPrefix, "prefix", "/etcdctl-check-perf/", "The prefix for writing the performance check's keys.")
	cmd.Flags().BoolVar(&autoCompact, "auto-compact", false, "Compact storage with last revision after test is finished.")
	cmd.Flags().BoolVar(&autoDefrag, "auto-defrag", false, "Defragment storage after test is finished.")

	return cmd
}

// newCheckPerfCommand executes the "check perf" command.
func newCheckPerfCommand(cmd *cobra.Command, args []string) {
	var checkPerfAlias = map[string]string{
		"s": "s", "small": "s",
		"m": "m", "medium": "m",
		"l": "l", "large": "l",
		"xl": "xl", "xLarge": "xl",
	}

	model, ok := checkPerfAlias[checkPerfLoad]
	if !ok {
		ExitWithError(ExitBadFeature, fmt.Errorf("unknown load option %v", checkPerfLoad))
	}
	cfg := checkPerfCfgMap[model]

	requests := make(chan v3.Op, cfg.clients)
	limit := rate.NewLimiter(rate.Limit(cfg.limit), 1)

	cc := clientConfigFromCmd(cmd)
	clients := make([]*v3.Client, cfg.clients)
	for i := 0; i < cfg.clients; i++ {
		clients[i] = cc.mustClient()
	}

	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(cfg.duration)*time.Second)
	resp, err := clients[0].Get(ctx, checkPerfPrefix, v3.WithPrefix(), v3.WithLimit(1))
	cancel()
	if err != nil {
		ExitWithError(ExitError, err)
	}
	if len(resp.Kvs) > 0 {
		ExitWithError(ExitInvalidInput, fmt.Errorf("prefix %q has keys. Delete with etcdctl del --prefix %s first.", checkPerfPrefix, checkPerfPrefix))
	}

	ksize, vsize := 256, 1024
	k, v := make([]byte, ksize), string(make([]byte, vsize))

	bar := pb.New(cfg.duration)
	bar.Format("Bom !")
	bar.Start()

	r := report.NewReport("%4.4f")
	var wg sync.WaitGroup

	wg.Add(len(clients))
	for i := range clients {
		go func(c *v3.Client) {
			defer wg.Done()
			for op := range requests {
				st := time.Now()
				_, derr := c.Do(context.Background(), op)
				r.Results() <- report.Result{Err: derr, Start: st, End: time.Now()}
			}
		}(clients[i])
	}

	go func() {
		cctx, ccancel := context.WithTimeout(context.Background(), time.Duration(cfg.duration)*time.Second)
		defer ccancel()
		for limit.Wait(cctx) == nil {
			binary.PutVarint(k, int64(rand.Int63n(math.MaxInt64)))
			requests <- v3.OpPut(checkPerfPrefix+string(k), v)
		}
		close(requests)
	}()

	go func() {
		for i := 0; i < cfg.duration; i++ {
			time.Sleep(time.Second)
			bar.Add(1)
		}
		bar.Finish()
	}()

	sc := r.Stats()
	wg.Wait()
	close(r.Results())

	s := <-sc

	ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second)
	dresp, err := clients[0].Delete(ctx, checkPerfPrefix, v3.WithPrefix())
	cancel()
	if err != nil {
		ExitWithError(ExitError, err)
	}

	if autoCompact {
		compact(clients[0], dresp.Header.Revision)
	}

	if autoDefrag {
		for _, ep := range clients[0].Endpoints() {
			defrag(clients[0], ep)
		}
	}

	ok = true
	if len(s.ErrorDist) != 0 {
		fmt.Println("FAIL: too many errors")
		for k, v := range s.ErrorDist {
			fmt.Printf("FAIL: ERROR(%v) -> %d\n", k, v)
		}
		ok = false
	}

	if s.RPS/float64(cfg.limit) <= 0.9 {
		fmt.Printf("FAIL: Throughput too low: %d writes/s\n", int(s.RPS)+1)
		ok = false
	} else {
		fmt.Printf("PASS: Throughput is %d writes/s\n", int(s.RPS)+1)
	}
	if s.Slowest > 0.5 { // slowest request > 500ms
		fmt.Printf("Slowest request took too long: %fs\n", s.Slowest)
		ok = false
	} else {
		fmt.Printf("PASS: Slowest request took %fs\n", s.Slowest)
	}
	if s.Stddev > 0.1 { // stddev > 100ms
		fmt.Printf("Stddev too high: %fs\n", s.Stddev)
		ok = false
	} else {
		fmt.Printf("PASS: Stddev is %fs\n", s.Stddev)
	}

	if ok {
		fmt.Println("PASS")
	} else {
		fmt.Println("FAIL")
		os.Exit(ExitError)
	}
}

// NewCheckDatascaleCommand returns the cobra command for "check datascale".
func NewCheckDatascaleCommand() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "datascale [options]",
		Short: "Check the memory usage of holding data for different workloads on a given server endpoint.",
		Long:  "If no endpoint is provided, localhost will be used. If multiple endpoints are provided, first endpoint will be used.",
		Run:   newCheckDatascaleCommand,
	}

	cmd.Flags().StringVar(&checkDatascaleLoad, "load", "s", "The datascale check's workload model. Accepted workloads: s(small), m(medium), l(large), xl(xLarge)")
	cmd.Flags().StringVar(&checkDatascalePrefix, "prefix", "/etcdctl-check-datascale/", "The prefix for writing the datascale check's keys.")
	cmd.Flags().BoolVar(&autoCompact, "auto-compact", false, "Compact storage with last revision after test is finished.")
	cmd.Flags().BoolVar(&autoDefrag, "auto-defrag", false, "Defragment storage after test is finished.")

	return cmd
}

// newCheckDatascaleCommand executes the "check datascale" command.
func newCheckDatascaleCommand(cmd *cobra.Command, args []string) {
	var checkDatascaleAlias = map[string]string{
		"s": "s", "small": "s",
		"m": "m", "medium": "m",
		"l": "l", "large": "l",
		"xl": "xl", "xLarge": "xl",
	}

	model, ok := checkDatascaleAlias[checkDatascaleLoad]
	if !ok {
		ExitWithError(ExitBadFeature, fmt.Errorf("unknown load option %v", checkDatascaleLoad))
	}
	cfg := checkDatascaleCfgMap[model]

	requests := make(chan v3.Op, cfg.clients)

	cc := clientConfigFromCmd(cmd)
	clients := make([]*v3.Client, cfg.clients)
	for i := 0; i < cfg.clients; i++ {
		clients[i] = cc.mustClient()
	}

	// get endpoints
	eps, errEndpoints := endpointsFromCmd(cmd)
	if errEndpoints != nil {
		ExitWithError(ExitError, errEndpoints)
	}

	ctx, cancel := context.WithCancel(context.Background())
	resp, err := clients[0].Get(ctx, checkDatascalePrefix, v3.WithPrefix(), v3.WithLimit(1))
	cancel()
	if err != nil {
		ExitWithError(ExitError, err)
	}
	if len(resp.Kvs) > 0 {
		ExitWithError(ExitInvalidInput, fmt.Errorf("prefix %q has keys. Delete with etcdctl del --prefix %s first.", checkDatascalePrefix, checkDatascalePrefix))
	}

	ksize, vsize := 512, 512
	k, v := make([]byte, ksize), string(make([]byte, vsize))

	r := report.NewReport("%4.4f")
	var wg sync.WaitGroup
	wg.Add(len(clients))

	// get the process_resident_memory_bytes and process_virtual_memory_bytes before the put operations
	bytesBefore := endpointMemoryMetrics(eps[0])
	if bytesBefore == 0 {
		fmt.Println("FAIL: Could not read process_resident_memory_bytes before the put operations.")
		os.Exit(ExitError)
	}

	fmt.Println(fmt.Sprintf("Start data scale check for work load [%v key-value pairs, %v bytes per key-value, %v concurrent clients].", cfg.limit, cfg.kvSize, cfg.clients))
	for i := range clients {
		go func(c *v3.Client) {
			defer wg.Done()
			for op := range requests {
				st := time.Now()
				_, derr := c.Do(context.Background(), op)
				r.Results() <- report.Result{Err: derr, Start: st, End: time.Now()}
			}
		}(clients[i])
	}

	go func() {
		for i := 0; i < cfg.limit; i++ {
			binary.PutVarint(k, int64(rand.Int63n(math.MaxInt64)))
			requests <- v3.OpPut(checkDatascalePrefix+string(k), v)
		}
		close(requests)
	}()

	sc := r.Stats()
	wg.Wait()
	close(r.Results())
	s := <-sc

	// get the process_resident_memory_bytes after the put operations
	bytesAfter := endpointMemoryMetrics(eps[0])
	if bytesAfter == 0 {
		fmt.Println("FAIL: Could not read process_resident_memory_bytes after the put operations.")
		os.Exit(ExitError)
	}

	// delete the created kv pairs
	ctx, cancel = context.WithCancel(context.Background())
	dresp, derr := clients[0].Delete(ctx, checkDatascalePrefix, v3.WithPrefix())
	defer cancel()
	if derr != nil {
		ExitWithError(ExitError, derr)
	}

	if autoCompact {
		compact(clients[0], dresp.Header.Revision)
	}

	if autoDefrag {
		for _, ep := range clients[0].Endpoints() {
			defrag(clients[0], ep)
		}
	}

	if bytesAfter == 0 {
		fmt.Println("FAIL: Could not read process_resident_memory_bytes after the put operations.")
		os.Exit(ExitError)
	}

	bytesUsed := bytesAfter - bytesBefore
	mbUsed := bytesUsed / (1024 * 1024)

	if len(s.ErrorDist) != 0 {
		fmt.Println("FAIL: too many errors")
		for k, v := range s.ErrorDist {
			fmt.Printf("FAIL: ERROR(%v) -> %d\n", k, v)
		}
		os.Exit(ExitError)
	} else {
		fmt.Println(fmt.Sprintf("PASS: Approximate system memory used : %v MB.", strconv.FormatFloat(float64(mbUsed), 'f', 2, 64)))
	}
}
<|endoftext|>