<commit_before>package applyspec_test

import (
	"encoding/json"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	. "bosh/agent/applier/applyspec"
	models "bosh/agent/applier/models"
)

func init() {
	Describe("V1ApplySpec", func() {
		It("v1 apply spec json conversion", func() {
			specJSON := `{
				"properties": {
					"logging": {"max_log_file_size": "10M"}
				},
				"job": {
					"name": "router",
					"template": "router template",
					"version": "1.0",
					"sha1": "router sha1",
					"blobstore_id": "router-blob-id-1",
					"templates": [
						{"name": "template 1", "version": "0.1", "sha1": "template 1 sha1", "blobstore_id": "template-blob-id-1"},
						{"name": "template 2", "version": "0.2", "sha1": "template 2 sha1", "blobstore_id": "template-blob-id-2"}
					]
				},
				"packages": {
					"package 1": {"name": "package 1", "version": "0.1", "sha1": "package 1 sha1", "blobstore_id": "package-blob-id-1"},
					"package 2": {"name": "package 2", "version": "0.2", "sha1": "package 2 sha1", "blobstore_id": "package-blob-id-2"}
				},
				"networks": {
					"manual-net": {
						"cloud_properties": {
							"subnet": "subnet-xxxxxx"
						},
						"default": [
							"dns",
							"gateway"
						],
						"dns": [
							"xx.xx.xx.xx"
						],
						"dns_record_name": "job-index.job-name.manual-net.deployment-name.bosh",
						"gateway": "xx.xx.xx.xx",
						"ip": "xx.xx.xx.xx",
						"netmask": "xx.xx.xx.xx"
					},
					"vip-net": {
						"cloud_properties": {
							"security_groups": [
								"bosh"
							]
						},
						"dns_record_name": "job-index.job-name.vip-net.deployment-name.bosh",
						"ip": "xx.xx.xx.xx",
						"type": "vip"
					}
				},
				"rendered_templates_archive": {
					"sha1": "archive sha 1",
					"blobstore_id": "archive-blob-id-1"
				}
			}`

			expectedNetworks := map[string]interface{}{
				"manual-net": map[string]interface{}{
					"cloud_properties": map[string]interface{}{"subnet": "subnet-xxxxxx"},
					"default":          []interface{}{"dns", "gateway"},
					"dns":              []interface{}{"xx.xx.xx.xx"},
					"dns_record_name":  "job-index.job-name.manual-net.deployment-name.bosh",
					"gateway":          "xx.xx.xx.xx",
					"ip":               "xx.xx.xx.xx",
					"netmask":          "xx.xx.xx.xx",
				},
				"vip-net": map[string]interface{}{
					"cloud_properties": map[string]interface{}{"security_groups": []interface{}{"bosh"}},
					"dns_record_name":  "job-index.job-name.vip-net.deployment-name.bosh",
					"ip":               "xx.xx.xx.xx",
					"type":             "vip",
				},
			}

			jobName := "router"

			expectedSpec := V1ApplySpec{
				PropertiesSpec: PropertiesSpec{
					LoggingSpec: LoggingSpec{MaxLogFileSize: "10M"},
				},
				JobSpec: JobSpec{
					Name:        &jobName,
					Template:    "router template",
					Version:     "1.0",
					Sha1:        "router sha1",
					BlobstoreID: "router-blob-id-1",
					JobTemplateSpecs: []JobTemplateSpec{
						{Name: "template 1", Version: "0.1", Sha1: "template 1 sha1", BlobstoreID: "template-blob-id-1"},
						{Name: "template 2", Version: "0.2", Sha1: "template 2 sha1", BlobstoreID: "template-blob-id-2"},
					},
				},
				PackageSpecs: map[string]PackageSpec{
					"package 1": PackageSpec{Name: "package 1", Version: "0.1", Sha1: "package 1 sha1", BlobstoreID: "package-blob-id-1"},
					"package 2": PackageSpec{Name: "package 2", Version: "0.2", Sha1: "package 2 sha1", BlobstoreID: "package-blob-id-2"},
				},
				RenderedTemplatesArchiveSpec: RenderedTemplatesArchiveSpec{
					Sha1:        "archive sha 1",
					BlobstoreID: "archive-blob-id-1",
				},
				NetworkSpecs: expectedNetworks,
			}

			spec := V1ApplySpec{}
			err := json.Unmarshal([]byte(specJSON), &spec)
			Expect(err).ToNot(HaveOccurred())
			Expect(spec).To(Equal(expectedSpec))
			Expect(spec.NetworkSpecs).To(Equal(expectedNetworks))
		})

		It("jobs with specified job templates", func() {
			jobName := "fake-job-legacy-name"

			spec := V1ApplySpec{
				JobSpec: JobSpec{
					Name:        &jobName,
					Version:     "fake-job-legacy-version",
					Sha1:        "fake-job-legacy-sha1",
					BlobstoreID: "fake-job-legacy-blobstore-id",
					JobTemplateSpecs: []JobTemplateSpec{
						JobTemplateSpec{
							Name:        "fake-job1-name",
							Version:     "fake-job1-version",
							Sha1:        "fake-job1-sha1",
							BlobstoreID: "fake-job1-blobstore-id",
						},
						JobTemplateSpec{
							Name:        "fake-job2-name",
							Version:     "fake-job2-version",
							Sha1:        "fake-job2-sha1",
							BlobstoreID: "fake-job2-blobstore-id",
						},
					},
				},
				PackageSpecs: map[string]PackageSpec{
					"fake-package1": PackageSpec{
						Name:        "fake-package1-name",
						Version:     "fake-package1-version",
						Sha1:        "fake-package1-sha1",
						BlobstoreID: "fake-package1-blob-id",
					},
					"fake-package2": PackageSpec{
						Name:        "fake-package2-name",
						Version:     "fake-package2-version",
						Sha1:        "fake-package2-sha1",
						BlobstoreID: "fake-package2-blob-id",
					},
				},
				RenderedTemplatesArchiveSpec: RenderedTemplatesArchiveSpec{
					Sha1:        "fake-rendered-templates-archive-sha1",
					BlobstoreID: "fake-rendered-templates-archive-blobstore-id",
				},
			}

			expectedPackagesOnEachJob := []models.Package{
				models.Package{
					Name:    "fake-package1-name",
					Version: "fake-package1-version",
					Source: models.Source{
						Sha1:          "fake-package1-sha1",
						BlobstoreID:   "fake-package1-blob-id",
						PathInArchive: "",
					},
				},
				models.Package{
					Name:    "fake-package2-name",
					Version: "fake-package2-version",
					Source: models.Source{
						Sha1:          "fake-package2-sha1",
						BlobstoreID:   "fake-package2-blob-id",
						PathInArchive: "",
					},
				},
			}

			Expect(spec.Jobs()).To(Equal([]models.Job{
				models.Job{
					Name:    "fake-job1-name",
					Version: "fake-job1-version",
					Source: models.Source{
						Sha1:          "fake-rendered-templates-archive-sha1",
						BlobstoreID:   "fake-rendered-templates-archive-blobstore-id",
						PathInArchive: "fake-job1-name",
					},
					Packages: expectedPackagesOnEachJob,
				},
				models.Job{
					Name:    "fake-job2-name",
					Version: "fake-job2-version",
					Source: models.Source{
						Sha1:          "fake-rendered-templates-archive-sha1",
						BlobstoreID:   "fake-rendered-templates-archive-blobstore-id",
						PathInArchive: "fake-job2-name",
					},
					Packages: expectedPackagesOnEachJob,
				},
			}))
		})

		It("jobs when no jobs specified", func() {
			spec := V1ApplySpec{}
			Expect(spec.Jobs()).To(Equal([]models.Job{}))
		})

		It("packages", func() {
			spec := V1ApplySpec{
				PackageSpecs: map[string]PackageSpec{
					"fake-package1-name-key": PackageSpec{
						Name:        "fake-package1-name",
						Version:     "fake-package1-version",
						Sha1:        "fake-package1-sha1",
						BlobstoreID: "fake-package1-blobstore-id",
					},
				},
			}

			Expect(spec.Packages()).To(Equal([]models.Package{
				models.Package{
					Name:    "fake-package1-name",
					Version: "fake-package1-version",
					Source: models.Source{
						Sha1:        "fake-package1-sha1",
						BlobstoreID: "fake-package1-blobstore-id",
					},
				},
			}))
		})

		It("packages when no packages specified", func() {
			spec := V1ApplySpec{}
			Expect(spec.Packages()).To(Equal([]models.Package{}))
		})

		It("max log file size", func() {
			spec := V1ApplySpec{}
			Expect(spec.MaxLogFileSize()).To(Equal("50M"))

			spec.PropertiesSpec.LoggingSpec.MaxLogFileSize = "fake-size"
			Expect(spec.MaxLogFileSize()).To(Equal("fake-size"))
		})
	})
}
<commit_msg>formatting<commit_after>package applyspec_test

import (
	"encoding/json"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	. "bosh/agent/applier/applyspec"
	models "bosh/agent/applier/models"
)

var _ = Describe("V1ApplySpec", func() {
	Describe("json unmarshalling", func() {
		It("returns parsed apply spec from json", func() {
			specJSON := `{
				"properties": {
					"logging": {"max_log_file_size": "10M"}
				},
				"job": {
					"name": "router",
					"template": "router template",
					"version": "1.0",
					"sha1": "router sha1",
					"blobstore_id": "router-blob-id-1",
					"templates": [
						{"name": "template 1", "version": "0.1", "sha1": "template 1 sha1", "blobstore_id": "template-blob-id-1"},
						{"name": "template 2", "version": "0.2", "sha1": "template 2 sha1", "blobstore_id": "template-blob-id-2"}
					]
				},
				"packages": {
					"package 1": {"name": "package 1", "version": "0.1", "sha1": "package 1 sha1", "blobstore_id": "package-blob-id-1"},
					"package 2": {"name": "package 2", "version": "0.2", "sha1": "package 2 sha1", "blobstore_id": "package-blob-id-2"}
				},
				"networks": {
					"manual-net": {
						"cloud_properties": {
							"subnet": "subnet-xxxxxx"
						},
						"default": [
							"dns",
							"gateway"
						],
						"dns": [
							"xx.xx.xx.xx"
						],
						"dns_record_name": "job-index.job-name.manual-net.deployment-name.bosh",
						"gateway": "xx.xx.xx.xx",
						"ip": "xx.xx.xx.xx",
						"netmask": "xx.xx.xx.xx"
					},
					"vip-net": {
						"cloud_properties": {
							"security_groups": [
								"bosh"
							]
						},
						"dns_record_name": "job-index.job-name.vip-net.deployment-name.bosh",
						"ip": "xx.xx.xx.xx",
						"type": "vip"
					}
				},
				"rendered_templates_archive": {
					"sha1": "archive sha 1",
					"blobstore_id": "archive-blob-id-1"
				}
			}`

			spec := V1ApplySpec{}
			err := json.Unmarshal([]byte(specJSON), &spec)
			Expect(err).ToNot(HaveOccurred())

			jobName := "router"

			expectedNetworks := map[string]interface{}{
				"manual-net": map[string]interface{}{
					"cloud_properties": map[string]interface{}{"subnet": "subnet-xxxxxx"},
					"default":          []interface{}{"dns", "gateway"},
					"dns":              []interface{}{"xx.xx.xx.xx"},
					"dns_record_name":  "job-index.job-name.manual-net.deployment-name.bosh",
					"gateway":          "xx.xx.xx.xx",
					"ip":               "xx.xx.xx.xx",
					"netmask":          "xx.xx.xx.xx",
				},
				"vip-net": map[string]interface{}{
					"cloud_properties": map[string]interface{}{"security_groups": []interface{}{"bosh"}},
					"dns_record_name":  "job-index.job-name.vip-net.deployment-name.bosh",
					"ip":               "xx.xx.xx.xx",
					"type":             "vip",
				},
			}

			expectedSpec := V1ApplySpec{
				PropertiesSpec: PropertiesSpec{
					LoggingSpec: LoggingSpec{MaxLogFileSize: "10M"},
				},
				JobSpec: JobSpec{
					Name:        &jobName,
					Template:    "router template",
					Version:     "1.0",
					Sha1:        "router sha1",
					BlobstoreID: "router-blob-id-1",
					JobTemplateSpecs: []JobTemplateSpec{
						JobTemplateSpec{Name: "template 1", Version: "0.1", Sha1: "template 1 sha1", BlobstoreID: "template-blob-id-1"},
						JobTemplateSpec{Name: "template 2", Version: "0.2", Sha1: "template 2 sha1", BlobstoreID: "template-blob-id-2"},
					},
				},
				PackageSpecs: map[string]PackageSpec{
					"package 1": PackageSpec{Name: "package 1", Version: "0.1", Sha1: "package 1 sha1", BlobstoreID: "package-blob-id-1"},
					"package 2": PackageSpec{Name: "package 2", Version: "0.2", Sha1: "package 2 sha1", BlobstoreID: "package-blob-id-2"},
				},
				RenderedTemplatesArchiveSpec: RenderedTemplatesArchiveSpec{
					Sha1:        "archive sha 1",
					BlobstoreID: "archive-blob-id-1",
				},
				NetworkSpecs: expectedNetworks,
			}

			Expect(spec).To(Equal(expectedSpec))
			Expect(spec.NetworkSpecs).To(Equal(expectedNetworks))
		})
	})

	Describe("Jobs", func() {
		It("returns jobs specified in job specs", func() {
			jobName := "fake-job-legacy-name"

			spec := V1ApplySpec{
				JobSpec: JobSpec{
					Name:        &jobName,
					Version:     "fake-job-legacy-version",
					Sha1:        "fake-job-legacy-sha1",
					BlobstoreID: "fake-job-legacy-blobstore-id",
					JobTemplateSpecs: []JobTemplateSpec{
						JobTemplateSpec{
							Name:        "fake-job1-name",
							Version:     "fake-job1-version",
							Sha1:        "fake-job1-sha1",
							BlobstoreID: "fake-job1-blobstore-id",
						},
						JobTemplateSpec{
							Name:        "fake-job2-name",
							Version:     "fake-job2-version",
							Sha1:        "fake-job2-sha1",
							BlobstoreID: "fake-job2-blobstore-id",
						},
					},
				},
				PackageSpecs: map[string]PackageSpec{
					"fake-package1": PackageSpec{
						Name:        "fake-package1-name",
						Version:     "fake-package1-version",
						Sha1:        "fake-package1-sha1",
						BlobstoreID: "fake-package1-blob-id",
					},
					"fake-package2": PackageSpec{
						Name:        "fake-package2-name",
						Version:     "fake-package2-version",
						Sha1:        "fake-package2-sha1",
						BlobstoreID: "fake-package2-blob-id",
					},
				},
				RenderedTemplatesArchiveSpec: RenderedTemplatesArchiveSpec{
					Sha1:        "fake-rendered-templates-archive-sha1",
					BlobstoreID: "fake-rendered-templates-archive-blobstore-id",
				},
			}

			expectedPackagesOnEachJob := []models.Package{
				models.Package{
					Name:    "fake-package1-name",
					Version: "fake-package1-version",
					Source: models.Source{
						Sha1:          "fake-package1-sha1",
						BlobstoreID:   "fake-package1-blob-id",
						PathInArchive: "",
					},
				},
				models.Package{
					Name:    "fake-package2-name",
					Version: "fake-package2-version",
					Source: models.Source{
						Sha1:          "fake-package2-sha1",
						BlobstoreID:   "fake-package2-blob-id",
						PathInArchive: "",
					},
				},
			}

			Expect(spec.Jobs()).To(Equal([]models.Job{
				models.Job{
					Name:    "fake-job1-name",
					Version: "fake-job1-version",
					Source: models.Source{
						Sha1:          "fake-rendered-templates-archive-sha1",
						BlobstoreID:   "fake-rendered-templates-archive-blobstore-id",
						PathInArchive: "fake-job1-name",
					},
					Packages: expectedPackagesOnEachJob,
				},
				models.Job{
					Name:    "fake-job2-name",
					Version: "fake-job2-version",
					Source: models.Source{
						Sha1:          "fake-rendered-templates-archive-sha1",
						BlobstoreID:   "fake-rendered-templates-archive-blobstore-id",
						PathInArchive: "fake-job2-name",
					},
					Packages: expectedPackagesOnEachJob,
				},
			}))
		})

		It("returns no jobs when no jobs specified", func() {
			spec := V1ApplySpec{}
			Expect(spec.Jobs()).To(Equal([]models.Job{}))
		})
	})

	Describe("Packages", func() {
		It("retuns packages", func() {
			spec := V1ApplySpec{
				PackageSpecs: map[string]PackageSpec{
					"fake-package1-name-key": PackageSpec{
						Name:        "fake-package1-name",
						Version:     "fake-package1-version",
						Sha1:        "fake-package1-sha1",
						BlobstoreID: "fake-package1-blobstore-id",
					},
				},
			}

			Expect(spec.Packages()).To(Equal([]models.Package{
				models.Package{
					Name:    "fake-package1-name",
					Version: "fake-package1-version",
					Source: models.Source{
						Sha1:        "fake-package1-sha1",
						BlobstoreID: "fake-package1-blobstore-id",
					},
				},
			}))
		})

		It("returns no packages when no packages specified", func() {
			spec := V1ApplySpec{}
			Expect(spec.Packages()).To(Equal([]models.Package{}))
		})
	})

	Describe("MaxLogFileSize", func() {
		It("returns 50M if size is not provided", func() {
			spec := V1ApplySpec{}
			Expect(spec.MaxLogFileSize()).To(Equal("50M"))
		})

		It("returns provided size", func() {
			spec := V1ApplySpec{}
			spec.PropertiesSpec.LoggingSpec.MaxLogFileSize = "fake-size"
			Expect(spec.MaxLogFileSize()).To(Equal("fake-size"))
		})
	})
})
<|endoftext|>


<commit_before>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package broker

import (
	"fmt"
	"net"

	"github.com/TheThingsNetwork/ttn/core"
	"github.com/TheThingsNetwork/ttn/utils/errors"
	"github.com/TheThingsNetwork/ttn/utils/stats"
	"github.com/TheThingsNetwork/ttn/utils/tokenkey"
	"github.com/apex/log"
	"github.com/brocaar/lorawan"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
)

// component implements the core.BrokerServer interface
type component struct {
	Components
	NetAddrUp        string
	NetAddrDown      string
	TokenKeyProvider tokenkey.Provider
	MaxDevNonces     uint
}

// Components defines a structure to make the instantiation easier to read
type Components struct {
	NetworkController NetworkController
	AppStorage        AppStorage
	Ctx               log.Interface
}

// Options defines a structure to make the instantiation easier to read
type Options struct {
	NetAddrUp        string
	NetAddrDown      string
	TokenKeyProvider tokenkey.Provider
}

// Interface defines the Broker interface
type Interface interface {
	core.BrokerServer
	core.BrokerManagerServer
	Start() error
}

// New construct a new Broker component
func New(c Components, o Options) Interface {
	return component{
		Components:       c,
		NetAddrUp:        o.NetAddrUp,
		NetAddrDown:      o.NetAddrDown,
		TokenKeyProvider: o.TokenKeyProvider,
		MaxDevNonces:     10,
	}
}

// Start actually runs the component and starts the rpc server
func (b component) Start() error {
	connUp, err := net.Listen("tcp", b.NetAddrUp)
	if err != nil {
		return errors.New(errors.Operational, err)
	}

	connDown, err := net.Listen("tcp", b.NetAddrDown)
	if err != nil {
		return errors.New(errors.Operational, err)
	}

	if b.TokenKeyProvider != nil {
		tokenKey, err := b.TokenKeyProvider.Get(true)
		if err != nil {
			return errors.New(errors.Operational, fmt.Sprintf("Failed to refresh token key: %s", err.Error()))
		}
		b.Ctx.WithField("provider", b.TokenKeyProvider).Infof("Got token key for algorithm %v", tokenKey.Algorithm)
	}

	server := grpc.NewServer()
	core.RegisterBrokerServer(server, b)
	core.RegisterBrokerManagerServer(server, b)

	cherr := make(chan error)

	go func() {
		cherr <- server.Serve(connUp)
	}()

	go func() {
		cherr <- server.Serve(connDown)
	}()

	if err := <-cherr; err != nil {
		return errors.New(errors.Operational, err)
	}
	return nil
}

// HandleJoin implements the core.BrokerServer interface
func (b component) HandleJoin(bctx context.Context, req *core.JoinBrokerReq) (*core.JoinBrokerRes, error) {
	// Validate incoming data
	if req == nil || len(req.DevEUI) != 8 || len(req.AppEUI) != 8 || len(req.DevNonce) != 2 || len(req.MIC) != 4 || req.Metadata == nil {
		b.Ctx.Debug("Invalid join request. Parameters are incorrect.")
		return new(core.JoinBrokerRes), errors.New(errors.Structural, "Invalid parameters")
	}

	ctx := b.Ctx.WithFields(log.Fields{
		"AppEUI": req.AppEUI,
		"DevEUI": req.DevEUI,
	})

	ctx.Debug("Handle join request")

	// Check if devNonce already referenced
	appEntry, err := b.AppStorage.read(req.AppEUI)
	if err != nil {
		ctx.WithError(err).Debug("Unable to lookup activation")
		return new(core.JoinBrokerRes), err
	}
	nonces, err := b.NetworkController.readNonces(req.AppEUI, req.DevEUI)
	if err != nil {
		if err.(errors.Failure).Nature != errors.NotFound {
			ctx.WithError(err).Debug("Unable to retrieve associated devNonces")
			return new(core.JoinBrokerRes), err
		}
		nonces = noncesEntry{
			AppEUI: req.AppEUI,
			DevEUI: req.DevEUI,
		}
	}

	var found bool
	for _, n := range nonces.DevNonces {
		if n[0] == req.DevNonce[0] && n[1] == req.DevNonce[1] {
			found = true
			break
		}
	}
	if found {
		ctx.Debug("DevNonce already used in the past")
		return new(core.JoinBrokerRes), errors.New(errors.Structural, "DevNonce used by the past")
	}

	// Forward the registration to the handler
	handler, closer, err := appEntry.Dialer.Dial()
	if err != nil {
		ctx.WithError(err).Debug("Unable to dial handler")
		return new(core.JoinBrokerRes), err
	}
	defer closer.Close()
	res, err := handler.HandleJoin(context.Background(), &core.JoinHandlerReq{
		DevEUI:   req.DevEUI,
		AppEUI:   req.AppEUI,
		DevNonce: req.DevNonce,
		MIC:      req.MIC,
		Metadata: req.Metadata,
	})
	if err != nil {
		ctx.WithError(err).Debug("Error while contacting handler")
		return new(core.JoinBrokerRes), errors.New(errors.Operational, err)
	}

	// Handle the response
	if res == nil || res.Payload == nil {
		ctx.Debug("No join-accept received")
		return new(core.JoinBrokerRes), nil
	}

	if len(res.DevAddr) != 4 || len(res.NwkSKey) != 16 {
		ctx.Debug("Invalid response from handler")
		return new(core.JoinBrokerRes), errors.New(errors.Operational, "Invalid response from handler")
	}

	ctx.WithField("DevAddr", res.DevAddr).Debug("Handle join-accept")

	// Update the DevNonce
	nonces.DevNonces = append(nonces.DevNonces, req.DevNonce)
	if uint(len(nonces.DevNonces)) > b.MaxDevNonces {
		nonces.DevNonces = nonces.DevNonces[1:]
	}
	err = b.NetworkController.upsertNonces(nonces)
	if err != nil {
		ctx.WithError(err).Debug("Unable to update activation entry")
		return new(core.JoinBrokerRes), err
	}

	var nwkSKey [16]byte
	copy(nwkSKey[:], res.NwkSKey)
	err = b.NetworkController.upsert(devEntry{
		Dialer:  appEntry.Dialer,
		DevAddr: res.DevAddr,
		AppEUI:  req.AppEUI,
		DevEUI:  req.DevEUI,
		NwkSKey: nwkSKey,
		DevMode: false,
		FCntUp:  0,
	})
	if err != nil {
		ctx.WithError(err).Debug("Unable to store device")
		return new(core.JoinBrokerRes), err
	}
	return &core.JoinBrokerRes{
		Payload:  res.Payload,
		Metadata: res.Metadata,
	}, nil
}

// HandleData implements the core.BrokerServer interface
func (b component) HandleData(bctx context.Context, req *core.DataBrokerReq) (*core.DataBrokerRes, error) {
	// Get some logs / analytics
	stats.MarkMeter("broker.uplink.in")

	// Validate incoming data
	uplinkPayload, err := core.NewLoRaWANData(req.Payload, true)
	if err != nil {
		b.Ctx.WithError(err).Debug("Unable to interpret LoRaWAN payload")
		return new(core.DataBrokerRes), errors.New(errors.Structural, err)
	}
	devAddr := req.Payload.MACPayload.FHDR.DevAddr // No nil ref, ensured by NewLoRaWANData()

	ctx := b.Ctx.WithField("DevAddr", devAddr)
	ctx.Debug("Handle uplink")

	// Check whether we should handle it
	entries, err := b.NetworkController.read(devAddr)
	if err != nil {
		switch err.(errors.Failure).Nature {
		case errors.NotFound:
			stats.MarkMeter("broker.uplink.handler_lookup.device_not_found")
			ctx.Debug("Uplink device not found")
		default:
			ctx.WithError(err).Warn("Database lookup failed")
		}
		return new(core.DataBrokerRes), err
	}
	stats.UpdateHistogram("broker.uplink.handler_lookup.entries", int64(len(entries)))

	// Several handlers might be associated to the same device, we distinguish them using
	// MIC check. Only one should verify the MIC check
	// The device only stores a 16-bits counter but could reflect a 32-bits one.
	// The counter is used for the MIC computation, thus, we're gonna try both 16-bits and
	// 32-bits counters.
	// We keep track of the real counter in the network controller.
	fhdr := &uplinkPayload.MACPayload.(*lorawan.MACPayload).FHDR // No nil ref, ensured by NewLoRaWANData()
	fcnt16 := fhdr.FCnt                                          // Keep a reference to the original counter

	fcntReset := false
	var mEntry *devEntry
	for _, entry := range entries {
		// retrieve the network session key
		key := lorawan.AES128Key(entry.NwkSKey)

		// Check with 16-bits counters
		fhdr.FCnt = fcnt16
		fcnt32, counterReset, err := b.NetworkController.wholeCounter(fcnt16, entry.FCntUp)
		if err != nil {
			continue
		}

		if counterReset {
			dtx := ctx.WithFields(log.Fields{
				"DevMode": entry.DevMode,
			})
			dtx.Debug("Counter reset detected")
		}

		ok, err := uplinkPayload.ValidateMIC(key)
		if err != nil {
			continue
		}
		fhdr.FCnt = fcnt32
		if !ok { // Check with 32-bits counter
			ok, err = uplinkPayload.ValidateMIC(key)
			if err != nil {
				continue
			}
		}

		if ok {
			if counterReset {
				if !entry.DevMode {
					continue
				}
				fcntReset = true
			}
			mEntry = &entry
			stats.MarkMeter("broker.uplink.handler_lookup.mic_match")
			ctx = ctx.WithFields(log.Fields{
				"DevEUI":  mEntry.DevEUI,
				"Handler": string(entry.Dialer.MarshalSafely()),
			})
			ctx.Debug("MIC check succeeded")
			break // We stop at the first valid check ...
		}
	}

	if mEntry == nil {
		stats.MarkMeter("broker.uplink.handler_lookup.no_mic_match")
		err := errors.New(errors.NotFound, "FCntUp or MIC check did not match")
		ctx.WithError(err).Debug("Unable to handle uplink")
		return new(core.DataBrokerRes), err
	}

	// It does matter here to use the DevEUI from the entry and not from the packet.
	// The packet actually holds a DevAddr and the real DevEUI has been determined thanks
	// to the MIC check + persistence
	mEntry.FCntUp = fhdr.FCnt
	if err := b.NetworkController.upsert(*mEntry); err != nil {
		ctx.WithError(err).Debug("Unable to update Frame Counter")
		return new(core.DataBrokerRes), err
	}

	// Then we forward the packet to the handler and wait for the response
	handler, closer, err := mEntry.Dialer.Dial()
	if err != nil {
		ctx.WithError(err).Debug("Unable to dial handler")
		return new(core.DataBrokerRes), err
	}
	defer closer.Close()
	resp, err := handler.HandleDataUp(context.Background(), &core.DataUpHandlerReq{
		Payload:  req.Payload.MACPayload.FRMPayload,
		DevEUI:   mEntry.DevEUI,
		AppEUI:   mEntry.AppEUI,
		FCnt:     fhdr.FCnt,
		FPort:    req.Payload.MACPayload.FPort,
		MType:    req.Payload.MHDR.MType,
		Metadata: req.Metadata,
		FCntUpReset:  fcntReset,
	})

	if err != nil {
		stats.MarkMeter("broker.uplink.bad_handler_response")
		ctx.WithError(err).Debug("Unexpected answer from handler")
		return new(core.DataBrokerRes), errors.New(errors.Operational, err)
	}
	stats.MarkMeter("broker.uplink.ok")

	// No response, we stop here and propagate the "no answer".
	// In case of confirmed data, the handler is in charge of creating the confirmation
	if resp == nil || resp.Payload == nil {
		ctx.Debug("No downlink response")
		return new(core.DataBrokerRes), nil
	}
	ctx.Debug("Handle downlink response")

	// If a response was sent, i.e. a downlink data, we need to compute the right MIC
	downlinkPayload, err := core.NewLoRaWANData(resp.Payload, false)
	if err != nil {
		ctx.WithError(err).Debug("Unable to interpret LoRaWAN downlink datagram")
		return new(core.DataBrokerRes), errors.New(errors.Structural, err)
	}
	stats.MarkMeter("broker.downlink.in")
	if err := downlinkPayload.SetMIC(lorawan.AES128Key(mEntry.NwkSKey)); err != nil {
		ctx.WithError(err).Debug("Unable to set MIC")
		return new(core.DataBrokerRes), errors.New(errors.Structural, "Unable to set response MIC")
	}
	resp.Payload.MIC = downlinkPayload.MIC[:]

	// And finally, we acknowledge the answer
	return &core.DataBrokerRes{
		Payload:  resp.Payload,
		Metadata: resp.Metadata,
	}, nil
}
<commit_msg>Fix formatting errors from merge<commit_after>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package broker

import (
	"fmt"
	"net"

	"github.com/TheThingsNetwork/ttn/core"
	"github.com/TheThingsNetwork/ttn/utils/errors"
	"github.com/TheThingsNetwork/ttn/utils/stats"
	"github.com/TheThingsNetwork/ttn/utils/tokenkey"
	"github.com/apex/log"
	"github.com/brocaar/lorawan"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
)

// component implements the core.BrokerServer interface
type component struct {
	Components
	NetAddrUp        string
	NetAddrDown      string
	TokenKeyProvider tokenkey.Provider
	MaxDevNonces     uint
}

// Components defines a structure to make the instantiation easier to read
type Components struct {
	NetworkController NetworkController
	AppStorage        AppStorage
	Ctx               log.Interface
}

// Options defines a structure to make the instantiation easier to read
type Options struct {
	NetAddrUp        string
	NetAddrDown      string
	TokenKeyProvider tokenkey.Provider
}

// Interface defines the Broker interface
type Interface interface {
	core.BrokerServer
	core.BrokerManagerServer
	Start() error
}

// New construct a new Broker component
func New(c Components, o Options) Interface {
	return component{
		Components:       c,
		NetAddrUp:        o.NetAddrUp,
		NetAddrDown:      o.NetAddrDown,
		TokenKeyProvider: o.TokenKeyProvider,
		MaxDevNonces:     10,
	}
}

// Start actually runs the component and starts the rpc server
func (b component) Start() error {
	connUp, err := net.Listen("tcp", b.NetAddrUp)
	if err != nil {
		return errors.New(errors.Operational, err)
	}

	connDown, err := net.Listen("tcp", b.NetAddrDown)
	if err != nil {
		return errors.New(errors.Operational, err)
	}

	if b.TokenKeyProvider != nil {
		tokenKey, err := b.TokenKeyProvider.Get(true)
		if err != nil {
			return errors.New(errors.Operational, fmt.Sprintf("Failed to refresh token key: %s", err.Error()))
		}
		b.Ctx.WithField("provider", b.TokenKeyProvider).Infof("Got token key for algorithm %v", tokenKey.Algorithm)
	}

	server := grpc.NewServer()
	core.RegisterBrokerServer(server, b)
	core.RegisterBrokerManagerServer(server, b)

	cherr := make(chan error)

	go func() {
		cherr <- server.Serve(connUp)
	}()

	go func() {
		cherr <- server.Serve(connDown)
	}()

	if err := <-cherr; err != nil {
		return errors.New(errors.Operational, err)
	}
	return nil
}

// HandleJoin implements the core.BrokerServer interface
func (b component) HandleJoin(bctx context.Context, req *core.JoinBrokerReq) (*core.JoinBrokerRes, error) {
	// Validate incoming data
	if req == nil || len(req.DevEUI) != 8 || len(req.AppEUI) != 8 || len(req.DevNonce) != 2 || len(req.MIC) != 4 || req.Metadata == nil {
		b.Ctx.Debug("Invalid join request. Parameters are incorrect.")
		return new(core.JoinBrokerRes), errors.New(errors.Structural, "Invalid parameters")
	}

	ctx := b.Ctx.WithFields(log.Fields{
		"AppEUI": req.AppEUI,
		"DevEUI": req.DevEUI,
	})

	ctx.Debug("Handle join request")

	// Check if devNonce already referenced
	appEntry, err := b.AppStorage.read(req.AppEUI)
	if err != nil {
		ctx.WithError(err).Debug("Unable to lookup activation")
		return new(core.JoinBrokerRes), err
	}
	nonces, err := b.NetworkController.readNonces(req.AppEUI, req.DevEUI)
	if err != nil {
		if err.(errors.Failure).Nature != errors.NotFound {
			ctx.WithError(err).Debug("Unable to retrieve associated devNonces")
			return new(core.JoinBrokerRes), err
		}
		nonces = noncesEntry{
			AppEUI: req.AppEUI,
			DevEUI: req.DevEUI,
		}
	}

	var found bool
	for _, n := range nonces.DevNonces {
		if n[0] == req.DevNonce[0] && n[1] == req.DevNonce[1] {
			found = true
			break
		}
	}
	if found {
		ctx.Debug("DevNonce already used in the past")
		return new(core.JoinBrokerRes), errors.New(errors.Structural, "DevNonce used by the past")
	}

	// Forward the registration to the handler
	handler, closer, err := appEntry.Dialer.Dial()
	if err != nil {
		ctx.WithError(err).Debug("Unable to dial handler")
		return new(core.JoinBrokerRes), err
	}
	defer closer.Close()
	res, err := handler.HandleJoin(context.Background(), &core.JoinHandlerReq{
		DevEUI:   req.DevEUI,
		AppEUI:   req.AppEUI,
		DevNonce: req.DevNonce,
		MIC:      req.MIC,
		Metadata: req.Metadata,
	})
	if err != nil {
		ctx.WithError(err).Debug("Error while contacting handler")
		return new(core.JoinBrokerRes), errors.New(errors.Operational, err)
	}

	// Handle the response
	if res == nil || res.Payload == nil {
		ctx.Debug("No join-accept received")
		return new(core.JoinBrokerRes), nil
	}

	if len(res.DevAddr) != 4 || len(res.NwkSKey) != 16 {
		ctx.Debug("Invalid response from handler")
		return new(core.JoinBrokerRes), errors.New(errors.Operational, "Invalid response from handler")
	}

	ctx.WithField("DevAddr", res.DevAddr).Debug("Handle join-accept")

	// Update the DevNonce
	nonces.DevNonces = append(nonces.DevNonces, req.DevNonce)
	if uint(len(nonces.DevNonces)) > b.MaxDevNonces {
		nonces.DevNonces = nonces.DevNonces[1:]
	}
	err = b.NetworkController.upsertNonces(nonces)
	if err != nil {
		ctx.WithError(err).Debug("Unable to update activation entry")
		return new(core.JoinBrokerRes), err
	}

	var nwkSKey [16]byte
	copy(nwkSKey[:], res.NwkSKey)
	err = b.NetworkController.upsert(devEntry{
		Dialer:  appEntry.Dialer,
		DevAddr: res.DevAddr,
		AppEUI:  req.AppEUI,
		DevEUI:  req.DevEUI,
		NwkSKey: nwkSKey,
		DevMode: false,
		FCntUp:  0,
	})
	if err != nil {
		ctx.WithError(err).Debug("Unable to store device")
		return new(core.JoinBrokerRes), err
	}
	return &core.JoinBrokerRes{
		Payload:  res.Payload,
		Metadata: res.Metadata,
	}, nil
}

// HandleData implements the core.BrokerServer interface
func (b component) HandleData(bctx context.Context, req *core.DataBrokerReq) (*core.DataBrokerRes, error) {
	// Get some logs / analytics
	stats.MarkMeter("broker.uplink.in")

	// Validate incoming data
	uplinkPayload, err := core.NewLoRaWANData(req.Payload, true)
	if err != nil {
		b.Ctx.WithError(err).Debug("Unable to interpret LoRaWAN payload")
		return new(core.DataBrokerRes), errors.New(errors.Structural, err)
	}
	devAddr := req.Payload.MACPayload.FHDR.DevAddr // No nil ref, ensured by NewLoRaWANData()

	ctx := b.Ctx.WithField("DevAddr", devAddr)
	ctx.Debug("Handle uplink")

	// Check whether we should handle it
	entries, err := b.NetworkController.read(devAddr)
	if err != nil {
		switch err.(errors.Failure).Nature {
		case errors.NotFound:
			stats.MarkMeter("broker.uplink.handler_lookup.device_not_found")
			ctx.Debug("Uplink device not found")
		default:
			ctx.WithError(err).Warn("Database lookup failed")
		}
		return new(core.DataBrokerRes), err
	}
	stats.UpdateHistogram("broker.uplink.handler_lookup.entries", int64(len(entries)))

	// Several handlers might be associated to the same device, we distinguish them using
	// MIC check. Only one should verify the MIC check
	// The device only stores a 16-bits counter but could reflect a 32-bits one.
	// The counter is used for the MIC computation, thus, we're gonna try both 16-bits and
	// 32-bits counters.
	// We keep track of the real counter in the network controller.
	fhdr := &uplinkPayload.MACPayload.(*lorawan.MACPayload).FHDR // No nil ref, ensured by NewLoRaWANData()
	fcnt16 := fhdr.FCnt                                          // Keep a reference to the original counter

	fcntReset := false
	var mEntry *devEntry
	for _, entry := range entries {
		// retrieve the network session key
		key := lorawan.AES128Key(entry.NwkSKey)

		// Check with 16-bits counters
		fhdr.FCnt = fcnt16
		fcnt32, counterReset, err := b.NetworkController.wholeCounter(fcnt16, entry.FCntUp)
		if err != nil {
			continue
		}

		if counterReset {
			dtx := ctx.WithFields(log.Fields{
				"DevMode": entry.DevMode,
			})
			dtx.Debug("Counter reset detected")
		}

		ok, err := uplinkPayload.ValidateMIC(key)
		if err != nil {
			continue
		}
		fhdr.FCnt = fcnt32
		if !ok { // Check with 32-bits counter
			ok, err = uplinkPayload.ValidateMIC(key)
			if err != nil {
				continue
			}
		}

		if ok {
			if counterReset {
				if !entry.DevMode {
					continue
				}
				fcntReset = true
			}
			mEntry = &entry
			stats.MarkMeter("broker.uplink.handler_lookup.mic_match")
			ctx = ctx.WithFields(log.Fields{
				"DevEUI":  mEntry.DevEUI,
				"Handler": string(entry.Dialer.MarshalSafely()),
			})
			ctx.Debug("MIC check succeeded")
			break // We stop at the first valid check ...
		}
	}

	if mEntry == nil {
		stats.MarkMeter("broker.uplink.handler_lookup.no_mic_match")
		err := errors.New(errors.NotFound, "FCntUp or MIC check did not match")
		ctx.WithError(err).Debug("Unable to handle uplink")
		return new(core.DataBrokerRes), err
	}

	// It does matter here to use the DevEUI from the entry and not from the packet.
	// The packet actually holds a DevAddr and the real DevEUI has been determined thanks
	// to the MIC check + persistence
	mEntry.FCntUp = fhdr.FCnt
	if err := b.NetworkController.upsert(*mEntry); err != nil {
		ctx.WithError(err).Debug("Unable to update Frame Counter")
		return new(core.DataBrokerRes), err
	}

	// Then we forward the packet to the handler and wait for the response
	handler, closer, err := mEntry.Dialer.Dial()
	if err != nil {
		ctx.WithError(err).Debug("Unable to dial handler")
		return new(core.DataBrokerRes), err
	}
	defer closer.Close()
	resp, err := handler.HandleDataUp(context.Background(), &core.DataUpHandlerReq{
		Payload:     req.Payload.MACPayload.FRMPayload,
		DevEUI:      mEntry.DevEUI,
		AppEUI:      mEntry.AppEUI,
		FCnt:        fhdr.FCnt,
		FPort:       req.Payload.MACPayload.FPort,
		MType:       req.Payload.MHDR.MType,
		Metadata:    req.Metadata,
		FCntUpReset: fcntReset,
	})

	if err != nil {
		stats.MarkMeter("broker.uplink.bad_handler_response")
		ctx.WithError(err).Debug("Unexpected answer from handler")
		return new(core.DataBrokerRes), errors.New(errors.Operational, err)
	}
	stats.MarkMeter("broker.uplink.ok")

	// No response, we stop here and propagate the "no answer".
	// In case of confirmed data, the handler is in charge of creating the confirmation
	if resp == nil || resp.Payload == nil {
		ctx.Debug("No downlink response")
		return new(core.DataBrokerRes), nil
	}
	ctx.Debug("Handle downlink response")

	// If a response was sent, i.e. a downlink data, we need to compute the right MIC
	downlinkPayload, err := core.NewLoRaWANData(resp.Payload, false)
	if err != nil {
		ctx.WithError(err).Debug("Unable to interpret LoRaWAN downlink datagram")
		return new(core.DataBrokerRes), errors.New(errors.Structural, err)
	}
	stats.MarkMeter("broker.downlink.in")
	if err := downlinkPayload.SetMIC(lorawan.AES128Key(mEntry.NwkSKey)); err != nil {
		ctx.WithError(err).Debug("Unable to set MIC")
		return new(core.DataBrokerRes), errors.New(errors.Structural, "Unable to set response MIC")
	}
	resp.Payload.MIC = downlinkPayload.MIC[:]

	// And finally, we acknowledge the answer
	return &core.DataBrokerRes{
		Payload:  resp.Payload,
		Metadata: resp.Metadata,
	}, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2021 The Ceph-CSI Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"context"
	"encoding/json"
	"fmt"
	"strings"

	"github.com/ceph/ceph-csi/internal/util"

	v1 "k8s.io/api/core/v1"
	apierrs "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/kubernetes/test/e2e/framework"
)

func deleteConfigMap(pluginPath string) error {
	path := pluginPath + configMap

	return retryKubectlFile(cephCSINamespace, kubectlDelete, path, deployTimeout)
}

func createConfigMap(pluginPath string, c kubernetes.Interface, f *framework.Framework) error {
	path := pluginPath + configMap
	cm := v1.ConfigMap{}
	err := unmarshal(path, &cm)
	if err != nil {
		return err
	}

	fsID, stdErr, err := execCommandInToolBoxPod(f, "ceph fsid", rookNamespace)
	if err != nil {
		return err
	}
	if stdErr != "" {
		return fmt.Errorf("error getting fsid %v", stdErr)
	}
	// remove new line present in fsID
	fsID = strings.Trim(fsID, "\n")
	// get mon list
	mons, err := getMons(rookNamespace, c)
	if err != nil {
		return err
	}
	conmap := []util.ClusterInfo{{
		ClusterID:      fsID,
		Monitors:       mons,
		RadosNamespace: radosNamespace,
	}}
	if upgradeTesting {
		subvolumegroup = "csi"
	}
	conmap[0].CephFS.SubvolumeGroup = subvolumegroup
	data, err := json.Marshal(conmap)
	if err != nil {
		return err
	}
	cm.Data["config.json"] = string(data)
	cm.Namespace = cephCSINamespace
	// if the configmap is present update it,during cephcsi helm charts
	// deployment empty configmap gets created we need to override it
	_, err = c.CoreV1().ConfigMaps(cephCSINamespace).Get(context.TODO(), cm.Name, metav1.GetOptions{})

	if err == nil {
		_, updateErr := c.CoreV1().ConfigMaps(cephCSINamespace).Update(context.TODO(), &cm, metav1.UpdateOptions{})
		if updateErr != nil {
			return fmt.Errorf("failed to update configmap: %w", updateErr)
		}
	}
	if apierrs.IsNotFound(err) {
		_, err = c.CoreV1().ConfigMaps(cephCSINamespace).Create(context.TODO(), &cm, metav1.CreateOptions{})
	}

	return err
}

// createCustomConfigMap provides multiple clusters information.
func createCustomConfigMap(
	c kubernetes.Interface,
	pluginPath string,
	clusterInfo map[string]map[string]string) error {
	path := pluginPath + configMap
	cm := v1.ConfigMap{}
	err := unmarshal(path, &cm)
	if err != nil {
		return err
	}
	// get mon list
	mons, err := getMons(rookNamespace, c)
	if err != nil {
		return err
	}
	// get clusterIDs
	var clusterID []string
	for key := range clusterInfo {
		clusterID = append(clusterID, key)
	}
	conmap := make([]util.ClusterInfo, len(clusterID))

	for i, j := range clusterID {
		conmap[i].ClusterID = j
		conmap[i].Monitors = mons
	}

	// fill radosNamespace and subvolgroups
	for cluster, confItems := range clusterInfo {
		for i, j := range confItems {
			switch i {
			case "subvolumeGroup":
				for c := range conmap {
					if conmap[c].ClusterID == cluster {
						conmap[c].CephFS.SubvolumeGroup = j
					}
				}
			case "radosNamespace":
				for c := range conmap {
					if conmap[c].ClusterID == cluster {
						conmap[c].RadosNamespace = j
					}
				}
			}
		}
	}

	data, err := json.Marshal(conmap)
	if err != nil {
		return err
	}
	cm.Data["config.json"] = string(data)
	cm.Namespace = cephCSINamespace
	// since a configmap is already created, update the existing configmap
	_, err = c.CoreV1().ConfigMaps(cephCSINamespace).Update(context.TODO(), &cm, metav1.UpdateOptions{})
	if err != nil {
		return fmt.Errorf("failed to update configmap: %w", err)
	}

	return nil
}
<commit_msg>e2e: return useful error message when ConfigMap creation fails<commit_after>/*
Copyright 2021 The Ceph-CSI Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"context"
	"encoding/json"
	"fmt"
	"strings"

	"github.com/ceph/ceph-csi/internal/util"

	v1 "k8s.io/api/core/v1"
	apierrs "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/kubernetes/test/e2e/framework"
)

func deleteConfigMap(pluginPath string) error {
	path := pluginPath + configMap

	return retryKubectlFile(cephCSINamespace, kubectlDelete, path, deployTimeout)
}

func createConfigMap(pluginPath string, c kubernetes.Interface, f *framework.Framework) error {
	path := pluginPath + configMap
	cm := v1.ConfigMap{}
	err := unmarshal(path, &cm)
	if err != nil {
		return err
	}

	fsID, stdErr, err := execCommandInToolBoxPod(f, "ceph fsid", rookNamespace)
	if err != nil {
		return fmt.Errorf("failed to exec command in toolbox: %w", err)
	}
	if stdErr != "" {
		return fmt.Errorf("error getting fsid %v", stdErr)
	}
	// remove new line present in fsID
	fsID = strings.Trim(fsID, "\n")
	// get mon list
	mons, err := getMons(rookNamespace, c)
	if err != nil {
		return err
	}
	conmap := []util.ClusterInfo{{
		ClusterID:      fsID,
		Monitors:       mons,
		RadosNamespace: radosNamespace,
	}}
	if upgradeTesting {
		subvolumegroup = "csi"
	}
	conmap[0].CephFS.SubvolumeGroup = subvolumegroup
	data, err := json.Marshal(conmap)
	if err != nil {
		return err
	}
	cm.Data["config.json"] = string(data)
	cm.Namespace = cephCSINamespace
	// if the configmap is present update it,during cephcsi helm charts
	// deployment empty configmap gets created we need to override it
	_, err = c.CoreV1().ConfigMaps(cephCSINamespace).Get(context.TODO(), cm.Name, metav1.GetOptions{})

	if err == nil {
		_, updateErr := c.CoreV1().ConfigMaps(cephCSINamespace).Update(context.TODO(), &cm, metav1.UpdateOptions{})
		if updateErr != nil {
			return fmt.Errorf("failed to update configmap: %w", updateErr)
		}
	}
	if apierrs.IsNotFound(err) {
		_, err = c.CoreV1().ConfigMaps(cephCSINamespace).Create(context.TODO(), &cm, metav1.CreateOptions{})
	}

	return err
}

// createCustomConfigMap provides multiple clusters information.
func createCustomConfigMap(
	c kubernetes.Interface,
	pluginPath string,
	clusterInfo map[string]map[string]string) error {
	path := pluginPath + configMap
	cm := v1.ConfigMap{}
	err := unmarshal(path, &cm)
	if err != nil {
		return err
	}
	// get mon list
	mons, err := getMons(rookNamespace, c)
	if err != nil {
		return err
	}
	// get clusterIDs
	var clusterID []string
	for key := range clusterInfo {
		clusterID = append(clusterID, key)
	}
	conmap := make([]util.ClusterInfo, len(clusterID))

	for i, j := range clusterID {
		conmap[i].ClusterID = j
		conmap[i].Monitors = mons
	}

	// fill radosNamespace and subvolgroups
	for cluster, confItems := range clusterInfo {
		for i, j := range confItems {
			switch i {
			case "subvolumeGroup":
				for c := range conmap {
					if conmap[c].ClusterID == cluster {
						conmap[c].CephFS.SubvolumeGroup = j
					}
				}
			case "radosNamespace":
				for c := range conmap {
					if conmap[c].ClusterID == cluster {
						conmap[c].RadosNamespace = j
					}
				}
			}
		}
	}

	data, err := json.Marshal(conmap)
	if err != nil {
		return err
	}
	cm.Data["config.json"] = string(data)
	cm.Namespace = cephCSINamespace
	// since a configmap is already created, update the existing configmap
	_, err = c.CoreV1().ConfigMaps(cephCSINamespace).Update(context.TODO(), &cm, metav1.UpdateOptions{})
	if err != nil {
		return fmt.Errorf("failed to update configmap: %w", err)
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kubelet

import (
	"fmt"
	"net/url"
	"strings"
	"time"

	. "k8s.io/heapster/metrics/core"

	"github.com/golang/glog"
	cadvisor "github.com/google/cadvisor/info/v1"
	"github.com/prometheus/client_golang/prometheus"
	kube_api "k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/client/cache"
	kube_client "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/fields"
)

const (
	infraContainerName = "POD"
	// TODO: following constants are copied from k8s, change to use them directly
	kubernetesPodNameLabel      = "io.kubernetes.pod.name"
	kubernetesPodNamespaceLabel = "io.kubernetes.pod.namespace"
	kubernetesPodUID            = "io.kubernetes.pod.uid"
	kubernetesContainerLabel    = "io.kubernetes.container.name"

	CustomMetricPrefix = "CM:"
)

var (
	// The Kubelet request latencies in microseconds.
	kubeletRequestLatency = prometheus.NewSummaryVec(
		prometheus.SummaryOpts{
			Namespace: "heapster",
			Subsystem: "kubelet",
			Name:      "request_duration_microseconds",
			Help:      "The Kubelet request latencies in microseconds.",
		},
		[]string{"node"},
	)
)

func init() {
	prometheus.MustRegister(kubeletRequestLatency)
}

type cpuVal struct {
	Val       int64
	Timestamp time.Time
}

// Kubelet-provided metrics for pod and system container.
type kubeletMetricsSource struct {
	host          Host
	kubeletClient *KubeletClient
	nodename      string
	hostname      string
	hostId        string
	cpuLastVal    map[string]cpuVal
}

func (this *kubeletMetricsSource) Name() string {
	return this.String()
}

func (this *kubeletMetricsSource) String() string {
	return fmt.Sprintf("kubelet:%s:%d", this.host.IP, this.host.Port)
}

func (this *kubeletMetricsSource) decodeMetrics(c *cadvisor.ContainerInfo) (string, *MetricSet) {
	var metricSetKey string
	cMetrics := &MetricSet{
		MetricValues: map[string]MetricValue{},
		Labels: map[string]string{
			LabelNodename.Key: this.nodename,
			LabelHostname.Key: this.hostname,
			LabelHostID.Key:   this.hostId,
		},
		LabeledMetrics: []LabeledMetric{},
	}

	if isNode(c) {
		metricSetKey = NodeKey(this.nodename)
		cMetrics.Labels[LabelMetricSetType.Key] = MetricSetTypeNode
	} else if isSysContainer(c) {
		cName := getSysContainerName(c)
		metricSetKey = NodeContainerKey(this.nodename, cName)
		cMetrics.Labels[LabelMetricSetType.Key] = MetricSetTypeSystemContainer
		cMetrics.Labels[LabelContainerName.Key] = cName
	} else {
		cName := c.Spec.Labels[kubernetesContainerLabel]
		ns := c.Spec.Labels[kubernetesPodNamespaceLabel]
		podName := c.Spec.Labels[kubernetesPodNameLabel]

		// Support for kubernetes 1.0.*
		if ns == "" && strings.Contains(podName, "/") {
			tokens := strings.SplitN(podName, "/", 2)
			if len(tokens) == 2 {
				ns = tokens[0]
				podName = tokens[1]
			}
		}
		if cName == "" {
			// Better this than nothing. This is a temporary hack for new heapster to work
			// with Kubernetes 1.0.*.
			// TODO: fix this with POD list.
			// Parsing name like:
			// k8s_kube-ui.7f9b83f6_kube-ui-v1-bxj1w_kube-system_9abfb0bd-811f-11e5-b548-42010af00002_e6841e8d
			pos := strings.Index(c.Name, ".")
			if pos >= 0 {
				// remove first 4 chars.
				cName = c.Name[len("k8s_"):pos]
			}
		}

		if cName == "" || ns == "" || podName == "" {
			glog.Errorf("Missing metadata for container %v. Got: %+v", c.Name, c.Spec.Labels)
			return "", nil
		}

		if cName == infraContainerName {
			metricSetKey = PodKey(ns, podName)
			cMetrics.Labels[LabelMetricSetType.Key] = MetricSetTypePod
		} else {
			metricSetKey = PodContainerKey(ns, podName, cName)
			cMetrics.Labels[LabelMetricSetType.Key] = MetricSetTypePodContainer
			cMetrics.Labels[LabelContainerName.Key] = cName
			cMetrics.Labels[LabelContainerBaseImage.Key] = c.Spec.Image
		}
		cMetrics.Labels[LabelPodId.Key] = c.Spec.Labels[kubernetesPodUID]
		cMetrics.Labels[LabelPodName.Key] = podName
		cMetrics.Labels[LabelNamespaceName.Key] = ns
		// Needed for backward compatibility
		cMetrics.Labels[LabelPodNamespace.Key] = ns
	}

	for _, metric := range StandardMetrics {
		if metric.HasValue != nil && metric.HasValue(&c.Spec) {
			cMetrics.MetricValues[metric.Name] = metric.GetValue(&c.Spec, c.Stats[0])
		}
	}

	for _, metric := range LabeledMetrics {
		if metric.HasLabeledMetric != nil && metric.HasLabeledMetric(&c.Spec) {
			labeledMetrics := metric.GetLabeledMetric(&c.Spec, c.Stats[0])
			cMetrics.LabeledMetrics = append(cMetrics.LabeledMetrics, labeledMetrics...)
		}
	}

	if c.Spec.HasCustomMetrics {
	metricloop:
		for _, spec := range c.Spec.CustomMetrics {
			if cmValue, ok := c.Stats[0].CustomMetrics[spec.Name]; ok && cmValue != nil && len(cmValue) >= 1 {
				newest := cmValue[0]
				for _, metricVal := range cmValue {
					if newest.Timestamp.Before(metricVal.Timestamp) {
						newest = metricVal
					}
				}
				mv := MetricValue{}
				switch spec.Type {
				case cadvisor.MetricGauge:
					mv.MetricType = MetricGauge
				case cadvisor.MetricCumulative:
					mv.MetricType = MetricCumulative
				default:
					glog.V(4).Infof("Skipping %s: unknown custom metric type: %v", spec.Name, spec.Type)
					continue metricloop
				}

				switch spec.Format {
				case cadvisor.IntType:
					mv.ValueType = ValueInt64
					mv.IntValue = newest.IntValue
				case cadvisor.FloatType:
					mv.ValueType = ValueFloat
					mv.FloatValue = float32(newest.FloatValue)
				default:
					glog.V(4).Infof("Skipping %s: unknown custom metric format", spec.Name, spec.Format)
					continue metricloop
				}

				cMetrics.MetricValues[CustomMetricPrefix+spec.Name] = mv
			}
		}
	}

	// This is temporary workaround to support cpu/usege_rate metric.
	if currentVal, ok := cMetrics.MetricValues["cpu/usage"]; ok {
		if lastVal, ok := this.cpuLastVal[metricSetKey]; ok {
			// cpu/usage values are in nanoseconds; we want to have it in millicores (that's why constant 1000 is here).
			rateVal := 1000 * (currentVal.IntValue - lastVal.Val) / (c.Stats[0].Timestamp.UnixNano() - lastVal.Timestamp.UnixNano())
			cMetrics.MetricValues["cpu/usage_rate"] = MetricValue{
				ValueType:  ValueInt64,
				MetricType: MetricGauge,
				IntValue:   rateVal,
			}
		}
		this.cpuLastVal[metricSetKey] = cpuVal{
			Val:       currentVal.IntValue,
			Timestamp: c.Stats[0].Timestamp,
		}
	}
	// TODO: add labels: LabelPodNamespaceUID, LabelLabels, LabelResourceID

	return metricSetKey, cMetrics
}

func (this *kubeletMetricsSource) ScrapeMetrics(start, end time.Time) *DataBatch {
	containers, err := this.scrapeKubelet(this.kubeletClient, this.host, start, end)
	if err != nil {
		glog.Errorf("error while getting containers from Kubelet: %v", err)
	}
	glog.Infof("successfully obtained stats for %v containers", len(containers))

	result := &DataBatch{
		Timestamp:  end,
		MetricSets: map[string]*MetricSet{},
	}
	keys := make(map[string]bool)
	for _, c := range containers {
		name, metrics := this.decodeMetrics(&c)
		if name == "" {
			continue
		}
		result.MetricSets[name] = metrics
		keys[name] = true
	}
	// No remember data for pods that have been removed.
	for key := range this.cpuLastVal {
		if _, ok := keys[key]; !ok {
			delete(this.cpuLastVal, key)
		}
	}
	return result
}

func (this *kubeletMetricsSource) scrapeKubelet(client *KubeletClient, host Host, start, end time.Time) ([]cadvisor.ContainerInfo, error) {
	startTime := time.Now()
	defer kubeletRequestLatency.WithLabelValues(this.hostname).Observe(float64(time.Since(startTime)))
	return client.GetAllRawContainers(host, start, end)
}

type kubeletProvider struct {
	nodeLister    *cache.StoreToNodeLister
	reflector     *cache.Reflector
	kubeletClient *KubeletClient
	cpuLastVals   map[string]map[string]cpuVal
}

func (this *kubeletProvider) GetMetricsSources() []MetricsSource {
	sources := []MetricsSource{}
	nodes, err := this.nodeLister.List()
	if err != nil {
		glog.Errorf("error while listing nodes: %v", err)
		return sources
	}

	nodeNames := make(map[string]bool)
	for _, node := range nodes.Items {
		nodeNames[node.Name] = true
		hostname, ip, err := getNodeHostnameAndIP(&node)
		if err != nil {
			glog.Errorf("%v", err)
			continue
		}
		if _, ok := this.cpuLastVals[node.Name]; !ok {
			this.cpuLastVals[node.Name] = make(map[string]cpuVal)
		}
		sources = append(sources, &kubeletMetricsSource{
			host:          Host{IP: ip, Port: this.kubeletClient.GetPort()},
			kubeletClient: this.kubeletClient,
			nodename:      node.Name,
			hostname:      hostname,
			hostId:        node.Spec.ExternalID,
			cpuLastVal:    this.cpuLastVals[node.Name],
		})
	}

	for key := range this.cpuLastVals {
		if _, ok := nodeNames[key]; !ok {
			delete(this.cpuLastVals, key)
		}
	}

	return sources
}

func getNodeHostnameAndIP(node *kube_api.Node) (string, string, error) {
	for _, c := range node.Status.Conditions {
		if c.Type == kube_api.NodeReady && c.Status != kube_api.ConditionTrue {
			return "", "", fmt.Errorf("Node %v is not ready", node.Name)
		}
	}
	hostname, ip := node.Name, ""
	for _, addr := range node.Status.Addresses {
		if addr.Type == kube_api.NodeHostName && addr.Address != "" {
			hostname = addr.Address
		}
		if addr.Type == kube_api.NodeInternalIP && addr.Address != "" {
			ip = addr.Address
		}
	}
	if ip != "" {
		return hostname, ip, nil
	}
	return "", "", fmt.Errorf("Node %v has no valid hostname and/or IP address: %v %v", node.Name, hostname, ip)
}

func NewKubeletProvider(uri *url.URL) (MetricsSourceProvider, error) {
	// create clients
	kubeConfig, kubeletConfig, err := getKubeConfigs(uri)
	if err != nil {
		return nil, err
	}
	kubeClient := kube_client.NewOrDie(kubeConfig)
	kubeletClient, err := NewKubeletClient(kubeletConfig)
	if err != nil {
		return nil, err
	}
	// watch nodes
	lw := cache.NewListWatchFromClient(kubeClient, "nodes", kube_api.NamespaceAll, fields.Everything())
	nodeLister := &cache.StoreToNodeLister{Store: cache.NewStore(cache.MetaNamespaceKeyFunc)}
	reflector := cache.NewReflector(lw, &kube_api.Node{}, nodeLister.Store, time.Hour)
	reflector.Run()

	return &kubeletProvider{
		nodeLister:    nodeLister,
		reflector:     reflector,
		kubeletClient: kubeletClient,
		cpuLastVals:   make(map[string]map[string]cpuVal),
	}, nil
}
<commit_msg>Check if cAdvisor/kubelet response has any stats<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kubelet

import (
	"fmt"
	"net/url"
	"strings"
	"time"

	. "k8s.io/heapster/metrics/core"

	"github.com/golang/glog"
	cadvisor "github.com/google/cadvisor/info/v1"
	"github.com/prometheus/client_golang/prometheus"
	kube_api "k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/client/cache"
	kube_client "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/fields"
)

const (
	infraContainerName = "POD"
	// TODO: following constants are copied from k8s, change to use them directly
	kubernetesPodNameLabel      = "io.kubernetes.pod.name"
	kubernetesPodNamespaceLabel = "io.kubernetes.pod.namespace"
	kubernetesPodUID            = "io.kubernetes.pod.uid"
	kubernetesContainerLabel    = "io.kubernetes.container.name"

	CustomMetricPrefix = "CM:"
)

var (
	// The Kubelet request latencies in microseconds.
	kubeletRequestLatency = prometheus.NewSummaryVec(
		prometheus.SummaryOpts{
			Namespace: "heapster",
			Subsystem: "kubelet",
			Name:      "request_duration_microseconds",
			Help:      "The Kubelet request latencies in microseconds.",
		},
		[]string{"node"},
	)
)

func init() {
	prometheus.MustRegister(kubeletRequestLatency)
}

type cpuVal struct {
	Val       int64
	Timestamp time.Time
}

// Kubelet-provided metrics for pod and system container.
type kubeletMetricsSource struct {
	host          Host
	kubeletClient *KubeletClient
	nodename      string
	hostname      string
	hostId        string
	cpuLastVal    map[string]cpuVal
}

func (this *kubeletMetricsSource) Name() string {
	return this.String()
}

func (this *kubeletMetricsSource) String() string {
	return fmt.Sprintf("kubelet:%s:%d", this.host.IP, this.host.Port)
}

func (this *kubeletMetricsSource) decodeMetrics(c *cadvisor.ContainerInfo) (string, *MetricSet) {
	if len(c.Stats) == 0 {
		return "", nil
	}

	var metricSetKey string
	cMetrics := &MetricSet{
		MetricValues: map[string]MetricValue{},
		Labels: map[string]string{
			LabelNodename.Key: this.nodename,
			LabelHostname.Key: this.hostname,
			LabelHostID.Key:   this.hostId,
		},
		LabeledMetrics: []LabeledMetric{},
	}

	if isNode(c) {
		metricSetKey = NodeKey(this.nodename)
		cMetrics.Labels[LabelMetricSetType.Key] = MetricSetTypeNode
	} else if isSysContainer(c) {
		cName := getSysContainerName(c)
		metricSetKey = NodeContainerKey(this.nodename, cName)
		cMetrics.Labels[LabelMetricSetType.Key] = MetricSetTypeSystemContainer
		cMetrics.Labels[LabelContainerName.Key] = cName
	} else {
		cName := c.Spec.Labels[kubernetesContainerLabel]
		ns := c.Spec.Labels[kubernetesPodNamespaceLabel]
		podName := c.Spec.Labels[kubernetesPodNameLabel]

		// Support for kubernetes 1.0.*
		if ns == "" && strings.Contains(podName, "/") {
			tokens := strings.SplitN(podName, "/", 2)
			if len(tokens) == 2 {
				ns = tokens[0]
				podName = tokens[1]
			}
		}
		if cName == "" {
			// Better this than nothing. This is a temporary hack for new heapster to work
			// with Kubernetes 1.0.*.
			// TODO: fix this with POD list.
			// Parsing name like:
			// k8s_kube-ui.7f9b83f6_kube-ui-v1-bxj1w_kube-system_9abfb0bd-811f-11e5-b548-42010af00002_e6841e8d
			pos := strings.Index(c.Name, ".")
			if pos >= 0 {
				// remove first 4 chars.
				cName = c.Name[len("k8s_"):pos]
			}
		}

		if cName == "" || ns == "" || podName == "" {
			glog.Errorf("Missing metadata for container %v. Got: %+v", c.Name, c.Spec.Labels)
			return "", nil
		}

		if cName == infraContainerName {
			metricSetKey = PodKey(ns, podName)
			cMetrics.Labels[LabelMetricSetType.Key] = MetricSetTypePod
		} else {
			metricSetKey = PodContainerKey(ns, podName, cName)
			cMetrics.Labels[LabelMetricSetType.Key] = MetricSetTypePodContainer
			cMetrics.Labels[LabelContainerName.Key] = cName
			cMetrics.Labels[LabelContainerBaseImage.Key] = c.Spec.Image
		}
		cMetrics.Labels[LabelPodId.Key] = c.Spec.Labels[kubernetesPodUID]
		cMetrics.Labels[LabelPodName.Key] = podName
		cMetrics.Labels[LabelNamespaceName.Key] = ns
		// Needed for backward compatibility
		cMetrics.Labels[LabelPodNamespace.Key] = ns
	}

	for _, metric := range StandardMetrics {
		if metric.HasValue != nil && metric.HasValue(&c.Spec) {
			cMetrics.MetricValues[metric.Name] = metric.GetValue(&c.Spec, c.Stats[0])
		}
	}

	for _, metric := range LabeledMetrics {
		if metric.HasLabeledMetric != nil && metric.HasLabeledMetric(&c.Spec) {
			labeledMetrics := metric.GetLabeledMetric(&c.Spec, c.Stats[0])
			cMetrics.LabeledMetrics = append(cMetrics.LabeledMetrics, labeledMetrics...)
		}
	}

	if c.Spec.HasCustomMetrics {
	metricloop:
		for _, spec := range c.Spec.CustomMetrics {
			if cmValue, ok := c.Stats[0].CustomMetrics[spec.Name]; ok && cmValue != nil && len(cmValue) >= 1 {
				newest := cmValue[0]
				for _, metricVal := range cmValue {
					if newest.Timestamp.Before(metricVal.Timestamp) {
						newest = metricVal
					}
				}
				mv := MetricValue{}
				switch spec.Type {
				case cadvisor.MetricGauge:
					mv.MetricType = MetricGauge
				case cadvisor.MetricCumulative:
					mv.MetricType = MetricCumulative
				default:
					glog.V(4).Infof("Skipping %s: unknown custom metric type: %v", spec.Name, spec.Type)
					continue metricloop
				}

				switch spec.Format {
				case cadvisor.IntType:
					mv.ValueType = ValueInt64
					mv.IntValue = newest.IntValue
				case cadvisor.FloatType:
					mv.ValueType = ValueFloat
					mv.FloatValue = float32(newest.FloatValue)
				default:
					glog.V(4).Infof("Skipping %s: unknown custom metric format", spec.Name, spec.Format)
					continue metricloop
				}

				cMetrics.MetricValues[CustomMetricPrefix+spec.Name] = mv
			}
		}
	}

	// This is temporary workaround to support cpu/usege_rate metric.
	if currentVal, ok := cMetrics.MetricValues["cpu/usage"]; ok {
		if lastVal, ok := this.cpuLastVal[metricSetKey]; ok {
			// cpu/usage values are in nanoseconds; we want to have it in millicores (that's why constant 1000 is here).
			rateVal := 1000 * (currentVal.IntValue - lastVal.Val) / (c.Stats[0].Timestamp.UnixNano() - lastVal.Timestamp.UnixNano())
			cMetrics.MetricValues["cpu/usage_rate"] = MetricValue{
				ValueType:  ValueInt64,
				MetricType: MetricGauge,
				IntValue:   rateVal,
			}
		}
		this.cpuLastVal[metricSetKey] = cpuVal{
			Val:       currentVal.IntValue,
			Timestamp: c.Stats[0].Timestamp,
		}
	}
	// TODO: add labels: LabelPodNamespaceUID, LabelLabels, LabelResourceID

	return metricSetKey, cMetrics
}

func (this *kubeletMetricsSource) ScrapeMetrics(start, end time.Time) *DataBatch {
	containers, err := this.scrapeKubelet(this.kubeletClient, this.host, start, end)
	if err != nil {
		glog.Errorf("error while getting containers from Kubelet: %v", err)
	}
	glog.Infof("successfully obtained stats for %v containers", len(containers))

	result := &DataBatch{
		Timestamp:  end,
		MetricSets: map[string]*MetricSet{},
	}
	keys := make(map[string]bool)
	for _, c := range containers {
		name, metrics := this.decodeMetrics(&c)
		if name == "" || metrics == nil {
			continue
		}
		result.MetricSets[name] = metrics
		keys[name] = true
	}
	// No remember data for pods that have been removed.
	for key := range this.cpuLastVal {
		if _, ok := keys[key]; !ok {
			delete(this.cpuLastVal, key)
		}
	}
	return result
}

func (this *kubeletMetricsSource) scrapeKubelet(client *KubeletClient, host Host, start, end time.Time) ([]cadvisor.ContainerInfo, error) {
	startTime := time.Now()
	defer kubeletRequestLatency.WithLabelValues(this.hostname).Observe(float64(time.Since(startTime)))
	return client.GetAllRawContainers(host, start, end)
}

type kubeletProvider struct {
	nodeLister    *cache.StoreToNodeLister
	reflector     *cache.Reflector
	kubeletClient *KubeletClient
	cpuLastVals   map[string]map[string]cpuVal
}

func (this *kubeletProvider) GetMetricsSources() []MetricsSource {
	sources := []MetricsSource{}
	nodes, err := this.nodeLister.List()
	if err != nil {
		glog.Errorf("error while listing nodes: %v", err)
		return sources
	}

	nodeNames := make(map[string]bool)
	for _, node := range nodes.Items {
		nodeNames[node.Name] = true
		hostname, ip, err := getNodeHostnameAndIP(&node)
		if err != nil {
			glog.Errorf("%v", err)
			continue
		}
		if _, ok := this.cpuLastVals[node.Name]; !ok {
			this.cpuLastVals[node.Name] = make(map[string]cpuVal)
		}
		sources = append(sources, &kubeletMetricsSource{
			host:          Host{IP: ip, Port: this.kubeletClient.GetPort()},
			kubeletClient: this.kubeletClient,
			nodename:      node.Name,
			hostname:      hostname,
			hostId:        node.Spec.ExternalID,
			cpuLastVal:    this.cpuLastVals[node.Name],
		})
	}

	for key := range this.cpuLastVals {
		if _, ok := nodeNames[key]; !ok {
			delete(this.cpuLastVals, key)
		}
	}

	return sources
}

func getNodeHostnameAndIP(node *kube_api.Node) (string, string, error) {
	for _, c := range node.Status.Conditions {
		if c.Type == kube_api.NodeReady && c.Status != kube_api.ConditionTrue {
			return "", "", fmt.Errorf("Node %v is not ready", node.Name)
		}
	}
	hostname, ip := node.Name, ""
	for _, addr := range node.Status.Addresses {
		if addr.Type == kube_api.NodeHostName && addr.Address != "" {
			hostname = addr.Address
		}
		if addr.Type == kube_api.NodeInternalIP && addr.Address != "" {
			ip = addr.Address
		}
	}
	if ip != "" {
		return hostname, ip, nil
	}
	return "", "", fmt.Errorf("Node %v has no valid hostname and/or IP address: %v %v", node.Name, hostname, ip)
}

func NewKubeletProvider(uri *url.URL) (MetricsSourceProvider, error) {
	// create clients
	kubeConfig, kubeletConfig, err := getKubeConfigs(uri)
	if err != nil {
		return nil, err
	}
	kubeClient := kube_client.NewOrDie(kubeConfig)
	kubeletClient, err := NewKubeletClient(kubeletConfig)
	if err != nil {
		return nil, err
	}
	// watch nodes
	lw := cache.NewListWatchFromClient(kubeClient, "nodes", kube_api.NamespaceAll, fields.Everything())
	nodeLister := &cache.StoreToNodeLister{Store: cache.NewStore(cache.MetaNamespaceKeyFunc)}
	reflector := cache.NewReflector(lw, &kube_api.Node{}, nodeLister.Store, time.Hour)
	reflector.Run()

	return &kubeletProvider{
		nodeLister:    nodeLister,
		reflector:     reflector,
		kubeletClient: kubeletClient,
		cpuLastVals:   make(map[string]map[string]cpuVal),
	}, nil
}
<|endoftext|>


<commit_before>package fuse

import (
	"fmt"
	"log"
	"path/filepath"
	"strings"
	"sync"
	"unsafe"
)

type mountData struct {
	// If non-nil the file system mounted here.
	fs FileSystem

	// If yes, we are looking to unmount the mounted fs.
	//
	// To be technically correct, we'd have to have a mutex
	// protecting this.  We don't, keeping the following in mind:
	//
	//  * eventual consistency is OK here
	//
	//  * the kernel controls when to ask for updates,
	//  so we can't make entries disappear directly anyway.
	unmountPending bool

	// We could have separate treeLocks per mount; something to
	// consider if we can measure significant contention for
	// multi-mount filesystems.
}

func newMount(fs FileSystem) *mountData {
	return &mountData{fs: fs}
}

// Tests should set to true.
var paranoia = false

// TODO should rename to dentry?
type inode struct {
	Parent      *inode
	Children    map[string]*inode
	NodeId      uint64
	Name        string
	LookupCount int
	OpenCount   int

	mount *mountData
}

// Should be called with treeLock and fileLock held.
func (me *inode) totalOpenCount() int {
	o := me.OpenCount
	for _, v := range me.Children {
		o += v.totalOpenCount()
	}
	return o
}

// Should be called with treeLock held.
func (me *inode) totalMountCount() int {
	o := 0
	if me.mount != nil && !me.mount.unmountPending {
		o++
	}
	for _, v := range me.Children {
		o += v.totalMountCount()
	}
	return o
}

func (me *inode) IsDir() bool {
	return me.Children != nil
}

const initDirSize = 20

func (me *inode) verify() {
	if !(me.NodeId == FUSE_ROOT_ID || me.LookupCount > 0 || len(me.Children) > 0) {
		panic(fmt.Sprintf("node should be dead: %v", me))
	}
	for n, ch := range me.Children {
		if ch == nil {
			panic("Found nil child.")
		}
		if ch.Name != n {
			panic(fmt.Sprintf("parent/child name corrupted %v %v",
				ch.Name, n))
		}
		if ch.Parent != me {
			panic(fmt.Sprintf("parent/child relation corrupted %v %v %v",
				ch.Parent, me, ch))
		}
	}
}

func (me *inode) GetPath() (path string, mount *mountData) {
	rev_components := make([]string, 0, 10)
	inode := me

	for ; inode != nil && inode.mount == nil; inode = inode.Parent {
		rev_components = append(rev_components, inode.Name)
	}
	if inode == nil {
		panic(fmt.Sprintf("did not find parent with mount: %v", rev_components))
	}
	mount = inode.mount

	if mount.unmountPending {
		return "", nil
	}
	components := make([]string, len(rev_components))
	for i, v := range rev_components {
		components[len(rev_components)-i-1] = v
	}
	fullPath := strings.Join(components, "/")
	return fullPath, mount
}

// Must be called with treeLock held.
func (me *inode) setParent(newParent *inode) {
	if me.Parent == newParent {
		return
	}
	if me.Parent != nil {
		me.Parent.Children[me.Name] = nil, false
		me.Parent = nil
	}
	if newParent != nil {
		me.Parent = newParent
		ch := me.Parent.Children[me.Name]
		if ch != nil {
			panic(fmt.Sprintf("Already have an inode with same name: %v: %v", me.Name, ch))
		}

		me.Parent.Children[me.Name] = me
	}
}

type TimeoutOptions struct {
	EntryTimeout    float64
	AttrTimeout     float64
	NegativeTimeout float64
}

func MakeTimeoutOptions() TimeoutOptions {
	return TimeoutOptions{
		NegativeTimeout: 0.0,
		AttrTimeout:     1.0,
		EntryTimeout:    1.0,
	}
}

type FileSystemConnectorOptions struct {
	TimeoutOptions
}

type FileSystemConnector struct {
	DefaultRawFileSystem

	options FileSystemConnectorOptions
	Debug   bool

	////////////////

	// Protects the inodeMap and each node's Children/Parent
	// relations.
	treeLock sync.RWMutex

	// Invariants: see the verify() method.
	inodeMap      map[uint64]*inode
	rootNode      *inode

	// Open files/directories.
	openFiles      map[uint64]*interfaceBridge

	// Protects openFiles and OpenCount in all of the nodes.
	fileLock       sync.RWMutex
}

type interfaceBridge struct {
	Iface interface{}
}

func (me *FileSystemConnector) DebugString() string {
	me.treeLock.RLock()
	defer me.treeLock.RUnlock()

	me.fileLock.RLock()
	defer me.fileLock.RUnlock()

	root := me.rootNode
	return fmt.Sprintf("Mounts %20d\nFiles %20d\nInodes %20d\n",
		root.totalMountCount(),
		len(me.openFiles), len(me.inodeMap))
}

func (me *FileSystemConnector) unregisterFile(node *inode, handle uint64) interface{} {
	me.fileLock.Lock()
	defer me.fileLock.Unlock()
	b, ok := me.openFiles[handle]
	if !ok {
		panic("invalid handle")
	}
	me.openFiles[handle] = nil, false
	node.OpenCount--
	return b.Iface
}

func (me *FileSystemConnector) registerFile(node *inode, f interface{}) uint64 {
	me.fileLock.Lock()
	defer me.fileLock.Unlock()

	b := &interfaceBridge{
	Iface: f,
	}
	h := uint64(uintptr(unsafe.Pointer(b)))
	_, ok := me.openFiles[h]
	if ok {
		panic("handle counter wrapped")
	}

	node.OpenCount++
	me.openFiles[h] = b
	return h
}

func (me *FileSystemConnector) getDir(h uint64) RawDir {
	b := (*interfaceBridge)(unsafe.Pointer(uintptr(h)))
	return b.Iface.(RawDir)
}

func (me *FileSystemConnector) getFile(h uint64) File {
	b := (*interfaceBridge)(unsafe.Pointer(uintptr(h)))
	return b.Iface.(File)
}

func (me *FileSystemConnector) verify() {
	if !paranoia {
		return
	}
	me.treeLock.Lock()
	defer me.treeLock.Unlock()
	me.fileLock.Lock()
	defer me.fileLock.Unlock()

	hiddenOpen := 0
	for k, v := range me.inodeMap {
		if v.NodeId != k {
			panic(fmt.Sprintf("nodeid mismatch %v %v", v, k))
		}
		if v.Parent == nil && v != me.rootNode {
			hiddenOpen += v.OpenCount
		}
	}

	root := me.rootNode
	root.verify()

	open := root.totalOpenCount()
	openFiles := len(me.openFiles)
	if open+hiddenOpen != openFiles {
		panic(fmt.Sprintf("opencount mismatch totalOpen=%v openFiles=%v mounted=%v hidden=%v", open, openFiles, hiddenOpen))
	}
}

func (me *FileSystemConnector) newInode(root bool, isDir bool) *inode {
	data := new(inode)
	if root {
		data.NodeId = FUSE_ROOT_ID
		me.rootNode = data
	} else {
		data.NodeId = uint64(uintptr(unsafe.Pointer(data)))
	}
	me.inodeMap[data.NodeId] = data
	if isDir {
		data.Children = make(map[string]*inode, initDirSize)
	}

	return data
}

func (me *FileSystemConnector) lookupUpdate(parent *inode, name string, isDir bool) *inode {
	defer me.verify()

	me.treeLock.Lock()
	defer me.treeLock.Unlock()

	data, ok := parent.Children[name]
	if !ok {
		data = me.newInode(false, isDir)
		data.Name = name
		data.setParent(parent)
	}

	return data
}

func (me *FileSystemConnector) getInodeData(nodeid uint64) *inode {
	if nodeid == FUSE_ROOT_ID {
		return me.rootNode
	}

	return (*inode)(unsafe.Pointer(uintptr(nodeid)))
}

func (me *FileSystemConnector) forgetUpdate(nodeId uint64, forgetCount int) {
	defer me.verify()
	me.treeLock.Lock()
	defer me.treeLock.Unlock()

	data, ok := me.inodeMap[nodeId]
	if ok {
		data.LookupCount -= forgetCount
		me.considerDropInode(data)
	}
}

func (me *FileSystemConnector) considerDropInode(n *inode) {
	if n.LookupCount <= 0 && len(n.Children) == 0 && (n.mount == nil || n.mount.unmountPending) &&
		n.OpenCount <= 0 {
		n.setParent(nil)
		me.inodeMap[n.NodeId] = nil, false
	}
}

func (me *FileSystemConnector) renameUpdate(oldParent *inode, oldName string, newParent *inode, newName string) {
	defer me.verify()
	me.treeLock.Lock()
	defer me.treeLock.Unlock()

	node := oldParent.Children[oldName]
	if node == nil {
		panic("Source of rename does not exist")
	}

	dest := newParent.Children[newName]
	if dest != nil {
		dest.setParent(nil)
	}
	node.setParent(nil)
	node.Name = newName
	node.setParent(newParent)
}

func (me *FileSystemConnector) unlinkUpdate(parent *inode, name string) {
	defer me.verify()
	me.treeLock.Lock()
	defer me.treeLock.Unlock()

	node := parent.Children[name]
	node.setParent(nil)
}

// Walk the file system starting from the root.
func (me *FileSystemConnector) findInode(fullPath string) *inode {
	fullPath = strings.TrimLeft(filepath.Clean(fullPath), "/")
	comps := strings.Split(fullPath, "/", -1)

	me.treeLock.RLock()
	defer me.treeLock.RUnlock()

	node := me.rootNode
	for i, component := range comps {
		if len(component) == 0 {
			continue
		}

		node = node.Children[component]
		if node == nil {
			panic(fmt.Sprintf("findInode: %v %v", i, fullPath))
		}
	}
	return node
}

////////////////////////////////////////////////////////////////

func EmptyFileSystemConnector() (out *FileSystemConnector) {
	out = new(FileSystemConnector)
	out.inodeMap = make(map[uint64]*inode)
	out.openFiles = make(map[uint64]*interfaceBridge)

	rootData := out.newInode(true, true)
	rootData.Children = make(map[string]*inode, initDirSize)

	out.options.NegativeTimeout = 0.0
	out.options.AttrTimeout = 1.0
	out.options.EntryTimeout = 1.0
	out.verify()
	return out
}

func (me *FileSystemConnector) Mount(mountPoint string, fs FileSystem) Status {
	var node *inode

	if mountPoint != "/" {
		dirParent, base := filepath.Split(mountPoint)
		dirParentNode := me.findInode(dirParent)

		// Make sure we know the mount point.
		_, _ = me.internalLookup(dirParentNode, base, 0)
	}

	node = me.findInode(mountPoint)
	if !node.IsDir() {
		return EINVAL
	}

	me.treeLock.RLock()
	hasChildren := len(node.Children) > 0
	// don't use defer, as we dont want to hold the lock during
	// fs.Mount().
	me.treeLock.RUnlock()

	if hasChildren {
		return EBUSY
	}

	code := fs.Mount(me)
	if code != OK {
		if me.Debug {
			log.Println("Mount error: ", mountPoint, code)
		}
		return code
	}

	if me.Debug {
		log.Println("Mount: ", fs, "on", mountPoint, node)
	}

	node.mount = newMount(fs)

	return OK
}

func (me *FileSystemConnector) Unmount(path string) Status {
	node := me.findInode(path)
	if node == nil {
		panic(path)
	}

	// Need to lock to look at node.Children
	me.treeLock.RLock()
	me.fileLock.Lock()

	unmountError := OK

	mount := node.mount
	if mount == nil || mount.unmountPending {
		unmountError = EINVAL
	}

	// don't use defer: we don't want to call out to
	// mount.fs.Unmount() with lock held.
	if unmountError == OK && (node.totalOpenCount() > 0 || node.totalMountCount() > 1) {
		unmountError = EBUSY
	}

	if unmountError == OK {
		// We settle for eventual consistency.
		mount.unmountPending = true
	}
	me.fileLock.Unlock()
	me.treeLock.RUnlock()

	if unmountError == OK {
		if me.Debug {
			log.Println("Unmount: ", mount)
		}

		mount.fs.Unmount()
	}
	return unmountError
}

func (me *FileSystemConnector) GetPath(nodeid uint64) (path string, mount *mountData, node *inode) {
	n := me.getInodeData(nodeid)

	// Need to lock because renames create invalid states.
	me.treeLock.RLock()
	defer me.treeLock.RUnlock()

	p, m := n.GetPath()
	return p, m, n
}
<commit_msg>Check for children overwrite only in paranoia mode.<commit_after>package fuse

import (
	"fmt"
	"log"
	"path/filepath"
	"strings"
	"sync"
	"unsafe"
)

type mountData struct {
	// If non-nil the file system mounted here.
	fs FileSystem

	// If yes, we are looking to unmount the mounted fs.
	//
	// To be technically correct, we'd have to have a mutex
	// protecting this.  We don't, keeping the following in mind:
	//
	//  * eventual consistency is OK here
	//
	//  * the kernel controls when to ask for updates,
	//  so we can't make entries disappear directly anyway.
	unmountPending bool

	// We could have separate treeLocks per mount; something to
	// consider if we can measure significant contention for
	// multi-mount filesystems.
}

func newMount(fs FileSystem) *mountData {
	return &mountData{fs: fs}
}

// Tests should set to true.
var paranoia = false

// TODO should rename to dentry?
type inode struct {
	Parent      *inode
	Children    map[string]*inode
	NodeId      uint64
	Name        string
	LookupCount int
	OpenCount   int

	mount *mountData
}

// Should be called with treeLock and fileLock held.
func (me *inode) totalOpenCount() int {
	o := me.OpenCount
	for _, v := range me.Children {
		o += v.totalOpenCount()
	}
	return o
}

// Should be called with treeLock held.
func (me *inode) totalMountCount() int {
	o := 0
	if me.mount != nil && !me.mount.unmountPending {
		o++
	}
	for _, v := range me.Children {
		o += v.totalMountCount()
	}
	return o
}

func (me *inode) IsDir() bool {
	return me.Children != nil
}

const initDirSize = 20

func (me *inode) verify() {
	if !(me.NodeId == FUSE_ROOT_ID || me.LookupCount > 0 || len(me.Children) > 0) {
		panic(fmt.Sprintf("node should be dead: %v", me))
	}
	for n, ch := range me.Children {
		if ch == nil {
			panic("Found nil child.")
		}
		if ch.Name != n {
			panic(fmt.Sprintf("parent/child name corrupted %v %v",
				ch.Name, n))
		}
		if ch.Parent != me {
			panic(fmt.Sprintf("parent/child relation corrupted %v %v %v",
				ch.Parent, me, ch))
		}
	}
}

func (me *inode) GetPath() (path string, mount *mountData) {
	rev_components := make([]string, 0, 10)
	inode := me

	for ; inode != nil && inode.mount == nil; inode = inode.Parent {
		rev_components = append(rev_components, inode.Name)
	}
	if inode == nil {
		panic(fmt.Sprintf("did not find parent with mount: %v", rev_components))
	}
	mount = inode.mount

	if mount.unmountPending {
		return "", nil
	}
	components := make([]string, len(rev_components))
	for i, v := range rev_components {
		components[len(rev_components)-i-1] = v
	}
	fullPath := strings.Join(components, "/")
	return fullPath, mount
}

// Must be called with treeLock held.
func (me *inode) setParent(newParent *inode) {
	if me.Parent == newParent {
		return
	}
	if me.Parent != nil {
		me.Parent.Children[me.Name] = nil, false
		me.Parent = nil
	}
	if newParent != nil {
		me.Parent = newParent

		if paranoia {
			ch := me.Parent.Children[me.Name]
			if ch != nil {
				panic(fmt.Sprintf("Already have an inode with same name: %v: %v", me.Name, ch))
			}
		}

		me.Parent.Children[me.Name] = me
	}
}

type TimeoutOptions struct {
	EntryTimeout    float64
	AttrTimeout     float64
	NegativeTimeout float64
}

func MakeTimeoutOptions() TimeoutOptions {
	return TimeoutOptions{
		NegativeTimeout: 0.0,
		AttrTimeout:     1.0,
		EntryTimeout:    1.0,
	}
}

type FileSystemConnectorOptions struct {
	TimeoutOptions
}

type FileSystemConnector struct {
	DefaultRawFileSystem

	options FileSystemConnectorOptions
	Debug   bool

	////////////////

	// Protects the inodeMap and each node's Children/Parent
	// relations.
	treeLock sync.RWMutex

	// Invariants: see the verify() method.
	inodeMap      map[uint64]*inode
	rootNode      *inode

	// Open files/directories.
	openFiles      map[uint64]*interfaceBridge

	// Protects openFiles and OpenCount in all of the nodes.
	fileLock       sync.RWMutex
}

type interfaceBridge struct {
	Iface interface{}
}

func (me *FileSystemConnector) DebugString() string {
	me.treeLock.RLock()
	defer me.treeLock.RUnlock()

	me.fileLock.RLock()
	defer me.fileLock.RUnlock()

	root := me.rootNode
	return fmt.Sprintf("Mounts %20d\nFiles %20d\nInodes %20d\n",
		root.totalMountCount(),
		len(me.openFiles), len(me.inodeMap))
}

func (me *FileSystemConnector) unregisterFile(node *inode, handle uint64) interface{} {
	me.fileLock.Lock()
	defer me.fileLock.Unlock()
	b, ok := me.openFiles[handle]
	if !ok {
		panic("invalid handle")
	}
	me.openFiles[handle] = nil, false
	node.OpenCount--
	return b.Iface
}

func (me *FileSystemConnector) registerFile(node *inode, f interface{}) uint64 {
	me.fileLock.Lock()
	defer me.fileLock.Unlock()

	b := &interfaceBridge{
	Iface: f,
	}
	h := uint64(uintptr(unsafe.Pointer(b)))
	_, ok := me.openFiles[h]
	if ok {
		panic("handle counter wrapped")
	}

	node.OpenCount++
	me.openFiles[h] = b
	return h
}

func (me *FileSystemConnector) getDir(h uint64) RawDir {
	b := (*interfaceBridge)(unsafe.Pointer(uintptr(h)))
	return b.Iface.(RawDir)
}

func (me *FileSystemConnector) getFile(h uint64) File {
	b := (*interfaceBridge)(unsafe.Pointer(uintptr(h)))
	return b.Iface.(File)
}

func (me *FileSystemConnector) verify() {
	if !paranoia {
		return
	}
	me.treeLock.Lock()
	defer me.treeLock.Unlock()
	me.fileLock.Lock()
	defer me.fileLock.Unlock()

	hiddenOpen := 0
	for k, v := range me.inodeMap {
		if v.NodeId != k {
			panic(fmt.Sprintf("nodeid mismatch %v %v", v, k))
		}
		if v.Parent == nil && v != me.rootNode {
			hiddenOpen += v.OpenCount
		}
	}

	root := me.rootNode
	root.verify()

	open := root.totalOpenCount()
	openFiles := len(me.openFiles)
	if open+hiddenOpen != openFiles {
		panic(fmt.Sprintf("opencount mismatch totalOpen=%v openFiles=%v mounted=%v hidden=%v", open, openFiles, hiddenOpen))
	}
}

func (me *FileSystemConnector) newInode(root bool, isDir bool) *inode {
	data := new(inode)
	if root {
		data.NodeId = FUSE_ROOT_ID
		me.rootNode = data
	} else {
		data.NodeId = uint64(uintptr(unsafe.Pointer(data)))
	}
	me.inodeMap[data.NodeId] = data
	if isDir {
		data.Children = make(map[string]*inode, initDirSize)
	}

	return data
}

func (me *FileSystemConnector) lookupUpdate(parent *inode, name string, isDir bool) *inode {
	defer me.verify()

	me.treeLock.Lock()
	defer me.treeLock.Unlock()

	data, ok := parent.Children[name]
	if !ok {
		data = me.newInode(false, isDir)
		data.Name = name
		data.setParent(parent)
	}

	return data
}

func (me *FileSystemConnector) getInodeData(nodeid uint64) *inode {
	if nodeid == FUSE_ROOT_ID {
		return me.rootNode
	}

	return (*inode)(unsafe.Pointer(uintptr(nodeid)))
}

func (me *FileSystemConnector) forgetUpdate(nodeId uint64, forgetCount int) {
	defer me.verify()
	me.treeLock.Lock()
	defer me.treeLock.Unlock()

	data, ok := me.inodeMap[nodeId]
	if ok {
		data.LookupCount -= forgetCount
		me.considerDropInode(data)
	}
}

func (me *FileSystemConnector) considerDropInode(n *inode) {
	if n.LookupCount <= 0 && len(n.Children) == 0 && (n.mount == nil || n.mount.unmountPending) &&
		n.OpenCount <= 0 {
		n.setParent(nil)
		me.inodeMap[n.NodeId] = nil, false
	}
}

func (me *FileSystemConnector) renameUpdate(oldParent *inode, oldName string, newParent *inode, newName string) {
	defer me.verify()
	me.treeLock.Lock()
	defer me.treeLock.Unlock()

	node := oldParent.Children[oldName]
	if node == nil {
		panic("Source of rename does not exist")
	}

	dest := newParent.Children[newName]
	if dest != nil {
		dest.setParent(nil)
	}
	node.setParent(nil)
	node.Name = newName
	node.setParent(newParent)
}

func (me *FileSystemConnector) unlinkUpdate(parent *inode, name string) {
	defer me.verify()
	me.treeLock.Lock()
	defer me.treeLock.Unlock()

	node := parent.Children[name]
	node.setParent(nil)
}

// Walk the file system starting from the root.
func (me *FileSystemConnector) findInode(fullPath string) *inode {
	fullPath = strings.TrimLeft(filepath.Clean(fullPath), "/")
	comps := strings.Split(fullPath, "/", -1)

	me.treeLock.RLock()
	defer me.treeLock.RUnlock()

	node := me.rootNode
	for i, component := range comps {
		if len(component) == 0 {
			continue
		}

		node = node.Children[component]
		if node == nil {
			panic(fmt.Sprintf("findInode: %v %v", i, fullPath))
		}
	}
	return node
}

////////////////////////////////////////////////////////////////

func EmptyFileSystemConnector() (out *FileSystemConnector) {
	out = new(FileSystemConnector)
	out.inodeMap = make(map[uint64]*inode)
	out.openFiles = make(map[uint64]*interfaceBridge)

	rootData := out.newInode(true, true)
	rootData.Children = make(map[string]*inode, initDirSize)

	out.options.NegativeTimeout = 0.0
	out.options.AttrTimeout = 1.0
	out.options.EntryTimeout = 1.0
	out.verify()
	return out
}

func (me *FileSystemConnector) Mount(mountPoint string, fs FileSystem) Status {
	var node *inode

	if mountPoint != "/" {
		dirParent, base := filepath.Split(mountPoint)
		dirParentNode := me.findInode(dirParent)

		// Make sure we know the mount point.
		_, _ = me.internalLookup(dirParentNode, base, 0)
	}

	node = me.findInode(mountPoint)
	if !node.IsDir() {
		return EINVAL
	}

	me.treeLock.RLock()
	hasChildren := len(node.Children) > 0
	// don't use defer, as we dont want to hold the lock during
	// fs.Mount().
	me.treeLock.RUnlock()

	if hasChildren {
		return EBUSY
	}

	code := fs.Mount(me)
	if code != OK {
		if me.Debug {
			log.Println("Mount error: ", mountPoint, code)
		}
		return code
	}

	if me.Debug {
		log.Println("Mount: ", fs, "on", mountPoint, node)
	}

	node.mount = newMount(fs)

	return OK
}

func (me *FileSystemConnector) Unmount(path string) Status {
	node := me.findInode(path)
	if node == nil {
		panic(path)
	}

	// Need to lock to look at node.Children
	me.treeLock.RLock()
	me.fileLock.Lock()

	unmountError := OK

	mount := node.mount
	if mount == nil || mount.unmountPending {
		unmountError = EINVAL
	}

	// don't use defer: we don't want to call out to
	// mount.fs.Unmount() with lock held.
	if unmountError == OK && (node.totalOpenCount() > 0 || node.totalMountCount() > 1) {
		unmountError = EBUSY
	}

	if unmountError == OK {
		// We settle for eventual consistency.
		mount.unmountPending = true
	}
	me.fileLock.Unlock()
	me.treeLock.RUnlock()

	if unmountError == OK {
		if me.Debug {
			log.Println("Unmount: ", mount)
		}

		mount.fs.Unmount()
	}
	return unmountError
}

func (me *FileSystemConnector) GetPath(nodeid uint64) (path string, mount *mountData, node *inode) {
	n := me.getInodeData(nodeid)

	// Need to lock because renames create invalid states.
	me.treeLock.RLock()
	defer me.treeLock.RUnlock()

	p, m := n.GetPath()
	return p, m, n
}
<|endoftext|>


<commit_before>package beacon

import (
	"crypto/rand"
	"encoding/hex"
	"fmt"
	"html/template"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"

	"appengine"
	"appengine/delay"
	"appengine/urlfetch"
)

const beaconURL = "http://www.google-analytics.com/collect"

var (
	pixel        = mustReadFile("static/pixel.gif")
	badge        = mustReadFile("static/badge.svg")
	badgeGif     = mustReadFile("static/badge.gif")
	badgeFlat    = mustReadFile("static/badge-flat.svg")
	badgeFlatGif = mustReadFile("static/badge-flat.gif")
	pageTemplate = template.Must(template.New("page").ParseFiles("ga-beacon/page.html"))
)

func init() {
	http.HandleFunc("/", handler)
}

func mustReadFile(path string) []byte {
	b, err := ioutil.ReadFile(path)
	if err != nil {
		panic(err)
	}
	return b
}

func generateUUID(cid *string) error {
	b := make([]byte, 16)
	_, err := rand.Read(b)
	if err != nil {
		return err
	}

	b[8] = (b[8] | 0x80) & 0xBF // what's the purpose ?
	b[6] = (b[6] | 0x40) & 0x4F // what's the purpose ?
	*cid = hex.EncodeToString(b)
	return nil
}

var delayHit = delay.Func("collect", logHit)

func log(c appengine.Context, ua string, ip string, cid string, values url.Values) error {
	req, _ := http.NewRequest("POST", beaconURL, strings.NewReader(values.Encode()))
	req.Header.Add("User-Agent", ua)
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")

	if resp, err := urlfetch.Client(c).Do(req); err != nil {
		c.Errorf("GA collector POST error: %s", err.Error())
		return err
	} else {
		c.Debugf("GA collector status: %v, cid: %v, ip: %s", resp.Status, cid, ip)
		c.Debugf("Reported payload: %v", values)
	}
	return nil
}

func logHit(c appengine.Context, params []string, query url.Values, ua string, ip string, cid string) error {
	// 1) Initialize default values from path structure
	// 2) Allow query param override to report arbitrary values to GA
	//
	// GA Protocol reference: https://developers.google.com/analytics/devguides/collection/protocol/v1/reference

	payload := url.Values{
		"v":   {"1"},        // protocol version = 1
		"t":   {"pageview"}, // hit type
		"tid": {params[0]},  // tracking / property ID
		"cid": {cid},        // unique client ID (server generated UUID)
		"dp":  {params[1]},  // page path
		"uip": {ip},         // IP address of the user
	}

	for key, val := range query {
		payload[key] = val
	}

	return log(c, ua, ip, cid, payload)
}

func handler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	params := strings.SplitN(strings.Trim(r.URL.Path, "/"), "/", 2)
	query, _ := url.ParseQuery(r.URL.RawQuery)
	refOrg := r.Header.Get("Referer")

	// / -> redirect
	if len(params[0]) == 0 {
		http.Redirect(w, r, "https://github.com/igrigorik/ga-beacon", http.StatusFound)
		return
	}

	// activate referrer path if ?useReferer is used and if referer exists
	if _, ok := query["useReferer"]; ok {
		if len(refOrg) != 0 {
			referer := strings.Replace(strings.Replace(refOrg, "http://", "", 1), "https://", "", 1);
			if len(referer) != 0 {
				// if the useReferer is present and the referer information exists
				//  the path is ignored and the beacon referer information is used instead.
				params = strings.SplitN(strings.Trim(r.URL.Path, "/") + "/" + referer, "/", 2)
			}
		}
	}
	// /account -> account template
	if len(params) == 1 {
		templateParams := struct {
			Account string
			Referer string
		}{
			Account: params[0],
			Referer: refOrg,
		}
		if err := pageTemplate.ExecuteTemplate(w, "page.html", templateParams); err != nil {
			http.Error(w, "could not show account page", 500)
			c.Errorf("Cannot execute template: %v", err)
		}
		return
	}

	// /account/page -> GIF + log pageview to GA collector
	var cid string
	if cookie, err := r.Cookie("cid"); err != nil {
		if err := generateUUID(&cid); err != nil {
			c.Debugf("Failed to generate client UUID: %v", err)
		} else {
			c.Debugf("Generated new client UUID: %v", cid)
			http.SetCookie(w, &http.Cookie{Name: "cid", Value: cid, Path: fmt.Sprint("/", params[0])})
		}
	} else {
		cid = cookie.Value
		c.Debugf("Existing CID found: %v", cid)
	}

	if len(cid) != 0 {
		w.Header().Set("Cache-Control", "no-cache")
		w.Header().Set("CID", cid)
		w.Header().Set("Expires", "Wed, 07 Jun 2017 19:47:34 GMT")

		logHit(c, params, query, r.Header.Get("User-Agent"), r.RemoteAddr, cid)
		// delayHit.Call(c, params, r.Header.Get("User-Agent"), cid)
	}

	// Write out GIF pixel or badge, based on presence of "pixel" param.
	if _, ok := query["pixel"]; ok {
		w.Header().Set("Content-Type", "image/gif")
		w.Write(pixel)
	} else if _, ok := query["gif"]; ok {
		w.Header().Set("Content-Type", "image/gif")
		w.Write(badgeGif)
	} else if _, ok := query["flat"]; ok {
		w.Header().Set("Content-Type", "image/svg+xml")
		w.Write(badgeFlat)
	} else if _, ok := query["flat-gif"]; ok {
		w.Header().Set("Content-Type", "image/gif")
		w.Write(badgeFlatGif)
	} else {
		w.Header().Set("Content-Type", "image/svg+xml")
		w.Write(badge)
	}
}
<commit_msg>Removed expires and update CacheControl<commit_after>package beacon

import (
	"crypto/rand"
	"encoding/hex"
	"fmt"
	"html/template"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"

	"appengine"
	"appengine/delay"
	"appengine/urlfetch"
)

const beaconURL = "http://www.google-analytics.com/collect"

var (
	pixel        = mustReadFile("static/pixel.gif")
	badge        = mustReadFile("static/badge.svg")
	badgeGif     = mustReadFile("static/badge.gif")
	badgeFlat    = mustReadFile("static/badge-flat.svg")
	badgeFlatGif = mustReadFile("static/badge-flat.gif")
	pageTemplate = template.Must(template.New("page").ParseFiles("ga-beacon/page.html"))
)

func init() {
	http.HandleFunc("/", handler)
}

func mustReadFile(path string) []byte {
	b, err := ioutil.ReadFile(path)
	if err != nil {
		panic(err)
	}
	return b
}

func generateUUID(cid *string) error {
	b := make([]byte, 16)
	_, err := rand.Read(b)
	if err != nil {
		return err
	}

	b[8] = (b[8] | 0x80) & 0xBF // what's the purpose ?
	b[6] = (b[6] | 0x40) & 0x4F // what's the purpose ?
	*cid = hex.EncodeToString(b)
	return nil
}

var delayHit = delay.Func("collect", logHit)

func log(c appengine.Context, ua string, ip string, cid string, values url.Values) error {
	req, _ := http.NewRequest("POST", beaconURL, strings.NewReader(values.Encode()))
	req.Header.Add("User-Agent", ua)
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")

	if resp, err := urlfetch.Client(c).Do(req); err != nil {
		c.Errorf("GA collector POST error: %s", err.Error())
		return err
	} else {
		c.Debugf("GA collector status: %v, cid: %v, ip: %s", resp.Status, cid, ip)
		c.Debugf("Reported payload: %v", values)
	}
	return nil
}

func logHit(c appengine.Context, params []string, query url.Values, ua string, ip string, cid string) error {
	// 1) Initialize default values from path structure
	// 2) Allow query param override to report arbitrary values to GA
	//
	// GA Protocol reference: https://developers.google.com/analytics/devguides/collection/protocol/v1/reference

	payload := url.Values{
		"v":   {"1"},        // protocol version = 1
		"t":   {"pageview"}, // hit type
		"tid": {params[0]},  // tracking / property ID
		"cid": {cid},        // unique client ID (server generated UUID)
		"dp":  {params[1]},  // page path
		"uip": {ip},         // IP address of the user
	}

	for key, val := range query {
		payload[key] = val
	}

	return log(c, ua, ip, cid, payload)
}

func handler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	params := strings.SplitN(strings.Trim(r.URL.Path, "/"), "/", 2)
	query, _ := url.ParseQuery(r.URL.RawQuery)
	refOrg := r.Header.Get("Referer")

	// / -> redirect
	if len(params[0]) == 0 {
		http.Redirect(w, r, "https://github.com/igrigorik/ga-beacon", http.StatusFound)
		return
	}

	// activate referrer path if ?useReferer is used and if referer exists
	if _, ok := query["useReferer"]; ok {
		if len(refOrg) != 0 {
			referer := strings.Replace(strings.Replace(refOrg, "http://", "", 1), "https://", "", 1);
			if len(referer) != 0 {
				// if the useReferer is present and the referer information exists
				//  the path is ignored and the beacon referer information is used instead.
				params = strings.SplitN(strings.Trim(r.URL.Path, "/") + "/" + referer, "/", 2)
			}
		}
	}
	// /account -> account template
	if len(params) == 1 {
		templateParams := struct {
			Account string
			Referer string
		}{
			Account: params[0],
			Referer: refOrg,
		}
		if err := pageTemplate.ExecuteTemplate(w, "page.html", templateParams); err != nil {
			http.Error(w, "could not show account page", 500)
			c.Errorf("Cannot execute template: %v", err)
		}
		return
	}

	// /account/page -> GIF + log pageview to GA collector
	var cid string
	if cookie, err := r.Cookie("cid"); err != nil {
		if err := generateUUID(&cid); err != nil {
			c.Debugf("Failed to generate client UUID: %v", err)
		} else {
			c.Debugf("Generated new client UUID: %v", cid)
			http.SetCookie(w, &http.Cookie{Name: "cid", Value: cid, Path: fmt.Sprint("/", params[0])})
		}
	} else {
		cid = cookie.Value
		c.Debugf("Existing CID found: %v", cid)
	}

	if len(cid) != 0 {
		w.Header().Set("Cache-Control", "private, no-store")
		w.Header().Set("CID", cid)

		logHit(c, params, query, r.Header.Get("User-Agent"), r.RemoteAddr, cid)
		// delayHit.Call(c, params, r.Header.Get("User-Agent"), cid)
	}

	// Write out GIF pixel or badge, based on presence of "pixel" param.
	if _, ok := query["pixel"]; ok {
		w.Header().Set("Content-Type", "image/gif")
		w.Write(pixel)
	} else if _, ok := query["gif"]; ok {
		w.Header().Set("Content-Type", "image/gif")
		w.Write(badgeGif)
	} else if _, ok := query["flat"]; ok {
		w.Header().Set("Content-Type", "image/svg+xml")
		w.Write(badgeFlat)
	} else if _, ok := query["flat-gif"]; ok {
		w.Header().Set("Content-Type", "image/gif")
		w.Write(badgeFlatGif)
	} else {
		w.Header().Set("Content-Type", "image/svg+xml")
		w.Write(badge)
	}
}
<|endoftext|>