<commit_before>package broker_acceptance_test

import (
	"encoding/json"
	"fmt"
	"strings"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	"github.com/google/uuid"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gexec"
)

var _ = Describe("Common service tests", func() {
	Context("Shareable services", func() {
		shareableServices := map[string]bool{
			"autoscaler":    false,
			"elasticsearch": true,
			"influxdb":      true,
			"mysql":         true,
			"postgres":      true,
			"redis":         true,
			"aws-s3-bucket": true,
			"aws-sqs-queue": true,
			"cdn-route":     false,
		}

		It("is service shareable", func() {

			retrieveServicesCommand := cf.Cf("curl", "/v3/service_offerings")

			Expect(retrieveServicesCommand.Wait(testConfig.DefaultTimeoutDuration())).To(Exit(0))

			var servicesCommandResp struct {
				Pagination struct {
					TotalResults int `json:"total_results"`
				}
				Resources []struct {
					Name          string `json:"name"`
					Description   string `json:"description"`
					BrokerCatalog struct {
						Metadata struct {
							Shareable bool `json:"shareable"`
						}
					} `json:"broker_catalog"`
				}
			}

			err := json.Unmarshal(retrieveServicesCommand.Buffer().Contents(), &servicesCommandResp)
			Expect(err).NotTo(HaveOccurred())

			var message string
			//during the run of the pipeline we are running few acceptance tests in parallel
			//this can bring a total number of services in the CF instance to greater than 7 (the normal count)
			//hence, while checking for 'shareable' services - we need to filter out the fake services created
			//by other tests.
			// fake service can be identified by having a value 'fake service' in desciption field.
			fakeServicesCount := 0
			for _, service := range servicesCommandResp.Resources {

				if service.Description == "fake service" {
					fakeServicesCount++
					continue
				}

				message = fmt.Sprintf("verifying that %s backing service is shareable", service.Name)
				By(message)
				if shareableServices[service.Name] {
					Expect(service.BrokerCatalog.Metadata.Shareable).NotTo(BeNil(),
						"Expected %s to have 'shareable' parameter", service.Name)

					Expect(service.BrokerCatalog.Metadata.Shareable).To(BeTrue(),
						"Expected %s to be shareable - i.e.: 'shareable' parameter set to 'true'", service.Name)
				} else {
					Expect(service.BrokerCatalog.Metadata.Shareable).To(BeFalse(),
						"Expected %s NOT to have 'shareable' parameter or to be set to 'false'", service.Name)
				}
			}

			Expect(servicesCommandResp.Pagination.TotalResults-fakeServicesCount).To(BeNumerically("==", len(shareableServices)), "the amount of services doesn't match")
		})
	})

	Context("Service offerings", func() {
		It("has a valid id", func() {
			offerings, err := cfClient.ListServices()
			Expect(err).NotTo(HaveOccurred())

			invalidServiceOfferings := []string{}
			for _, offering := range offerings {
				if strings.HasPrefix(offering.Label, "CATS-") {
					continue
				}

				if _, err := uuid.Parse(offering.UniqueID); err != nil {
					invalidServiceOfferings = append(invalidServiceOfferings, offering.Label)
				}
			}

			Expect(invalidServiceOfferings).To(
				BeEmpty(), "All service offerings should have a GUID unique ID",
			)
		})
	})

	Context("Service plans", func() {
		It("has a valid ID", func() {
			plans, err := cfClient.ListServicePlans()

			Expect(err).NotTo(HaveOccurred())

			invalidServicePlans := []string{}
			for _, plan := range plans {
				if plan.Name == "shared" {
					continue
				}

				if strings.HasPrefix(plan.Name, "fake-") || strings.HasPrefix(plan.UniqueId, "CATS-") {
					continue
				}

				if _, err := uuid.Parse(plan.UniqueId); err != nil {
					invalidServicePlans = append(invalidServicePlans, plan.Name)
				}
			}

			Expect(invalidServicePlans).To(
				BeEmpty(), "All service plans should have a GUID unique ID",
			)
		})
	})
})
<commit_msg>tests: skip flakey shared services tests<commit_after>package broker_acceptance_test

import (
	"encoding/json"
	"fmt"
	"strings"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	"github.com/google/uuid"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gexec"
)

var _ = Describe("Common service tests", func() {
	Context("Shareable services", func() {
		shareableServices := map[string]bool{
			"autoscaler":    false,
			"elasticsearch": true,
			"influxdb":      true,
			"mysql":         true,
			"postgres":      true,
			"redis":         true,
			"aws-s3-bucket": true,
			"aws-sqs-queue": true,
			"cdn-route":     false,
		}

		It("is service shareable", func() {

			retrieveServicesCommand := cf.Cf("curl", "/v3/service_offerings")

			Expect(retrieveServicesCommand.Wait(testConfig.DefaultTimeoutDuration())).To(Exit(0))

			var servicesCommandResp struct {
				Pagination struct {
					TotalResults int `json:"total_results"`
				}
				Resources []struct {
					Name          string `json:"name"`
					Description   string `json:"description"`
					BrokerCatalog struct {
						Metadata struct {
							Shareable bool `json:"shareable"`
						}
					} `json:"broker_catalog"`
				}
			}

			err := json.Unmarshal(retrieveServicesCommand.Buffer().Contents(), &servicesCommandResp)
			Expect(err).NotTo(HaveOccurred())

			var message string
			for _, service := range servicesCommandResp.Resources {
				message = fmt.Sprintf("verifying that %s backing service is shareable", service.Name)
				By(message)

				if _, found := shareableServices[service.Name]; !found {
					continue
				}

				if shareableServices[service.Name] {
					Expect(service.BrokerCatalog.Metadata.Shareable).NotTo(BeNil(),
						"Expected %s to have 'shareable' parameter", service.Name)

					Expect(service.BrokerCatalog.Metadata.Shareable).To(BeTrue(),
						"Expected %s to be shareable - i.e.: 'shareable' parameter set to 'true'", service.Name)
				} else {
					Expect(service.BrokerCatalog.Metadata.Shareable).To(BeFalse(),
						"Expected %s NOT to have 'shareable' parameter or to be set to 'false'", service.Name)
				}
			}
		})
	})

	Context("Service offerings", func() {
		It("has a valid id", func() {
			offerings, err := cfClient.ListServices()
			Expect(err).NotTo(HaveOccurred())

			invalidServiceOfferings := []string{}
			for _, offering := range offerings {
				if strings.HasPrefix(offering.Label, "CATS-") {
					continue
				}

				if _, err := uuid.Parse(offering.UniqueID); err != nil {
					invalidServiceOfferings = append(invalidServiceOfferings, offering.Label)
				}
			}

			Expect(invalidServiceOfferings).To(
				BeEmpty(), "All service offerings should have a GUID unique ID",
			)
		})
	})

	Context("Service plans", func() {
		It("has a valid ID", func() {
			plans, err := cfClient.ListServicePlans()

			Expect(err).NotTo(HaveOccurred())

			invalidServicePlans := []string{}
			for _, plan := range plans {
				if plan.Name == "shared" {
					continue
				}

				if strings.HasPrefix(plan.Name, "fake-") || strings.HasPrefix(plan.UniqueId, "CATS-") {
					continue
				}

				if _, err := uuid.Parse(plan.UniqueId); err != nil {
					invalidServicePlans = append(invalidServicePlans, plan.Name)
				}
			}

			Expect(invalidServicePlans).To(
				BeEmpty(), "All service plans should have a GUID unique ID",
			)
		})
	})
})
<|endoftext|>


<commit_before>/*
 * MinIO Cloud Storage, (C) 2018 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package target

import (
	"context"
	"crypto/tls"
	"encoding/json"
	"errors"
	"net/url"
	"os"
	"path/filepath"

	"github.com/nsqio/go-nsq"

	"github.com/minio/minio/pkg/event"
	xnet "github.com/minio/minio/pkg/net"
)

// NSQ constants
const (
	NSQAddress       = "nsqd_address"
	NSQTopic         = "topic"
	NSQTLS           = "tls"
	NSQTLSSkipVerify = "tls_skip_verify"
	NSQQueueDir      = "queue_dir"
	NSQQueueLimit    = "queue_limit"

	EnvNSQEnable        = "MINIO_NOTIFY_NSQ"
	EnvNSQAddress       = "MINIO_NOTIFY_NSQ_NSQD_ADDRESS"
	EnvNSQTopic         = "MINIO_NOTIFY_NSQ_TOPIC"
	EnvNSQTLS           = "MINIO_NOTIFY_NSQ_TLS"
	EnvNSQTLSSkipVerify = "MINIO_NOTIFY_NSQ_TLS_SKIP_VERIFY"
	EnvNSQQueueDir      = "MINIO_NOTIFY_NSQ_QUEUE_DIR"
	EnvNSQQueueLimit    = "MINIO_NOTIFY_NSQ_QUEUE_LIMIT"
)

// NSQArgs - NSQ target arguments.
type NSQArgs struct {
	Enable      bool      `json:"enable"`
	NSQDAddress xnet.Host `json:"nsqdAddress"`
	Topic       string    `json:"topic"`
	TLS         struct {
		Enable     bool `json:"enable"`
		SkipVerify bool `json:"skipVerify"`
	} `json:"tls"`
	QueueDir   string `json:"queueDir"`
	QueueLimit uint64 `json:"queueLimit"`
}

// Validate NSQArgs fields
func (n NSQArgs) Validate() error {
	if !n.Enable {
		return nil
	}

	if n.NSQDAddress.IsEmpty() {
		return errors.New("empty nsqdAddress")
	}

	if n.Topic == "" {
		return errors.New("empty topic")
	}
	if n.QueueDir != "" {
		if !filepath.IsAbs(n.QueueDir) {
			return errors.New("queueDir path should be absolute")
		}
	}

	return nil
}

// NSQTarget - NSQ target.
type NSQTarget struct {
	id         event.TargetID
	args       NSQArgs
	producer   *nsq.Producer
	store      Store
	config     *nsq.Config
	loggerOnce func(ctx context.Context, err error, id interface{}, errKind ...interface{})
}

// ID - returns target ID.
func (target *NSQTarget) ID() event.TargetID {
	return target.id
}

// HasQueueStore - Checks if the queueStore has been configured for the target
func (target *NSQTarget) HasQueueStore() bool {
	return target.store != nil
}

// IsActive - Return true if target is up and active
func (target *NSQTarget) IsActive() (bool, error) {
	if target.producer == nil {
		producer, err := nsq.NewProducer(target.args.NSQDAddress.String(), target.config)
		if err != nil {
			return false, err
		}
		target.producer = producer
	}

	if err := target.producer.Ping(); err != nil {
		// To treat "connection refused" errors as errNotConnected.
		if IsConnRefusedErr(err) {
			return false, errNotConnected
		}
		return false, err
	}
	return true, nil
}

// Save - saves the events to the store which will be replayed when the nsq connection is active.
func (target *NSQTarget) Save(eventData event.Event) error {
	if target.store != nil {
		return target.store.Put(eventData)
	}
	_, err := target.IsActive()
	if err != nil {
		return err
	}
	return target.send(eventData)
}

// send - sends an event to the NSQ.
func (target *NSQTarget) send(eventData event.Event) error {
	objectName, err := url.QueryUnescape(eventData.S3.Object.Key)
	if err != nil {
		return err
	}
	key := eventData.S3.Bucket.Name + "/" + objectName

	data, err := json.Marshal(event.Log{EventName: eventData.EventName, Key: key, Records: []event.Event{eventData}})
	if err != nil {
		return err
	}

	return target.producer.Publish(target.args.Topic, data)
}

// Send - reads an event from store and sends it to NSQ.
func (target *NSQTarget) Send(eventKey string) error {
	_, err := target.IsActive()
	if err != nil {
		return err
	}

	eventData, eErr := target.store.Get(eventKey)
	if eErr != nil {
		// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
		// Such events will not exist and wouldve been already been sent successfully.
		if os.IsNotExist(eErr) {
			return nil
		}
		return eErr
	}

	if err := target.send(eventData); err != nil {
		return err
	}

	// Delete the event from store.
	return target.store.Del(eventKey)
}

// Close - closes underneath connections to NSQD server.
func (target *NSQTarget) Close() (err error) {
	if target.producer != nil {
		// this blocks until complete:
		target.producer.Stop()
	}
	return nil
}

// NewNSQTarget - creates new NSQ target.
func NewNSQTarget(id string, args NSQArgs, doneCh <-chan struct{}, loggerOnce func(ctx context.Context, err error, id interface{}, kind ...interface{}), test bool) (*NSQTarget, error) {
	config := nsq.NewConfig()
	if args.TLS.Enable {
		config.TlsV1 = true
		config.TlsConfig = &tls.Config{
			InsecureSkipVerify: args.TLS.SkipVerify,
		}
	}

	var store Store

	target := &NSQTarget{
		id:         event.TargetID{ID: id, Name: "nsq"},
		args:       args,
		config:     config,
		loggerOnce: loggerOnce,
	}

	if args.QueueDir != "" {
		queueDir := filepath.Join(args.QueueDir, storePrefix+"-nsq-"+id)
		store = NewQueueStore(queueDir, args.QueueLimit)
		if oErr := store.Open(); oErr != nil {
			target.loggerOnce(context.Background(), oErr, target.ID())
			return nil, oErr
		}
		target.store = store
	}

	producer, err := nsq.NewProducer(args.NSQDAddress.String(), config)
	if err != nil {
		target.loggerOnce(context.Background(), err, target.ID())
		return target, err
	}
	target.producer = producer

	if err := target.producer.Ping(); err != nil {
		// To treat "connection refused" errors as errNotConnected.
		if target.store == nil || !(IsConnRefusedErr(err) || IsConnResetErr(err)) {
			target.loggerOnce(context.Background(), err, target.ID())
			return nil, err
		}
	}

	if target.store != nil && !test {
		// Replays the events from the store.
		eventKeyCh := replayEvents(target.store, doneCh, target.loggerOnce, target.ID())
		// Start replaying events from the store.
		go sendEvents(target, eventKeyCh, doneCh, target.loggerOnce)
	}

	return target, nil
}
<commit_msg>fix: a type in NSQ notification target environment key (#10118)<commit_after>/*
 * MinIO Cloud Storage, (C) 2018 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package target

import (
	"context"
	"crypto/tls"
	"encoding/json"
	"errors"
	"net/url"
	"os"
	"path/filepath"

	"github.com/nsqio/go-nsq"

	"github.com/minio/minio/pkg/event"
	xnet "github.com/minio/minio/pkg/net"
)

// NSQ constants
const (
	NSQAddress       = "nsqd_address"
	NSQTopic         = "topic"
	NSQTLS           = "tls"
	NSQTLSSkipVerify = "tls_skip_verify"
	NSQQueueDir      = "queue_dir"
	NSQQueueLimit    = "queue_limit"

	EnvNSQEnable        = "MINIO_NOTIFY_NSQ_ENABLE"
	EnvNSQAddress       = "MINIO_NOTIFY_NSQ_NSQD_ADDRESS"
	EnvNSQTopic         = "MINIO_NOTIFY_NSQ_TOPIC"
	EnvNSQTLS           = "MINIO_NOTIFY_NSQ_TLS"
	EnvNSQTLSSkipVerify = "MINIO_NOTIFY_NSQ_TLS_SKIP_VERIFY"
	EnvNSQQueueDir      = "MINIO_NOTIFY_NSQ_QUEUE_DIR"
	EnvNSQQueueLimit    = "MINIO_NOTIFY_NSQ_QUEUE_LIMIT"
)

// NSQArgs - NSQ target arguments.
type NSQArgs struct {
	Enable      bool      `json:"enable"`
	NSQDAddress xnet.Host `json:"nsqdAddress"`
	Topic       string    `json:"topic"`
	TLS         struct {
		Enable     bool `json:"enable"`
		SkipVerify bool `json:"skipVerify"`
	} `json:"tls"`
	QueueDir   string `json:"queueDir"`
	QueueLimit uint64 `json:"queueLimit"`
}

// Validate NSQArgs fields
func (n NSQArgs) Validate() error {
	if !n.Enable {
		return nil
	}

	if n.NSQDAddress.IsEmpty() {
		return errors.New("empty nsqdAddress")
	}

	if n.Topic == "" {
		return errors.New("empty topic")
	}
	if n.QueueDir != "" {
		if !filepath.IsAbs(n.QueueDir) {
			return errors.New("queueDir path should be absolute")
		}
	}

	return nil
}

// NSQTarget - NSQ target.
type NSQTarget struct {
	id         event.TargetID
	args       NSQArgs
	producer   *nsq.Producer
	store      Store
	config     *nsq.Config
	loggerOnce func(ctx context.Context, err error, id interface{}, errKind ...interface{})
}

// ID - returns target ID.
func (target *NSQTarget) ID() event.TargetID {
	return target.id
}

// HasQueueStore - Checks if the queueStore has been configured for the target
func (target *NSQTarget) HasQueueStore() bool {
	return target.store != nil
}

// IsActive - Return true if target is up and active
func (target *NSQTarget) IsActive() (bool, error) {
	if target.producer == nil {
		producer, err := nsq.NewProducer(target.args.NSQDAddress.String(), target.config)
		if err != nil {
			return false, err
		}
		target.producer = producer
	}

	if err := target.producer.Ping(); err != nil {
		// To treat "connection refused" errors as errNotConnected.
		if IsConnRefusedErr(err) {
			return false, errNotConnected
		}
		return false, err
	}
	return true, nil
}

// Save - saves the events to the store which will be replayed when the nsq connection is active.
func (target *NSQTarget) Save(eventData event.Event) error {
	if target.store != nil {
		return target.store.Put(eventData)
	}
	_, err := target.IsActive()
	if err != nil {
		return err
	}
	return target.send(eventData)
}

// send - sends an event to the NSQ.
func (target *NSQTarget) send(eventData event.Event) error {
	objectName, err := url.QueryUnescape(eventData.S3.Object.Key)
	if err != nil {
		return err
	}
	key := eventData.S3.Bucket.Name + "/" + objectName

	data, err := json.Marshal(event.Log{EventName: eventData.EventName, Key: key, Records: []event.Event{eventData}})
	if err != nil {
		return err
	}

	return target.producer.Publish(target.args.Topic, data)
}

// Send - reads an event from store and sends it to NSQ.
func (target *NSQTarget) Send(eventKey string) error {
	_, err := target.IsActive()
	if err != nil {
		return err
	}

	eventData, eErr := target.store.Get(eventKey)
	if eErr != nil {
		// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
		// Such events will not exist and wouldve been already been sent successfully.
		if os.IsNotExist(eErr) {
			return nil
		}
		return eErr
	}

	if err := target.send(eventData); err != nil {
		return err
	}

	// Delete the event from store.
	return target.store.Del(eventKey)
}

// Close - closes underneath connections to NSQD server.
func (target *NSQTarget) Close() (err error) {
	if target.producer != nil {
		// this blocks until complete:
		target.producer.Stop()
	}
	return nil
}

// NewNSQTarget - creates new NSQ target.
func NewNSQTarget(id string, args NSQArgs, doneCh <-chan struct{}, loggerOnce func(ctx context.Context, err error, id interface{}, kind ...interface{}), test bool) (*NSQTarget, error) {
	config := nsq.NewConfig()
	if args.TLS.Enable {
		config.TlsV1 = true
		config.TlsConfig = &tls.Config{
			InsecureSkipVerify: args.TLS.SkipVerify,
		}
	}

	var store Store

	target := &NSQTarget{
		id:         event.TargetID{ID: id, Name: "nsq"},
		args:       args,
		config:     config,
		loggerOnce: loggerOnce,
	}

	if args.QueueDir != "" {
		queueDir := filepath.Join(args.QueueDir, storePrefix+"-nsq-"+id)
		store = NewQueueStore(queueDir, args.QueueLimit)
		if oErr := store.Open(); oErr != nil {
			target.loggerOnce(context.Background(), oErr, target.ID())
			return target, oErr
		}
		target.store = store
	}

	producer, err := nsq.NewProducer(args.NSQDAddress.String(), config)
	if err != nil {
		target.loggerOnce(context.Background(), err, target.ID())
		return target, err
	}
	target.producer = producer

	if err := target.producer.Ping(); err != nil {
		// To treat "connection refused" errors as errNotConnected.
		if target.store == nil || !(IsConnRefusedErr(err) || IsConnResetErr(err)) {
			target.loggerOnce(context.Background(), err, target.ID())
			return target, err
		}
	}

	if target.store != nil && !test {
		// Replays the events from the store.
		eventKeyCh := replayEvents(target.store, doneCh, target.loggerOnce, target.ID())
		// Start replaying events from the store.
		go sendEvents(target, eventKeyCh, doneCh, target.loggerOnce)
	}

	return target, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The HAProxy Ingress Controller Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package haproxy

import (
	"fmt"
	"os/exec"
	"reflect"
	"regexp"
	"sort"
	"strconv"
	"strings"

	"github.com/jcmoraisjr/haproxy-ingress/pkg/acme"
	"github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/template"
	hatypes "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/types"
	hautils "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/utils"
	"github.com/jcmoraisjr/haproxy-ingress/pkg/types"
	"github.com/jcmoraisjr/haproxy-ingress/pkg/utils"
)

// InstanceOptions ...
type InstanceOptions struct {
	AcmeSigner        acme.Signer
	AcmeQueue         utils.Queue
	LeaderElector     types.LeaderElector
	MaxOldConfigFiles int
	HAProxyCmd        string
	HAProxyConfigFile string
	Metrics           types.Metrics
	ReloadCmd         string
	ReloadStrategy    string
	ValidateConfig    bool
}

// Instance ...
type Instance interface {
	AcmePeriodicCheck()
	ParseTemplates() error
	Config() Config
	CalcIdleMetric()
	Update(timer *utils.Timer)
}

// CreateInstance ...
func CreateInstance(logger types.Logger, options InstanceOptions) Instance {
	return &instance{
		logger:       logger,
		options:      &options,
		templates:    template.CreateConfig(),
		mapsTemplate: template.CreateConfig(),
		mapsDir:      "/etc/haproxy/maps",
		metrics:      options.Metrics,
	}
}

type instance struct {
	logger       types.Logger
	options      *InstanceOptions
	templates    *template.Config
	mapsTemplate *template.Config
	mapsDir      string
	oldConfig    Config
	curConfig    Config
	metrics      types.Metrics
}

func (i *instance) AcmePeriodicCheck() {
	if i.oldConfig == nil || i.options.AcmeQueue == nil {
		return
	}
	hasAccount := i.acmeEnsureConfig(i.oldConfig.AcmeData())
	if !hasAccount {
		return
	}
	le := i.options.LeaderElector
	if !le.IsLeader() {
		i.logger.Info("skipping acme periodic check, leader is %s", le.LeaderName())
		return
	}
	i.logger.Info("starting periodic certificate check")
	var count int
	for storage, domains := range i.oldConfig.AcmeData().Certs {
		i.acmeAddCert(storage, domains)
		count++
	}
	if count == 0 {
		i.logger.Info("certificate list is empty")
	} else {
		i.logger.Info("finish adding %d certificate(s) to the work queue", count)
	}
}

func (i *instance) acmeEnsureConfig(acmeConfig *hatypes.AcmeData) bool {
	signer := i.options.AcmeSigner
	signer.AcmeConfig(acmeConfig.Expiring)
	signer.AcmeAccount(acmeConfig.Endpoint, acmeConfig.Emails, acmeConfig.TermsAgreed)
	return signer.HasAccount()
}

func (i *instance) acmeBuildCert(storage string, domains map[string]struct{}) string {
	cert := make([]string, len(domains))
	n := 0
	for dom := range domains {
		cert[n] = dom
		n++
	}
	sort.Slice(cert, func(i, j int) bool {
		return cert[i] < cert[j]
	})
	return strings.Join(cert, ",")
}

func (i *instance) acmeAddCert(storage string, domains map[string]struct{}) {
	strcert := i.acmeBuildCert(storage, domains)
	i.logger.InfoV(2, "enqueue certificate for processing: storage=%s domain(s)=%s",
		storage, strcert)
	i.options.AcmeQueue.Add(storage + "," + strcert)
}

func (i *instance) acmeRemoveCert(storage string, domains map[string]struct{}) {
	strcert := i.acmeBuildCert(storage, domains)
	i.options.AcmeQueue.Remove(storage + "," + strcert)
}

func (i *instance) ParseTemplates() error {
	i.templates.ClearTemplates()
	i.mapsTemplate.ClearTemplates()
	if err := i.templates.NewTemplate(
		"spoe-modsecurity.tmpl",
		"/etc/haproxy/modsecurity/spoe-modsecurity.tmpl",
		"/etc/haproxy/spoe-modsecurity.conf",
		0,
		1024,
	); err != nil {
		return err
	}
	if err := i.templates.NewTemplate(
		"haproxy.tmpl",
		"/etc/haproxy/template/haproxy.tmpl",
		"/etc/haproxy/haproxy.cfg",
		i.options.MaxOldConfigFiles,
		16384,
	); err != nil {
		return err
	}
	err := i.mapsTemplate.NewTemplate(
		"map.tmpl",
		"/etc/haproxy/maptemplate/map.tmpl",
		"",
		0,
		2048,
	)
	return err
}

func (i *instance) Config() Config {
	if i.curConfig == nil {
		config := createConfig(options{
			mapsTemplate: i.mapsTemplate,
			mapsDir:      i.mapsDir,
		})
		i.curConfig = config
	}
	return i.curConfig
}

var idleRegex = regexp.MustCompile(`Idle_pct: ([0-9]+)`)

func (i *instance) CalcIdleMetric() {
	if i.oldConfig == nil {
		return
	}
	msg, err := hautils.HAProxyCommand(i.oldConfig.Global().AdminSocket, i.metrics.HAProxyShowInfoResponseTime, "show info")
	if err != nil {
		i.logger.Error("error reading admin socket: %v", err)
		return
	}
	idleStr := idleRegex.FindStringSubmatch(msg[0])
	if len(idleStr) < 2 {
		i.logger.Error("cannot find Idle_pct field in the show info socket command")
		return
	}
	idle, err := strconv.Atoi(idleStr[1])
	if err != nil {
		i.logger.Error("Idle_pct has an invalid integer: %s", idleStr[1])
	}
	i.metrics.AddIdleFactor(idle)
}

func (i *instance) Update(timer *utils.Timer) {
	i.acmeUpdate()
	i.haproxyUpdate(timer)
}

func (i *instance) acmeUpdate() {
	if i.oldConfig == nil || i.curConfig == nil || i.options.AcmeQueue == nil {
		return
	}
	le := i.options.LeaderElector
	if le.IsLeader() {
		hasAccount := i.acmeEnsureConfig(i.curConfig.AcmeData())
		if !hasAccount {
			return
		}
	}
	var updated bool
	oldCerts := i.oldConfig.AcmeData().Certs
	curCerts := i.curConfig.AcmeData().Certs
	// Remove from the retry queue certs that was removed from the config
	for storage, domains := range oldCerts {
		curdomains, found := curCerts[storage]
		if !found || !reflect.DeepEqual(domains, curdomains) {
			if le.IsLeader() {
				i.acmeRemoveCert(storage, domains)
			}
			updated = true
		}
	}
	// Add new certs to the work queue
	for storage, domains := range curCerts {
		olddomains, found := oldCerts[storage]
		if !found || !reflect.DeepEqual(domains, olddomains) {
			if le.IsLeader() {
				i.acmeAddCert(storage, domains)
			}
			updated = true
		}
	}
	if updated && !le.IsLeader() {
		i.logger.InfoV(2, "skipping acme update check, leader is %s", le.LeaderName())
	}
}

func (i *instance) haproxyUpdate(timer *utils.Timer) {
	// nil config, just ignore
	if i.curConfig == nil {
		i.logger.Info("new configuration is empty")
		return
	}
	//
	// this should be taken into account when refactoring this func:
	//   - dynUpdater might change config state, so it should be called before templates.Write()
	//   - i.metrics.IncUpdate<Status>() should be called always, but only once
	//   - i.metrics.UpdateSuccessful(<bool>) should be called always, but only once
	//
	defer i.rotateConfig()
	if err := i.curConfig.BuildFrontendGroup(); err != nil {
		i.logger.Error("error building configuration group: %v", err)
		i.metrics.IncUpdateNoop()
		i.metrics.UpdateSuccessful(false)
		return
	}
	if err := i.curConfig.BuildBackendMaps(); err != nil {
		i.logger.Error("error building backend maps: %v", err)
		i.metrics.IncUpdateNoop()
		i.metrics.UpdateSuccessful(false)
		return
	}
	if i.curConfig.Equals(i.oldConfig) {
		i.logger.InfoV(2, "old and new configurations match, skipping reload")
		i.metrics.IncUpdateNoop()
		i.metrics.UpdateSuccessful(true)
		return
	}
	updater := i.newDynUpdater()
	updated := updater.update()
	if !updated || updater.cmdCnt > 0 {
		// only need to rewrtite config files if:
		//   - !updated           - there are changes that cannot be dynamically applied
		//   - updater.cmdCnt > 0 - there are changes that was dynamically applied
		err := i.templates.Write(i.curConfig)
		timer.Tick("write_tmpl")
		if err != nil {
			i.logger.Error("error writing configuration: %v", err)
			i.metrics.IncUpdateNoop()
			i.metrics.UpdateSuccessful(false)
			return
		}
	}
	if updated {
		if updater.cmdCnt > 0 {
			var err error
			if i.options.ValidateConfig {
				if err = i.check(); err != nil {
					i.logger.Error("error validating config file:\n%v", err)
				}
				timer.Tick("validate_cfg")
			}
			i.logger.Info("HAProxy updated without needing to reload. Commands sent: %d", updater.cmdCnt)
			i.metrics.IncUpdateDynamic()
			i.metrics.UpdateSuccessful(err == nil)
		} else {
			i.logger.Info("old and new configurations match")
			i.metrics.IncUpdateNoop()
			i.metrics.UpdateSuccessful(true)
		}
		return
	}
	// A future implementation of ssl certs dynamic update should change this metric
	for _, host := range i.curConfig.Hosts() {
		if host.TLS.TLSHash != "" {
			i.metrics.SetCertExpireDate(host.Hostname, host.TLS.TLSNotAfter)
		}
	}
	i.metrics.IncUpdateFull()
	if err := i.reload(); err != nil {
		i.logger.Error("error reloading server:\n%v", err)
		i.metrics.UpdateSuccessful(false)
		return
	}
	timer.Tick("reload_haproxy")
	i.metrics.UpdateSuccessful(true)
	i.logger.Info("HAProxy successfully reloaded")
}

func (i *instance) check() error {
	if i.options.HAProxyCmd == "" {
		i.logger.Info("(test) check was skipped")
		return nil
	}
	out, err := exec.Command(i.options.HAProxyCmd, "-c", "-f", i.options.HAProxyConfigFile).CombinedOutput()
	outstr := filterOutput(out)
	if err != nil {
		return fmt.Errorf(outstr)
	}
	return nil
}

func (i *instance) reload() error {
	if i.options.ReloadCmd == "" {
		i.logger.Info("(test) reload was skipped")
		return nil
	}
	out, err := exec.Command(i.options.ReloadCmd, i.options.ReloadStrategy, i.options.HAProxyConfigFile).CombinedOutput()
	outstr := filterOutput(out)
	if len(outstr) > 0 {
		i.logger.Warn("output from haproxy:\n%v", outstr)
	}
	if err != nil {
		return err
	}
	return nil
}

func filterOutput(out []byte) string {
	// workaround a misplaced warn from haproxy until the fix is merged
	skip := "contains no embedded dots nor does not start with a dot"
	outstr := ""
	for _, line := range strings.Split(string(out), "\n") {
		if line != "" && strings.Index(line, skip) < 0 {
			outstr += line + "\n"
		}
	}
	return outstr
}

func (i *instance) rotateConfig() {
	// TODO releaseConfig (old support files, ...)
	i.oldConfig = i.curConfig
	i.curConfig = nil
}
<commit_msg>call update successful stat only if config was parsed<commit_after>/*
Copyright 2019 The HAProxy Ingress Controller Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package haproxy

import (
	"fmt"
	"os/exec"
	"reflect"
	"regexp"
	"sort"
	"strconv"
	"strings"

	"github.com/jcmoraisjr/haproxy-ingress/pkg/acme"
	"github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/template"
	hatypes "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/types"
	hautils "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/utils"
	"github.com/jcmoraisjr/haproxy-ingress/pkg/types"
	"github.com/jcmoraisjr/haproxy-ingress/pkg/utils"
)

// InstanceOptions ...
type InstanceOptions struct {
	AcmeSigner        acme.Signer
	AcmeQueue         utils.Queue
	LeaderElector     types.LeaderElector
	MaxOldConfigFiles int
	HAProxyCmd        string
	HAProxyConfigFile string
	Metrics           types.Metrics
	ReloadCmd         string
	ReloadStrategy    string
	ValidateConfig    bool
}

// Instance ...
type Instance interface {
	AcmePeriodicCheck()
	ParseTemplates() error
	Config() Config
	CalcIdleMetric()
	Update(timer *utils.Timer)
}

// CreateInstance ...
func CreateInstance(logger types.Logger, options InstanceOptions) Instance {
	return &instance{
		logger:       logger,
		options:      &options,
		templates:    template.CreateConfig(),
		mapsTemplate: template.CreateConfig(),
		mapsDir:      "/etc/haproxy/maps",
		metrics:      options.Metrics,
	}
}

type instance struct {
	logger       types.Logger
	options      *InstanceOptions
	templates    *template.Config
	mapsTemplate *template.Config
	mapsDir      string
	oldConfig    Config
	curConfig    Config
	metrics      types.Metrics
}

func (i *instance) AcmePeriodicCheck() {
	if i.oldConfig == nil || i.options.AcmeQueue == nil {
		return
	}
	hasAccount := i.acmeEnsureConfig(i.oldConfig.AcmeData())
	if !hasAccount {
		return
	}
	le := i.options.LeaderElector
	if !le.IsLeader() {
		i.logger.Info("skipping acme periodic check, leader is %s", le.LeaderName())
		return
	}
	i.logger.Info("starting periodic certificate check")
	var count int
	for storage, domains := range i.oldConfig.AcmeData().Certs {
		i.acmeAddCert(storage, domains)
		count++
	}
	if count == 0 {
		i.logger.Info("certificate list is empty")
	} else {
		i.logger.Info("finish adding %d certificate(s) to the work queue", count)
	}
}

func (i *instance) acmeEnsureConfig(acmeConfig *hatypes.AcmeData) bool {
	signer := i.options.AcmeSigner
	signer.AcmeConfig(acmeConfig.Expiring)
	signer.AcmeAccount(acmeConfig.Endpoint, acmeConfig.Emails, acmeConfig.TermsAgreed)
	return signer.HasAccount()
}

func (i *instance) acmeBuildCert(storage string, domains map[string]struct{}) string {
	cert := make([]string, len(domains))
	n := 0
	for dom := range domains {
		cert[n] = dom
		n++
	}
	sort.Slice(cert, func(i, j int) bool {
		return cert[i] < cert[j]
	})
	return strings.Join(cert, ",")
}

func (i *instance) acmeAddCert(storage string, domains map[string]struct{}) {
	strcert := i.acmeBuildCert(storage, domains)
	i.logger.InfoV(2, "enqueue certificate for processing: storage=%s domain(s)=%s",
		storage, strcert)
	i.options.AcmeQueue.Add(storage + "," + strcert)
}

func (i *instance) acmeRemoveCert(storage string, domains map[string]struct{}) {
	strcert := i.acmeBuildCert(storage, domains)
	i.options.AcmeQueue.Remove(storage + "," + strcert)
}

func (i *instance) ParseTemplates() error {
	i.templates.ClearTemplates()
	i.mapsTemplate.ClearTemplates()
	if err := i.templates.NewTemplate(
		"spoe-modsecurity.tmpl",
		"/etc/haproxy/modsecurity/spoe-modsecurity.tmpl",
		"/etc/haproxy/spoe-modsecurity.conf",
		0,
		1024,
	); err != nil {
		return err
	}
	if err := i.templates.NewTemplate(
		"haproxy.tmpl",
		"/etc/haproxy/template/haproxy.tmpl",
		"/etc/haproxy/haproxy.cfg",
		i.options.MaxOldConfigFiles,
		16384,
	); err != nil {
		return err
	}
	err := i.mapsTemplate.NewTemplate(
		"map.tmpl",
		"/etc/haproxy/maptemplate/map.tmpl",
		"",
		0,
		2048,
	)
	return err
}

func (i *instance) Config() Config {
	if i.curConfig == nil {
		config := createConfig(options{
			mapsTemplate: i.mapsTemplate,
			mapsDir:      i.mapsDir,
		})
		i.curConfig = config
	}
	return i.curConfig
}

var idleRegex = regexp.MustCompile(`Idle_pct: ([0-9]+)`)

func (i *instance) CalcIdleMetric() {
	if i.oldConfig == nil {
		return
	}
	msg, err := hautils.HAProxyCommand(i.oldConfig.Global().AdminSocket, i.metrics.HAProxyShowInfoResponseTime, "show info")
	if err != nil {
		i.logger.Error("error reading admin socket: %v", err)
		return
	}
	idleStr := idleRegex.FindStringSubmatch(msg[0])
	if len(idleStr) < 2 {
		i.logger.Error("cannot find Idle_pct field in the show info socket command")
		return
	}
	idle, err := strconv.Atoi(idleStr[1])
	if err != nil {
		i.logger.Error("Idle_pct has an invalid integer: %s", idleStr[1])
	}
	i.metrics.AddIdleFactor(idle)
}

func (i *instance) Update(timer *utils.Timer) {
	i.acmeUpdate()
	i.haproxyUpdate(timer)
}

func (i *instance) acmeUpdate() {
	if i.oldConfig == nil || i.curConfig == nil || i.options.AcmeQueue == nil {
		return
	}
	le := i.options.LeaderElector
	if le.IsLeader() {
		hasAccount := i.acmeEnsureConfig(i.curConfig.AcmeData())
		if !hasAccount {
			return
		}
	}
	var updated bool
	oldCerts := i.oldConfig.AcmeData().Certs
	curCerts := i.curConfig.AcmeData().Certs
	// Remove from the retry queue certs that was removed from the config
	for storage, domains := range oldCerts {
		curdomains, found := curCerts[storage]
		if !found || !reflect.DeepEqual(domains, curdomains) {
			if le.IsLeader() {
				i.acmeRemoveCert(storage, domains)
			}
			updated = true
		}
	}
	// Add new certs to the work queue
	for storage, domains := range curCerts {
		olddomains, found := oldCerts[storage]
		if !found || !reflect.DeepEqual(domains, olddomains) {
			if le.IsLeader() {
				i.acmeAddCert(storage, domains)
			}
			updated = true
		}
	}
	if updated && !le.IsLeader() {
		i.logger.InfoV(2, "skipping acme update check, leader is %s", le.LeaderName())
	}
}

func (i *instance) haproxyUpdate(timer *utils.Timer) {
	// nil config, just ignore
	if i.curConfig == nil {
		i.logger.Info("new configuration is empty")
		return
	}
	//
	// this should be taken into account when refactoring this func:
	//   - dynUpdater might change config state, so it should be called before templates.Write()
	//   - i.metrics.IncUpdate<Status>() should be called always, but only once
	//   - i.metrics.UpdateSuccessful(<bool>) should be called only if haproxy is reloaded or cfg is validated
	//
	defer i.rotateConfig()
	if err := i.curConfig.BuildFrontendGroup(); err != nil {
		i.logger.Error("error building configuration group: %v", err)
		i.metrics.IncUpdateNoop()
		return
	}
	if err := i.curConfig.BuildBackendMaps(); err != nil {
		i.logger.Error("error building backend maps: %v", err)
		i.metrics.IncUpdateNoop()
		return
	}
	if i.curConfig.Equals(i.oldConfig) {
		i.logger.InfoV(2, "old and new configurations match, skipping reload")
		i.metrics.IncUpdateNoop()
		return
	}
	updater := i.newDynUpdater()
	updated := updater.update()
	if !updated || updater.cmdCnt > 0 {
		// only need to rewrtite config files if:
		//   - !updated           - there are changes that cannot be dynamically applied
		//   - updater.cmdCnt > 0 - there are changes that was dynamically applied
		err := i.templates.Write(i.curConfig)
		timer.Tick("write_tmpl")
		if err != nil {
			i.logger.Error("error writing configuration: %v", err)
			i.metrics.IncUpdateNoop()
			return
		}
	}
	if updated {
		if updater.cmdCnt > 0 {
			if i.options.ValidateConfig {
				var err error
				if err = i.check(); err != nil {
					i.logger.Error("error validating config file:\n%v", err)
				}
				timer.Tick("validate_cfg")
				i.metrics.UpdateSuccessful(err == nil)
			}
			i.logger.Info("HAProxy updated without needing to reload. Commands sent: %d", updater.cmdCnt)
			i.metrics.IncUpdateDynamic()
		} else {
			i.logger.Info("old and new configurations match")
			i.metrics.IncUpdateNoop()
		}
		return
	}
	// A future implementation of ssl certs dynamic update should change this metric
	for _, host := range i.curConfig.Hosts() {
		if host.TLS.TLSHash != "" {
			i.metrics.SetCertExpireDate(host.Hostname, host.TLS.TLSNotAfter)
		}
	}
	i.metrics.IncUpdateFull()
	if err := i.reload(); err != nil {
		i.logger.Error("error reloading server:\n%v", err)
		i.metrics.UpdateSuccessful(false)
		return
	}
	timer.Tick("reload_haproxy")
	i.metrics.UpdateSuccessful(true)
	i.logger.Info("HAProxy successfully reloaded")
}

func (i *instance) check() error {
	if i.options.HAProxyCmd == "" {
		i.logger.Info("(test) check was skipped")
		return nil
	}
	out, err := exec.Command(i.options.HAProxyCmd, "-c", "-f", i.options.HAProxyConfigFile).CombinedOutput()
	outstr := filterOutput(out)
	if err != nil {
		return fmt.Errorf(outstr)
	}
	return nil
}

func (i *instance) reload() error {
	if i.options.ReloadCmd == "" {
		i.logger.Info("(test) reload was skipped")
		return nil
	}
	out, err := exec.Command(i.options.ReloadCmd, i.options.ReloadStrategy, i.options.HAProxyConfigFile).CombinedOutput()
	outstr := filterOutput(out)
	if len(outstr) > 0 {
		i.logger.Warn("output from haproxy:\n%v", outstr)
	}
	if err != nil {
		return err
	}
	return nil
}

func filterOutput(out []byte) string {
	// workaround a misplaced warn from haproxy until the fix is merged
	skip := "contains no embedded dots nor does not start with a dot"
	outstr := ""
	for _, line := range strings.Split(string(out), "\n") {
		if line != "" && strings.Index(line, skip) < 0 {
			outstr += line + "\n"
		}
	}
	return outstr
}

func (i *instance) rotateConfig() {
	// TODO releaseConfig (old support files, ...)
	i.oldConfig = i.curConfig
	i.curConfig = nil
}
<|endoftext|>


<commit_before>package instance

import (
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/cozy/cozy-stack/pkg/utils"
)

var managerClient = &http.Client{Timeout: 30 * time.Second}

// ManagerURLKind is an enum type for the different kinds of manager URLs.
type ManagerURLKind int

const (
	// ManagerTOSURL is the kind for changes of TOS URL.
	ManagerTOSURL ManagerURLKind = iota
	// ManagerPremiumURL is the kind for changing the account type of the
	// instance.
	ManagerPremiumURL
)

// ManagerURL returns an external string for the given ManagerURL kind.
func (i *Instance) ManagerURL(k ManagerURLKind) (s string, ok bool) {
	defer func() {
		if !ok {
			s = i.PageURL("/manager_url_is_not_specified", nil)
		}
	}()

	if i.UUID == "" {
		return
	}

	if i.managerURL == nil {
		ctx, err := i.SettingsContext()
		if err != nil {
			return
		}
		var u string
		u, ok = ctx["manager_url"].(string)
		if !ok {
			return
		}
		i.managerURL, err = url.Parse(u)
		if err != nil {
			return
		}
	}

	managerURL := utils.CloneURL(i.managerURL)
	var path string
	switch k {
	// TODO: we may want to rely on the contexts to avoid hardcoding the path
	// values of these kinds.
	case ManagerPremiumURL:
		path = fmt.Sprintf("/cozy/accounts/%s", url.PathEscape(i.UUID))
	case ManagerTOSURL:
		path = fmt.Sprintf("/cozy/instances/%s/tos", url.PathEscape(i.UUID))
	default:
		panic("unknown ManagerURLKind")
	}
	managerURL.Path = path

	s = managerURL.String()
	ok = true
	return
}

// ManagerSignTOS make a request to the manager in order to finalize the TOS
// signing flow.
func (i *Instance) ManagerSignTOS(originalReq *http.Request) error {
	if i.TOSLatest == "" {
		return nil
	}
	split := strings.SplitN(i.TOSLatest, "-", 2)
	if len(split) != 2 {
		return nil
	}
	u, ok := i.ManagerURL(ManagerTOSURL)
	if !ok {
		return Patch(i, &Options{TOSSigned: i.TOSLatest})
	}
	form := url.Values{"version": {split[0]}}
	res, err := doManagerRequest(http.MethodPut, u, form, originalReq)
	if err != nil {
		return err
	}
	return res.Body.Close()
}

func doManagerRequest(method string, url string, form url.Values, originalReq *http.Request) (*http.Response, error) {
	var body io.Reader
	if form != nil {
		body = strings.NewReader(form.Encode())
	}
	req, err := http.NewRequest(method, url, body)
	if err != nil {
		return nil, err
	}
	if form != nil {
		req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	}
	if originalReq != nil {
		var ip string
		if forwardedFor := req.Header.Get("X-Forwarded-For"); forwardedFor != "" {
			ip = strings.TrimSpace(strings.SplitN(forwardedFor, ",", 2)[0])
		}
		if ip == "" {
			ip = req.RemoteAddr
		}
		req.Header.Set("X-Forwarded-For", ip)
	}
	return managerClient.Do(req)
}
<commit_msg>Bad premium URL<commit_after>package instance

import (
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/cozy/cozy-stack/pkg/utils"
)

var managerClient = &http.Client{Timeout: 30 * time.Second}

// ManagerURLKind is an enum type for the different kinds of manager URLs.
type ManagerURLKind int

const (
	// ManagerTOSURL is the kind for changes of TOS URL.
	ManagerTOSURL ManagerURLKind = iota
	// ManagerPremiumURL is the kind for changing the account type of the
	// instance.
	ManagerPremiumURL
)

// ManagerURL returns an external string for the given ManagerURL kind.
func (i *Instance) ManagerURL(k ManagerURLKind) (s string, ok bool) {
	defer func() {
		if !ok {
			s = i.PageURL("/manager_url_is_not_specified", nil)
		}
	}()

	if i.UUID == "" {
		return
	}

	if i.managerURL == nil {
		ctx, err := i.SettingsContext()
		if err != nil {
			return
		}
		var u string
		u, ok = ctx["manager_url"].(string)
		if !ok {
			return
		}
		i.managerURL, err = url.Parse(u)
		if err != nil {
			return
		}
	}

	managerURL := utils.CloneURL(i.managerURL)
	var path string
	switch k {
	// TODO: we may want to rely on the contexts to avoid hardcoding the path
	// values of these kinds.
	case ManagerPremiumURL:
		path = fmt.Sprintf("/cozy/accounts/%s/premium", url.PathEscape(i.UUID))
	case ManagerTOSURL:
		path = fmt.Sprintf("/cozy/instances/%s/tos", url.PathEscape(i.UUID))
	default:
		panic("unknown ManagerURLKind")
	}
	managerURL.Path = path

	s = managerURL.String()
	ok = true
	return
}

// ManagerSignTOS make a request to the manager in order to finalize the TOS
// signing flow.
func (i *Instance) ManagerSignTOS(originalReq *http.Request) error {
	if i.TOSLatest == "" {
		return nil
	}
	split := strings.SplitN(i.TOSLatest, "-", 2)
	if len(split) != 2 {
		return nil
	}
	u, ok := i.ManagerURL(ManagerTOSURL)
	if !ok {
		return Patch(i, &Options{TOSSigned: i.TOSLatest})
	}
	form := url.Values{"version": {split[0]}}
	res, err := doManagerRequest(http.MethodPut, u, form, originalReq)
	if err != nil {
		return err
	}
	return res.Body.Close()
}

func doManagerRequest(method string, url string, form url.Values, originalReq *http.Request) (*http.Response, error) {
	var body io.Reader
	if form != nil {
		body = strings.NewReader(form.Encode())
	}
	req, err := http.NewRequest(method, url, body)
	if err != nil {
		return nil, err
	}
	if form != nil {
		req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	}
	if originalReq != nil {
		var ip string
		if forwardedFor := req.Header.Get("X-Forwarded-For"); forwardedFor != "" {
			ip = strings.TrimSpace(strings.SplitN(forwardedFor, ",", 2)[0])
		}
		if ip == "" {
			ip = req.RemoteAddr
		}
		req.Header.Set("X-Forwarded-For", ip)
	}
	return managerClient.Do(req)
}
<|endoftext|>


<commit_before>package loghttp

import (
	"strings"
)

// Version holds a loghttp version
type Version int

// Valid Version values
const (
	VersionLegacy = Version(iota)
	VersionV1
)

// GetVersion returns the loghttp version for a given path.
func GetVersion(uri string) Version {
	if strings.HasPrefix(strings.ToLower(uri), "/loki/api/v1") {
		return VersionV1
	}

	return VersionLegacy
}
<commit_msg>Fixes path prefix in the querier. (#2178)<commit_after>package loghttp

import (
	"strings"
)

// Version holds a loghttp version
type Version int

// Valid Version values
const (
	VersionLegacy = Version(iota)
	VersionV1
)

// GetVersion returns the loghttp version for a given path.
func GetVersion(uri string) Version {
	if strings.Contains(strings.ToLower(uri), "/loki/api/v1") {
		return VersionV1
	}

	return VersionLegacy
}
<|endoftext|>


<commit_before>package updatecmd

import (
	"errors"
	"fmt"
	"github.com/skatteetaten/aoc/pkg/versionutil"
	"io/ioutil"
	"net/http"
	"os"
)

const aoc5Url = "http://aoc-update-service-paas-aoc-update.utv.paas.skead.no"

func UpdateSelf(args []string, simulate bool, forceVersion string, forceUpdate bool) (output string, err error) {
	var releaseVersion string

	if forceVersion == "" {
		releaseVersion, err = getReleaseVersion()
		if err != nil {
			return "", errors.New("Update server unreachable: " + err.Error())
		}
	} else {
		releaseVersion = forceVersion
	}

	myVersion, _ := getMyVersion()

	if myVersion != releaseVersion || forceUpdate {
		output += "New version detected: Current version: " + myVersion + ".  Available version: " + releaseVersion
		if !simulate {
			err = doUpdate(releaseVersion)
			if err != nil {
				return
			}
			output += "\nAOC updated sucessfully"
		}
	} else {
		output += "No update available"
	}

	return
}

func doUpdate(version string) (err error) {
	releaseFilename := "aoc_" + version
	releaseUrl := aoc5Url + "/" + releaseFilename

	executablePath, err := os.Executable()
	if err != nil {
		return
	}

	releasePath := executablePath + "_" + version
	fmt.Println("DEBUG: Executable path: " + executablePath)
	fmt.Println("DEBUG: Release path: " + releasePath)
	body, err := getFile(releaseUrl)
	err = ioutil.WriteFile(releasePath, []byte(body), 0750)
	if err != nil {
		return
	}
	err = os.Rename(releasePath, executablePath)
	if err != nil {
		return
	}
	return
}

func getReleaseVersion() (version string, err error) {
	releaseinfoUrl := aoc5Url + "/releaseinfo.json"
	releaseinfo, err := getFile(releaseinfoUrl)
	if err != nil {
		return
	}
	releaseVersionStruct, err := versionutil.Json2Version(releaseinfo)

	version = releaseVersionStruct.Version
	return
}

func getMyVersion() (version string, err error) {
	var versionStruct versionutil.VersionStruct
	versionStruct.Init()

	version = versionStruct.Version
	return
}

func getFile(url string) (file []byte, err error) {
	req, err := http.NewRequest(http.MethodGet, url, nil)
	if err != nil {
		return
	}
	req.Header.Set("Content-Type", "application/octet-stream")

	client := &http.Client{}

	resp, err := client.Do(req)
	if err != nil {

		err = errors.New(fmt.Sprintf("Error downloading update from %v: %v", aoc5Url, err))
		return
	}

	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		err = errors.New(resp.Status)
		return
	}

	file, err = ioutil.ReadAll(resp.Body)
	return
}
<commit_msg>Removed DEBUG statements<commit_after>package updatecmd

import (
	"errors"
	"fmt"
	"github.com/skatteetaten/aoc/pkg/versionutil"
	"io/ioutil"
	"net/http"
	"os"
)

const aoc5Url = "http://aoc-update-service-paas-aoc-update.utv.paas.skead.no"

func UpdateSelf(args []string, simulate bool, forceVersion string, forceUpdate bool) (output string, err error) {
	var releaseVersion string

	if forceVersion == "" {
		releaseVersion, err = getReleaseVersion()
		if err != nil {
			return "", errors.New("Update server unreachable: " + err.Error())
		}
	} else {
		releaseVersion = forceVersion
	}

	myVersion, _ := getMyVersion()

	if myVersion != releaseVersion || forceUpdate {
		output += "New version detected: Current version: " + myVersion + ".  Available version: " + releaseVersion
		if !simulate {
			err = doUpdate(releaseVersion)
			if err != nil {
				return
			}
			output += "\nAOC updated sucessfully"
		}
	} else {
		output += "No update available"
	}

	return
}

func doUpdate(version string) (err error) {
	releaseFilename := "aoc_" + version
	releaseUrl := aoc5Url + "/" + releaseFilename

	executablePath, err := os.Executable()
	if err != nil {
		return
	}

	releasePath := executablePath + "_" + version
	body, err := getFile(releaseUrl)
	err = ioutil.WriteFile(releasePath, []byte(body), 0750)
	if err != nil {
		return
	}
	err = os.Rename(releasePath, executablePath)
	if err != nil {
		return
	}
	return
}

func getReleaseVersion() (version string, err error) {
	releaseinfoUrl := aoc5Url + "/releaseinfo.json"
	releaseinfo, err := getFile(releaseinfoUrl)
	if err != nil {
		return
	}
	releaseVersionStruct, err := versionutil.Json2Version(releaseinfo)

	version = releaseVersionStruct.Version
	return
}

func getMyVersion() (version string, err error) {
	var versionStruct versionutil.VersionStruct
	versionStruct.Init()

	version = versionStruct.Version
	return
}

func getFile(url string) (file []byte, err error) {
	req, err := http.NewRequest(http.MethodGet, url, nil)
	if err != nil {
		return
	}
	req.Header.Set("Content-Type", "application/octet-stream")

	client := &http.Client{}

	resp, err := client.Do(req)
	if err != nil {

		err = errors.New(fmt.Sprintf("Error downloading update from %v: %v", aoc5Url, err))
		return
	}

	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		err = errors.New(resp.Status)
		return
	}

	file, err = ioutil.ReadAll(resp.Body)
	return
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"

	"strings"

	"github.com/golang/glog"
	"k8s.io/api/core/v1"
	storage "k8s.io/api/storage/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/sets"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/kubernetes/pkg/api/legacyscheme"
	v1helper "k8s.io/kubernetes/pkg/api/v1/helper"
	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
	"k8s.io/kubernetes/pkg/util/mount"
)

const readyFileName = "ready"

// IsReady checks for the existence of a regular file
// called 'ready' in the given directory and returns
// true if that file exists.
func IsReady(dir string) bool {
	readyFile := path.Join(dir, readyFileName)
	s, err := os.Stat(readyFile)
	if err != nil {
		return false
	}

	if !s.Mode().IsRegular() {
		glog.Errorf("ready-file is not a file: %s", readyFile)
		return false
	}

	return true
}

// SetReady creates a file called 'ready' in the given
// directory.  It logs an error if the file cannot be
// created.
func SetReady(dir string) {
	if err := os.MkdirAll(dir, 0750); err != nil && !os.IsExist(err) {
		glog.Errorf("Can't mkdir %s: %v", dir, err)
		return
	}

	readyFile := path.Join(dir, readyFileName)
	file, err := os.Create(readyFile)
	if err != nil {
		glog.Errorf("Can't touch %s: %v", readyFile, err)
		return
	}
	file.Close()
}

// UnmountPath is a common unmount routine that unmounts the given path and
// deletes the remaining directory if successful.
func UnmountPath(mountPath string, mounter mount.Interface) error {
	return UnmountMountPoint(mountPath, mounter, false /* extensiveMountPointCheck */)
}

// UnmountMountPoint is a common unmount routine that unmounts the given path and
// deletes the remaining directory if successful.
// if extensiveMountPointCheck is true
// IsNotMountPoint will be called instead of IsLikelyNotMountPoint.
// IsNotMountPoint is more expensive but properly handles bind mounts.
func UnmountMountPoint(mountPath string, mounter mount.Interface, extensiveMountPointCheck bool) error {
	if pathExists, pathErr := PathExists(mountPath); pathErr != nil {
		return fmt.Errorf("Error checking if path exists: %v", pathErr)
	} else if !pathExists {
		glog.Warningf("Warning: Unmount skipped because path does not exist: %v", mountPath)
		return nil
	}

	var notMnt bool
	var err error

	if extensiveMountPointCheck {
		notMnt, err = mount.IsNotMountPoint(mounter, mountPath)
	} else {
		notMnt, err = mounter.IsLikelyNotMountPoint(mountPath)
	}

	if err != nil {
		return err
	}

	if notMnt {
		glog.Warningf("Warning: %q is not a mountpoint, deleting", mountPath)
		return os.Remove(mountPath)
	}

	// Unmount the mount path
	glog.V(4).Infof("%q is a mountpoint, unmounting", mountPath)
	if err := mounter.Unmount(mountPath); err != nil {
		return err
	}
	notMnt, mntErr := mounter.IsLikelyNotMountPoint(mountPath)
	if mntErr != nil {
		return err
	}
	if notMnt {
		glog.V(4).Infof("%q is unmounted, deleting the directory", mountPath)
		return os.Remove(mountPath)
	}
	return fmt.Errorf("Failed to unmount path %v", mountPath)
}

// PathExists returns true if the specified path exists.
func PathExists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	} else if os.IsNotExist(err) {
		return false, nil
	} else {
		return false, err
	}
}

// GetSecretForPod locates secret by name in the pod's namespace and returns secret map
func GetSecretForPod(pod *v1.Pod, secretName string, kubeClient clientset.Interface) (map[string]string, error) {
	secret := make(map[string]string)
	if kubeClient == nil {
		return secret, fmt.Errorf("Cannot get kube client")
	}
	secrets, err := kubeClient.CoreV1().Secrets(pod.Namespace).Get(secretName, metav1.GetOptions{})
	if err != nil {
		return secret, err
	}
	for name, data := range secrets.Data {
		secret[name] = string(data)
	}
	return secret, nil
}

// GetSecretForPV locates secret by name and namespace, verifies the secret type, and returns secret map
func GetSecretForPV(secretNamespace, secretName, volumePluginName string, kubeClient clientset.Interface) (map[string]string, error) {
	secret := make(map[string]string)
	if kubeClient == nil {
		return secret, fmt.Errorf("Cannot get kube client")
	}
	secrets, err := kubeClient.CoreV1().Secrets(secretNamespace).Get(secretName, metav1.GetOptions{})
	if err != nil {
		return secret, err
	}
	if secrets.Type != v1.SecretType(volumePluginName) {
		return secret, fmt.Errorf("Cannot get secret of type %s", volumePluginName)
	}
	for name, data := range secrets.Data {
		secret[name] = string(data)
	}
	return secret, nil
}

func GetClassForVolume(kubeClient clientset.Interface, pv *v1.PersistentVolume) (*storage.StorageClass, error) {
	if kubeClient == nil {
		return nil, fmt.Errorf("Cannot get kube client")
	}
	className := v1helper.GetPersistentVolumeClass(pv)
	if className == "" {
		return nil, fmt.Errorf("Volume has no storage class")
	}

	class, err := kubeClient.StorageV1().StorageClasses().Get(className, metav1.GetOptions{})
	if err != nil {
		return nil, err
	}
	return class, nil
}

// CheckNodeAffinity looks at the PV node affinity, and checks if the node has the same corresponding labels
// This ensures that we don't mount a volume that doesn't belong to this node
func CheckNodeAffinity(pv *v1.PersistentVolume, nodeLabels map[string]string) error {
	affinity, err := v1helper.GetStorageNodeAffinityFromAnnotation(pv.Annotations)
	if err != nil {
		return fmt.Errorf("Error getting storage node affinity: %v", err)
	}
	if affinity == nil {
		return nil
	}

	if affinity.RequiredDuringSchedulingIgnoredDuringExecution != nil {
		terms := affinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms
		glog.V(10).Infof("Match for RequiredDuringSchedulingIgnoredDuringExecution node selector terms %+v", terms)
		for _, term := range terms {
			selector, err := v1helper.NodeSelectorRequirementsAsSelector(term.MatchExpressions)
			if err != nil {
				return fmt.Errorf("Failed to parse MatchExpressions: %v", err)
			}
			if !selector.Matches(labels.Set(nodeLabels)) {
				return fmt.Errorf("NodeSelectorTerm %+v does not match node labels", term.MatchExpressions)
			}
		}
	}
	return nil
}

// LoadPodFromFile will read, decode, and return a Pod from a file.
func LoadPodFromFile(filePath string) (*v1.Pod, error) {
	if filePath == "" {
		return nil, fmt.Errorf("file path not specified")
	}
	podDef, err := ioutil.ReadFile(filePath)
	if err != nil {
		return nil, fmt.Errorf("failed to read file path %s: %+v", filePath, err)
	}
	if len(podDef) == 0 {
		return nil, fmt.Errorf("file was empty: %s", filePath)
	}
	pod := &v1.Pod{}

	codec := legacyscheme.Codecs.LegacyCodec(legacyscheme.Registry.GroupOrDie(v1.GroupName).GroupVersion)
	if err := runtime.DecodeInto(codec, podDef, pod); err != nil {
		return nil, fmt.Errorf("failed decoding file: %v", err)
	}
	return pod, nil
}

func ZonesSetToLabelValue(strSet sets.String) string {
	return strings.Join(strSet.UnsortedList(), kubeletapis.LabelMultiZoneDelimiter)
}

// ZonesToSet converts a string containing a comma separated list of zones to set
func ZonesToSet(zonesString string) (sets.String, error) {
	return stringToSet(zonesString, ",")
}

// LabelZonesToSet converts a PV label value from string containing a delimited list of zones to set
func LabelZonesToSet(labelZonesValue string) (sets.String, error) {
	return stringToSet(labelZonesValue, kubeletapis.LabelMultiZoneDelimiter)
}

// StringToSet converts a string containing list separated by specified delimiter to to a set
func stringToSet(str, delimiter string) (sets.String, error) {
	zonesSlice := strings.Split(str, delimiter)
	zonesSet := make(sets.String)
	for _, zone := range zonesSlice {
		trimmedZone := strings.TrimSpace(zone)
		if trimmedZone == "" {
			return make(sets.String), fmt.Errorf(
				"%q separated list (%q) must not contain an empty string",
				delimiter,
				str)
		}
		zonesSet.Insert(trimmedZone)
	}
	return zonesSet, nil
}
<commit_msg>pkg/apis/core: mechanical import fixes in dependencies<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"

	"strings"

	"github.com/golang/glog"
	"k8s.io/api/core/v1"
	storage "k8s.io/api/storage/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/sets"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/kubernetes/pkg/api/legacyscheme"
	v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
	"k8s.io/kubernetes/pkg/util/mount"
)

const readyFileName = "ready"

// IsReady checks for the existence of a regular file
// called 'ready' in the given directory and returns
// true if that file exists.
func IsReady(dir string) bool {
	readyFile := path.Join(dir, readyFileName)
	s, err := os.Stat(readyFile)
	if err != nil {
		return false
	}

	if !s.Mode().IsRegular() {
		glog.Errorf("ready-file is not a file: %s", readyFile)
		return false
	}

	return true
}

// SetReady creates a file called 'ready' in the given
// directory.  It logs an error if the file cannot be
// created.
func SetReady(dir string) {
	if err := os.MkdirAll(dir, 0750); err != nil && !os.IsExist(err) {
		glog.Errorf("Can't mkdir %s: %v", dir, err)
		return
	}

	readyFile := path.Join(dir, readyFileName)
	file, err := os.Create(readyFile)
	if err != nil {
		glog.Errorf("Can't touch %s: %v", readyFile, err)
		return
	}
	file.Close()
}

// UnmountPath is a common unmount routine that unmounts the given path and
// deletes the remaining directory if successful.
func UnmountPath(mountPath string, mounter mount.Interface) error {
	return UnmountMountPoint(mountPath, mounter, false /* extensiveMountPointCheck */)
}

// UnmountMountPoint is a common unmount routine that unmounts the given path and
// deletes the remaining directory if successful.
// if extensiveMountPointCheck is true
// IsNotMountPoint will be called instead of IsLikelyNotMountPoint.
// IsNotMountPoint is more expensive but properly handles bind mounts.
func UnmountMountPoint(mountPath string, mounter mount.Interface, extensiveMountPointCheck bool) error {
	if pathExists, pathErr := PathExists(mountPath); pathErr != nil {
		return fmt.Errorf("Error checking if path exists: %v", pathErr)
	} else if !pathExists {
		glog.Warningf("Warning: Unmount skipped because path does not exist: %v", mountPath)
		return nil
	}

	var notMnt bool
	var err error

	if extensiveMountPointCheck {
		notMnt, err = mount.IsNotMountPoint(mounter, mountPath)
	} else {
		notMnt, err = mounter.IsLikelyNotMountPoint(mountPath)
	}

	if err != nil {
		return err
	}

	if notMnt {
		glog.Warningf("Warning: %q is not a mountpoint, deleting", mountPath)
		return os.Remove(mountPath)
	}

	// Unmount the mount path
	glog.V(4).Infof("%q is a mountpoint, unmounting", mountPath)
	if err := mounter.Unmount(mountPath); err != nil {
		return err
	}
	notMnt, mntErr := mounter.IsLikelyNotMountPoint(mountPath)
	if mntErr != nil {
		return err
	}
	if notMnt {
		glog.V(4).Infof("%q is unmounted, deleting the directory", mountPath)
		return os.Remove(mountPath)
	}
	return fmt.Errorf("Failed to unmount path %v", mountPath)
}

// PathExists returns true if the specified path exists.
func PathExists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	} else if os.IsNotExist(err) {
		return false, nil
	} else {
		return false, err
	}
}

// GetSecretForPod locates secret by name in the pod's namespace and returns secret map
func GetSecretForPod(pod *v1.Pod, secretName string, kubeClient clientset.Interface) (map[string]string, error) {
	secret := make(map[string]string)
	if kubeClient == nil {
		return secret, fmt.Errorf("Cannot get kube client")
	}
	secrets, err := kubeClient.CoreV1().Secrets(pod.Namespace).Get(secretName, metav1.GetOptions{})
	if err != nil {
		return secret, err
	}
	for name, data := range secrets.Data {
		secret[name] = string(data)
	}
	return secret, nil
}

// GetSecretForPV locates secret by name and namespace, verifies the secret type, and returns secret map
func GetSecretForPV(secretNamespace, secretName, volumePluginName string, kubeClient clientset.Interface) (map[string]string, error) {
	secret := make(map[string]string)
	if kubeClient == nil {
		return secret, fmt.Errorf("Cannot get kube client")
	}
	secrets, err := kubeClient.CoreV1().Secrets(secretNamespace).Get(secretName, metav1.GetOptions{})
	if err != nil {
		return secret, err
	}
	if secrets.Type != v1.SecretType(volumePluginName) {
		return secret, fmt.Errorf("Cannot get secret of type %s", volumePluginName)
	}
	for name, data := range secrets.Data {
		secret[name] = string(data)
	}
	return secret, nil
}

func GetClassForVolume(kubeClient clientset.Interface, pv *v1.PersistentVolume) (*storage.StorageClass, error) {
	if kubeClient == nil {
		return nil, fmt.Errorf("Cannot get kube client")
	}
	className := v1helper.GetPersistentVolumeClass(pv)
	if className == "" {
		return nil, fmt.Errorf("Volume has no storage class")
	}

	class, err := kubeClient.StorageV1().StorageClasses().Get(className, metav1.GetOptions{})
	if err != nil {
		return nil, err
	}
	return class, nil
}

// CheckNodeAffinity looks at the PV node affinity, and checks if the node has the same corresponding labels
// This ensures that we don't mount a volume that doesn't belong to this node
func CheckNodeAffinity(pv *v1.PersistentVolume, nodeLabels map[string]string) error {
	affinity, err := v1helper.GetStorageNodeAffinityFromAnnotation(pv.Annotations)
	if err != nil {
		return fmt.Errorf("Error getting storage node affinity: %v", err)
	}
	if affinity == nil {
		return nil
	}

	if affinity.RequiredDuringSchedulingIgnoredDuringExecution != nil {
		terms := affinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms
		glog.V(10).Infof("Match for RequiredDuringSchedulingIgnoredDuringExecution node selector terms %+v", terms)
		for _, term := range terms {
			selector, err := v1helper.NodeSelectorRequirementsAsSelector(term.MatchExpressions)
			if err != nil {
				return fmt.Errorf("Failed to parse MatchExpressions: %v", err)
			}
			if !selector.Matches(labels.Set(nodeLabels)) {
				return fmt.Errorf("NodeSelectorTerm %+v does not match node labels", term.MatchExpressions)
			}
		}
	}
	return nil
}

// LoadPodFromFile will read, decode, and return a Pod from a file.
func LoadPodFromFile(filePath string) (*v1.Pod, error) {
	if filePath == "" {
		return nil, fmt.Errorf("file path not specified")
	}
	podDef, err := ioutil.ReadFile(filePath)
	if err != nil {
		return nil, fmt.Errorf("failed to read file path %s: %+v", filePath, err)
	}
	if len(podDef) == 0 {
		return nil, fmt.Errorf("file was empty: %s", filePath)
	}
	pod := &v1.Pod{}

	codec := legacyscheme.Codecs.LegacyCodec(legacyscheme.Registry.GroupOrDie(v1.GroupName).GroupVersion)
	if err := runtime.DecodeInto(codec, podDef, pod); err != nil {
		return nil, fmt.Errorf("failed decoding file: %v", err)
	}
	return pod, nil
}

func ZonesSetToLabelValue(strSet sets.String) string {
	return strings.Join(strSet.UnsortedList(), kubeletapis.LabelMultiZoneDelimiter)
}

// ZonesToSet converts a string containing a comma separated list of zones to set
func ZonesToSet(zonesString string) (sets.String, error) {
	return stringToSet(zonesString, ",")
}

// LabelZonesToSet converts a PV label value from string containing a delimited list of zones to set
func LabelZonesToSet(labelZonesValue string) (sets.String, error) {
	return stringToSet(labelZonesValue, kubeletapis.LabelMultiZoneDelimiter)
}

// StringToSet converts a string containing list separated by specified delimiter to to a set
func stringToSet(str, delimiter string) (sets.String, error) {
	zonesSlice := strings.Split(str, delimiter)
	zonesSet := make(sets.String)
	for _, zone := range zonesSlice {
		trimmedZone := strings.TrimSpace(zone)
		if trimmedZone == "" {
			return make(sets.String), fmt.Errorf(
				"%q separated list (%q) must not contain an empty string",
				delimiter,
				str)
		}
		zonesSet.Insert(trimmedZone)
	}
	return zonesSet, nil
}
<|endoftext|>


<commit_before>package goldingester

import (
	"encoding/json"
	"fmt"
	"sort"
	"strings"

	metrics "github.com/rcrowley/go-metrics"
	"github.com/skia-dev/glog"
	"skia.googlesource.com/buildbot.git/go/util"
	"skia.googlesource.com/buildbot.git/perf/go/ingester"
	"skia.googlesource.com/buildbot.git/perf/go/types"
)

// The JSON output from DM looks like this:
//
//  {
//     "build_number" : "20",
//     "gitHash" : "abcd",
//     "key" : {
//        "arch" : "x86",
//        "configuration" : "Debug",
//        "gpu" : "nvidia",
//        "model" : "z620",
//        "os" : "Ubuntu13.10"
//     },
//     "results" : [
//        {
//           "key" : {
//              "config" : "565",
//              "name" : "ninepatch-stretch",
//              "source_type" : "gm"
//           },
//           "md5" : "f78cfafcbabaf815f3dfcf61fb59acc7",
//           "options" : {
//              "ext" : "png"
//           }
//        },
//        {
//           "key" : {
//              "config" : "8888",
//              "name" : "ninepatch-stretch",
//              "source_type" : "gm"
//           },
//           "md5" : "3e8a42f35a1e76f00caa191e6310d789",
//           "options" : {
//              "ext" : "png"
//           }

// DMResults is the top level structure for decoding DM JSON output.
type DMResults struct {
	BuildNumber string            `json:"build_number"`
	GitHash     string            `json:"gitHash"`
	Key         map[string]string `json:"key"`
	Results     []*Result         `json:"results"`
}

type Result struct {
	Key     map[string]string `json:"key"`
	Options map[string]string `json:"options"`
	Digest  string            `json:"md5"`
}

// ResultAlias is a type alias to avoid recursive calls to UnmarshalJSON.
type ResultAlias Result

func (r *Result) UnmarshalJSON(data []byte) error {
	var ret ResultAlias
	err := json.Unmarshal(data, &ret)
	if err != nil {
		return err
	}
	*r = Result(ret)

	// Make sure ext is set.
	if ext, ok := r.Options["ext"]; !ok || ext == "" {
		r.Options["ext"] = "png"
	}
	return nil
}

func NewDMResults() *DMResults {
	return &DMResults{
		Key:     map[string]string{},
		Results: []*Result{},
	}
}

// idAndParams constructs the Trace ID and the Trace params from the keys and options.
func idAndParams(dm *DMResults, r *Result) (string, map[string]string) {
	traceIdParts := map[string]string{}
	params := map[string]string{}
	for k, v := range dm.Key {
		traceIdParts[k] = v
		params[k] = v
	}
	for k, v := range r.Key {
		traceIdParts[k] = v
		params[k] = v
	}
	for k, v := range r.Options {
		params[k] = v
	}

	keys := []string{}
	for k, _ := range traceIdParts {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	values := []string{}
	for _, k := range keys {
		values = append(values, traceIdParts[k])
	}
	return strings.Join(values, ":"), params
}

// addResultToTile adds the Digests from the DMResults to the tile at the given offset.
func addResultToTile(res *DMResults, tile *types.Tile, offset int, counter metrics.Counter) {
	for _, r := range res.Results {
		if ext, ok := r.Options["ext"]; !ok || ext != "png" {
			continue // Temporarily skip non-PNG results until we know how to ingest them.
		}

		traceID, params := idAndParams(res, r)

		var trace *types.GoldenTrace
		var ok bool
		needsUpdate := false
		if tr, ok := tile.Traces[traceID]; !ok {
			trace = types.NewGoldenTrace()
			tile.Traces[traceID] = trace
			needsUpdate = true
		} else {
			trace = tr.(*types.GoldenTrace)
			if !util.MapsEqual(params, tile.Traces[traceID].Params()) {
				needsUpdate = true
			}
		}
		trace.Params_ = params

		if needsUpdate {
			// Update the Tile's ParamSet with any new keys or values we see.
			for k, v := range params {
				if _, ok = tile.ParamSet[k]; !ok {
					tile.ParamSet[k] = []string{v}
				} else if !util.In(v, tile.ParamSet[k]) {
					tile.ParamSet[k] = append(tile.ParamSet[k], v)
				}
			}
		}
		trace.Values[offset] = r.Digest
		counter.Inc(1)
	}
}

// GoldIngester implements the ingester.ResultIngester interface.
type GoldIngester struct{}

func NewGoldIngester() ingester.ResultIngester {
	return GoldIngester{}
}

// See the ingester.ResultIngester interface.
func (i GoldIngester) Ingest(tt *ingester.TileTracker, opener ingester.Opener, fname string, counter metrics.Counter) error {
	r, err := opener()
	if err != nil {
		return err
	}
	defer r.Close()

	dec := json.NewDecoder(r)
	res := NewDMResults()
	if err := dec.Decode(res); err != nil {
		return fmt.Errorf("Failed to decode DM result: %s", err)
	}

	if res.GitHash != "" {
		glog.Infof("Got Git hash: %s", res.GitHash)
		if err := tt.Move(res.GitHash); err != nil {
			return fmt.Errorf("Failed to move to correct Tile: %s: %s", res.GitHash, err)
		}
		addResultToTile(res, tt.Tile(), tt.Offset(res.GitHash), counter)
	} else {
		return fmt.Errorf("Missing hash.")
	}

	return nil
}

// See the ingester.ResultIngester interface.
func (i GoldIngester) BatchFinished(counter metrics.Counter) error {
	return nil
}
<commit_msg>Fix ingestion.<commit_after>package goldingester

import (
	"encoding/json"
	"fmt"
	"sort"
	"strings"

	metrics "github.com/rcrowley/go-metrics"
	"github.com/skia-dev/glog"
	"skia.googlesource.com/buildbot.git/go/util"
	"skia.googlesource.com/buildbot.git/perf/go/ingester"
	"skia.googlesource.com/buildbot.git/perf/go/types"
)

// The JSON output from DM looks like this:
//
//  {
//     "build_number" : "20",
//     "gitHash" : "abcd",
//     "key" : {
//        "arch" : "x86",
//        "configuration" : "Debug",
//        "gpu" : "nvidia",
//        "model" : "z620",
//        "os" : "Ubuntu13.10"
//     },
//     "results" : [
//        {
//           "key" : {
//              "config" : "565",
//              "name" : "ninepatch-stretch",
//              "source_type" : "gm"
//           },
//           "md5" : "f78cfafcbabaf815f3dfcf61fb59acc7",
//           "options" : {
//              "ext" : "png"
//           }
//        },
//        {
//           "key" : {
//              "config" : "8888",
//              "name" : "ninepatch-stretch",
//              "source_type" : "gm"
//           },
//           "md5" : "3e8a42f35a1e76f00caa191e6310d789",
//           "options" : {
//              "ext" : "png"
//           }

// DMResults is the top level structure for decoding DM JSON output.
type DMResults struct {
	BuildNumber string            `json:"build_number"`
	GitHash     string            `json:"gitHash"`
	Key         map[string]string `json:"key"`
	Results     []*Result         `json:"results"`
}

type Result struct {
	Key     map[string]string `json:"key"`
	Options map[string]string `json:"options"`
	Digest  string            `json:"md5"`
}

// ResultAlias is a type alias to avoid recursive calls to UnmarshalJSON.
type ResultAlias Result

func (r *Result) UnmarshalJSON(data []byte) error {
	ret := ResultAlias{
		Key:     map[string]string{},
		Options: map[string]string{},
	}
	err := json.Unmarshal(data, &ret)
	if err != nil {
		return err
	}
	*r = Result(ret)

	// Make sure ext is set.
	if ext, ok := r.Options["ext"]; !ok || ext == "" {
		r.Options["ext"] = "png"
	}
	return nil
}

func NewDMResults() *DMResults {
	return &DMResults{
		Key:     map[string]string{},
		Results: []*Result{},
	}
}

// idAndParams constructs the Trace ID and the Trace params from the keys and options.
func idAndParams(dm *DMResults, r *Result) (string, map[string]string) {
	traceIdParts := map[string]string{}
	params := map[string]string{}
	for k, v := range dm.Key {
		traceIdParts[k] = v
		params[k] = v
	}
	for k, v := range r.Key {
		traceIdParts[k] = v
		params[k] = v
	}
	for k, v := range r.Options {
		params[k] = v
	}

	keys := []string{}
	for k, _ := range traceIdParts {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	values := []string{}
	for _, k := range keys {
		values = append(values, traceIdParts[k])
	}
	return strings.Join(values, ":"), params
}

// addResultToTile adds the Digests from the DMResults to the tile at the given offset.
func addResultToTile(res *DMResults, tile *types.Tile, offset int, counter metrics.Counter) {
	for _, r := range res.Results {
		if ext, ok := r.Options["ext"]; !ok || ext != "png" {
			continue // Temporarily skip non-PNG results until we know how to ingest them.
		}

		traceID, params := idAndParams(res, r)

		var trace *types.GoldenTrace
		var ok bool
		needsUpdate := false
		if tr, ok := tile.Traces[traceID]; !ok {
			trace = types.NewGoldenTrace()
			tile.Traces[traceID] = trace
			needsUpdate = true
		} else {
			trace = tr.(*types.GoldenTrace)
			if !util.MapsEqual(params, tile.Traces[traceID].Params()) {
				needsUpdate = true
			}
		}
		trace.Params_ = params

		if needsUpdate {
			// Update the Tile's ParamSet with any new keys or values we see.
			for k, v := range params {
				if _, ok = tile.ParamSet[k]; !ok {
					tile.ParamSet[k] = []string{v}
				} else if !util.In(v, tile.ParamSet[k]) {
					tile.ParamSet[k] = append(tile.ParamSet[k], v)
				}
			}
		}
		trace.Values[offset] = r.Digest
		counter.Inc(1)
	}
}

// GoldIngester implements the ingester.ResultIngester interface.
type GoldIngester struct{}

func NewGoldIngester() ingester.ResultIngester {
	return GoldIngester{}
}

// See the ingester.ResultIngester interface.
func (i GoldIngester) Ingest(tt *ingester.TileTracker, opener ingester.Opener, fname string, counter metrics.Counter) error {
	r, err := opener()
	if err != nil {
		return err
	}
	defer r.Close()

	dec := json.NewDecoder(r)
	res := NewDMResults()
	if err := dec.Decode(res); err != nil {
		return fmt.Errorf("Failed to decode DM result: %s", err)
	}

	if res.GitHash != "" {
		glog.Infof("Got Git hash: %s", res.GitHash)
		if err := tt.Move(res.GitHash); err != nil {
			return fmt.Errorf("Failed to move to correct Tile: %s: %s", res.GitHash, err)
		}
		addResultToTile(res, tt.Tile(), tt.Offset(res.GitHash), counter)
	} else {
		return fmt.Errorf("Missing hash.")
	}

	return nil
}

// See the ingester.ResultIngester interface.
func (i GoldIngester) BatchFinished(counter metrics.Counter) error {
	return nil
}
<|endoftext|>


<commit_before>package api_test

import (
	"net/http/httptest"
	"testing"

	"github.com/diyan/assimilator/migrations"
	"github.com/diyan/assimilator/testutil/factory"
	"github.com/diyan/assimilator/testutil/testclient"
	"github.com/diyan/assimilator/web"
	"github.com/gocraft/dbr"
	"github.com/labstack/echo"
	"github.com/parnurzeal/gorequest"
	"github.com/pkg/errors"
	"github.com/stretchr/testify/require"
	"github.com/stretchr/testify/suite"
)

type testSuite struct {
	suite.Suite
	HttpRecorder *httptest.ResponseRecorder
	Client       *gorequest.SuperAgent
	App          *echo.Echo
	Factory      factory.TestFactory
}

func (t *testSuite) SetupSuite() {
	// TODO check what is faster - re-create db or drop all tables?
	// select 'drop table "' || tablename || '" cascade;'
	// from pg_tables where schemaname = 'sentry_ci';

	// TODO remove duplicated code
	noError := require.New(t.T()).NoError
	conn, err := dbr.Open("postgres", "postgres://postgres@localhost/postgres?sslmode=disable", nil)
	noError(errors.Wrap(err, "failed to init db connection"))
	// dbr.Open calls sql.Open which returns err == nil even if there is no db connection,
	//   so it is required to explicitly ping the database
	err = conn.Ping()
	noError(errors.Wrap(err, "failed to ping db"))
	sess := conn.NewSession(nil)
	// Force drop db while others may be connected
	_, err = sess.Exec(`
		select pg_terminate_backend(pid)
		from pg_stat_activity
		where datname = 'sentry_ci';`)
	noError(err)
	_, err = sess.Exec("drop database if exists sentry_ci;")
	noError(err)
	_, err = sess.Exec("create database sentry_ci;")
	noError(err)
	noError(migrations.UpgradeDB(factory.MakeAppConfig().DatabaseURL))
}

func (t *testSuite) SetupTest() {
	t.App = web.GetApp(factory.MakeAppConfig())
	// TODO TestFactory does a side effect which is used by TestClient
	//  make this code more explicit
	t.Factory = factory.New(t.T(), t.App)
	t.Client = testclient.New(t.App)
}

func (t *testSuite) TearDownTest() {
	t.Factory.Reset()
}

func TestRunSuite(t *testing.T) {
	suite.Run(t, new(testSuite))
}
<commit_msg>Remove unused HttpRecorder from testSuite<commit_after>package api_test

import (
	"testing"

	"github.com/diyan/assimilator/migrations"
	"github.com/diyan/assimilator/testutil/factory"
	"github.com/diyan/assimilator/testutil/testclient"
	"github.com/diyan/assimilator/web"
	"github.com/gocraft/dbr"
	"github.com/labstack/echo"
	"github.com/parnurzeal/gorequest"
	"github.com/pkg/errors"
	"github.com/stretchr/testify/require"
	"github.com/stretchr/testify/suite"
)

type testSuite struct {
	suite.Suite
	Client  *gorequest.SuperAgent
	App     *echo.Echo
	Factory factory.TestFactory
}

func (t *testSuite) SetupSuite() {
	// TODO check what is faster - re-create db or drop all tables?
	// select 'drop table "' || tablename || '" cascade;'
	// from pg_tables where schemaname = 'sentry_ci';

	// TODO remove duplicated code
	noError := require.New(t.T()).NoError
	conn, err := dbr.Open("postgres", "postgres://postgres@localhost/postgres?sslmode=disable", nil)
	noError(errors.Wrap(err, "failed to init db connection"))
	// dbr.Open calls sql.Open which returns err == nil even if there is no db connection,
	//   so it is required to explicitly ping the database
	err = conn.Ping()
	noError(errors.Wrap(err, "failed to ping db"))
	sess := conn.NewSession(nil)
	// Force drop db while others may be connected
	_, err = sess.Exec(`
		select pg_terminate_backend(pid)
		from pg_stat_activity
		where datname = 'sentry_ci';`)
	noError(err)
	_, err = sess.Exec("drop database if exists sentry_ci;")
	noError(err)
	_, err = sess.Exec("create database sentry_ci;")
	noError(err)
	noError(migrations.UpgradeDB(factory.MakeAppConfig().DatabaseURL))
}

func (t *testSuite) SetupTest() {
	t.App = web.GetApp(factory.MakeAppConfig())
	// TODO TestFactory does a side effect which is used by TestClient
	//  make this code more explicit
	t.Factory = factory.New(t.T(), t.App)
	t.Client = testclient.New(t.App)
}

func (t *testSuite) TearDownTest() {
	t.Factory.Reset()
}

func TestRunSuite(t *testing.T) {
	suite.Run(t, new(testSuite))
}
<|endoftext|>


<commit_before>// Package path resolves using http path
package path

import (
	"errors"
	"net/http"
	"strings"

	"github.com/micro/go-micro/v2/api/resolver"
)

type Resolver struct{}

func (r *Resolver) Resolve(req *http.Request) (*resolver.Endpoint, error) {
	if req.URL.Path == "/" {
		return nil, errors.New("unknown name")
	}
	parts := strings.Split(req.URL.Path[1:], "/")
	return &resolver.Endpoint{
		Name:   parts[0],
		Host:   req.Host,
		Method: req.Method,
		Path:   req.URL.Path,
	}, nil
}

func (r *Resolver) String() string {
	return "path"
}

func NewResolver(opts ...resolver.Option) resolver.Resolver {
	return &Resolver{}
}
<commit_msg>Fix failing test<commit_after>// Package path resolves using http path
package path

import (
	"net/http"
	"strings"

	"github.com/micro/go-micro/v2/api/resolver"
)

type Resolver struct{}

func (r *Resolver) Resolve(req *http.Request) (*resolver.Endpoint, error) {
	if req.URL.Path == "/" {
		return nil, resolver.ErrNotFound
	}
	parts := strings.Split(req.URL.Path[1:], "/")
	return &resolver.Endpoint{
		Name:   parts[0],
		Host:   req.Host,
		Method: req.Method,
		Path:   req.URL.Path,
	}, nil
}

func (r *Resolver) String() string {
	return "path"
}

func NewResolver(opts ...resolver.Option) resolver.Resolver {
	return &Resolver{}
}
<|endoftext|>


<commit_before>package dht

// #include "dht.go.h"
// #cgo LDFLAGS: -lrt
import "C"

import (
	"bytes"
	"fmt"
	"reflect"
	"time"
	"unsafe"
)

type SensorType int

// Implement Stringer interface.
func (this SensorType) String() string {
	if this == DHT11 {
		return "DHT11"
	} else if this == DHT22 {
		return "DHT22"
	} else if this == AM2302 {
		return "AM2302"
	} else {
		return "!!! unknown !!!"
	}
}

const (
	// Most populare sensor
	DHT11 SensorType = iota + 1
	// More expensive and precise than DHT11
	DHT22
	// Aka DHT22
	AM2302 = DHT22
)

// Keep pulse state with how long it lasted.
type Pulse struct {
	Value    byte
	Duration time.Duration
}

// Activate sensor and get back bunch of pulses for further decoding.
// C function call wrapper.
func dialDHTxxAndGetResponse(pin int, boostPerfFlag bool) ([]Pulse, error) {
	var arr *C.int32_t
	var arrLen C.int32_t
	var list []int32
	var boost C.int32_t = 0
	if boostPerfFlag {
		boost = 1
	}
	// Return array: [pulse, duration, pulse, duration, ...]
	r := C.dial_DHTxx_and_read(4, boost, &arr, &arrLen)
	if r == -1 {
		err := fmt.Errorf("Error during call C.dial_DHTxx_and_read()")
		return nil, err
	}
	defer C.free(unsafe.Pointer(arr))
	// Convert original C array arr to Go slice list
	h := (*reflect.SliceHeader)(unsafe.Pointer(&list))
	h.Data = uintptr(unsafe.Pointer(arr))
	h.Len = int(arrLen)
	h.Cap = int(arrLen)
	pulses := make([]Pulse, len(list)/2)
	// Convert original int array ([pulse, duration, pulse, duration, ...])
	// to Pulse struct array
	for i := 0; i < len(list)/2; i++ {
		var value byte = 0
		if list[i*2] != 0 {
			value = 1
		}
		pulses[i] = Pulse{Value: value,
			Duration: time.Duration(list[i*2+1]) * time.Microsecond}
	}
	return pulses, nil
}

// TODO write comment to function
func decodeByte(pulses []Pulse, start int) (byte, error) {
	if len(pulses)-start < 16 {
		return 0, fmt.Errorf("Can't decode byte, since range between "+
			"index and array length is less than 16: %d, %d", start, len(pulses))
	}
	var b int = 0
	for i := 0; i < 8; i++ {
		pulseL := pulses[start+i*2]
		pulseH := pulses[start+i*2+1]
		if pulseL.Value != 0 {
			return 0, fmt.Errorf("Low edge value expected at index %d", start+i*2)
		}
		if pulseH.Value == 0 {
			return 0, fmt.Errorf("High edge value expected at index %d", start+i*2+1)
		}
		const HIGH_DUR_MAX = (70 + (70 + 54)) / 2 * time.Microsecond
		// Calc average value between 24us (bit 0) and 70us (bit 1).
		// Everything that less than this param is bit 0, bigger - bit 1.
		const HIGH_DUR_AVG = (24 + (70-24)/2) * time.Microsecond
		if pulseH.Duration > HIGH_DUR_MAX {
			return 0, fmt.Errorf("High edge value duration %v exceed "+
				"expected maximum amount %v", pulseH.Duration, HIGH_DUR_MAX)
		}
		if pulseH.Duration > HIGH_DUR_AVG {
			//fmt.Printf("bit %d is high\n", 7-i)
			b = b | (1 << uint(7-i))
		}
	}
	return byte(b), nil
}

// Decode bunch of pulse read from DHTxx sensors.
// Use pdf specifications from /docs folder to read 5 bytes and
// convert them to temperature and humidity.
func decodeDHT11Pulses(sensorType SensorType, pulses []Pulse) (temperature float32,
	humidity float32, err error) {
	if len(pulses) == 85 {
		pulses = pulses[3:]
	} else if len(pulses) == 84 {
		pulses = pulses[2:]
	} else if len(pulses) == 83 {
		pulses = pulses[1:]
	} else if len(pulses) != 82 {
		printPulseArrayForDebug(pulses)
		return -1, -1, fmt.Errorf("Can't decode pulse array received from "+
			"DHTxx sensor, since incorrect length: %d", len(pulses))
	}
	pulses = pulses[:80]
	// Decode 1st byte
	b0, err := decodeByte(pulses, 0)
	if err != nil {
		return -1, -1, err
	}
	// Decode 2nd byte
	b1, err := decodeByte(pulses, 16)
	if err != nil {
		return -1, -1, err
	}
	// Decode 3rd byte
	b2, err := decodeByte(pulses, 32)
	if err != nil {
		return -1, -1, err
	}
	// Decode 4th byte
	b3, err := decodeByte(pulses, 48)
	if err != nil {
		return -1, -1, err
	}
	// Decode 5th byte: control sum to verify all data received from sensor
	sum, err := decodeByte(pulses, 64)
	if err != nil {
		return -1, -1, err
	}
	// Produce data integrity check
	if sum != byte(b0+b1+b2+b3) {
		err := fmt.Errorf("Control sum %d doesn't match %d (%d+%d+%d+%d)",
			sum, byte(b0+b1+b2+b3), b0, b1, b2, b3)
		return -1, -1, err
	}
	// Debug output for 5 bytes
	log.Debug("Five bytes from DHTxx: [%d, %d, %d, %d, %d]", b0, b1, b2, b3, sum)
	// Extract temprature and humidity depending on sensor type
	temperature, humidity = 0.0, 0.0
	if sensorType == DHT11 {
		humidity = float32(b0)
		temperature = float32(b2)
	} else if sensorType == DHT22 {
		humidity = (float32(b0)*256 + float32(b1)) / 10.0
		temperature = (float32(b2&0x7F)*256 + float32(b3)) / 10.0
		if b2&0x80 != 0 {
			temperature *= -1.0
		}
	}
	if humidity > 100.0 {
		return -1, -1, fmt.Errorf("Humidity value exceed 100%%: %v", humidity)
	}
	// Success
	return temperature, humidity, nil
}

// Print bunch of pulses for debug purpose.
func printPulseArrayForDebug(pulses []Pulse) {
	var buf bytes.Buffer
	for i, pulse := range pulses {
		buf.WriteString(fmt.Sprintf("pulse %3d: %v, %v\n", i,
			pulse.Value, pulse.Duration))
	}
	log.Debug("Pulse count %d:\n%v", len(pulses), buf.String())
}

// Send activation request to DHTxx sensor via specific pin.
// Then decode pulses sent back with asynchronous
// protocol specific for DHTxx sensors.
//
// Input parameters:
// 1) sensor type: DHT11, DHT22 (aka AM2302);
// 2) pin number from GPIO connector to interract with sensor;
// 3) boost GPIO performance flag should be used for old devices
// such as Raspberry PI 1 (this will require root privileges).
//
// Return:
// 1) temperature in Celsius;
// 2) humidity in percent;
// 3) error if present.
func ReadDHTxx(sensorType SensorType, pin int,
	boostPerfFlag bool) (temperature float32, humidity float32, err error) {
	// Activate sensor and read data to pulses array
	pulses, err := dialDHTxxAndGetResponse(pin, boostPerfFlag)
	if err != nil {
		return -1, -1, err
	}
	// Output debug information
	printPulseArrayForDebug(pulses)
	// Decode pulses
	temp, hum, err := decodeDHT11Pulses(sensorType, pulses)
	if err != nil {
		return -1, -1, err
	}
	return temp, hum, nil
}

// Send activation request to DHTxx sensor via specific pin.
// Then decode pulses sent back with asynchronous
// protocol specific for DHTxx sensors. Retry n times in case of failure.
//
// Input parameters:
// 1) sensor type: DHT11, DHT22 (aka AM2302);
// 2) pin number from gadget GPIO to interract with sensor;
// 3) boost GPIO performance flag should be used for old devices
// such as Raspberry PI 1 (this will require root privileges);
// 4) how many times to retry until success either сounter is zeroed.
//
// Return:
// 1) temperature in Celsius;
// 2) humidity in percent;
// 3) number of extra retries data from sensor;
// 4) error if present.
func ReadDHTxxWithRetry(sensorType SensorType, pin int, boostPerfFlag bool,
	retry int) (temperature float32, humidity float32, retried int, err error) {
	retried = 0
	for {
		temp, hum, err := ReadDHTxx(sensorType, pin, boostPerfFlag)
		if err != nil {
			if retry > 0 {
				log.Warning("%v", err)
				retry--
				retried++
				// Sleep before new attempt
				time.Sleep(1500 * time.Millisecond)
				continue
			}
			return -1, -1, retried, err
		}
		return temp, hum, retried, nil
	}
}
<commit_msg>dht.go: Fix for using the fixed pin number<commit_after>package dht

// #include "dht.go.h"
// #cgo LDFLAGS: -lrt
import "C"

import (
	"bytes"
	"fmt"
	"reflect"
	"time"
	"unsafe"
)

type SensorType int

// Implement Stringer interface.
func (this SensorType) String() string {
	if this == DHT11 {
		return "DHT11"
	} else if this == DHT22 {
		return "DHT22"
	} else if this == AM2302 {
		return "AM2302"
	} else {
		return "!!! unknown !!!"
	}
}

const (
	// Most populare sensor
	DHT11 SensorType = iota + 1
	// More expensive and precise than DHT11
	DHT22
	// Aka DHT22
	AM2302 = DHT22
)

// Keep pulse state with how long it lasted.
type Pulse struct {
	Value    byte
	Duration time.Duration
}

// Activate sensor and get back bunch of pulses for further decoding.
// C function call wrapper.
func dialDHTxxAndGetResponse(pin int, boostPerfFlag bool) ([]Pulse, error) {
	var arr *C.int32_t
	var arrLen C.int32_t
	var list []int32
	var boost C.int32_t = 0
	if boostPerfFlag {
		boost = 1
	}
	// Return array: [pulse, duration, pulse, duration, ...]
	r := C.dial_DHTxx_and_read(C.int32_t(pin), boost, &arr, &arrLen)
	if r == -1 {
		err := fmt.Errorf("Error during call C.dial_DHTxx_and_read()")
		return nil, err
	}
	defer C.free(unsafe.Pointer(arr))
	// Convert original C array arr to Go slice list
	h := (*reflect.SliceHeader)(unsafe.Pointer(&list))
	h.Data = uintptr(unsafe.Pointer(arr))
	h.Len = int(arrLen)
	h.Cap = int(arrLen)
	pulses := make([]Pulse, len(list)/2)
	// Convert original int array ([pulse, duration, pulse, duration, ...])
	// to Pulse struct array
	for i := 0; i < len(list)/2; i++ {
		var value byte = 0
		if list[i*2] != 0 {
			value = 1
		}
		pulses[i] = Pulse{Value: value,
			Duration: time.Duration(list[i*2+1]) * time.Microsecond}
	}
	return pulses, nil
}

// TODO write comment to function
func decodeByte(pulses []Pulse, start int) (byte, error) {
	if len(pulses)-start < 16 {
		return 0, fmt.Errorf("Can't decode byte, since range between "+
			"index and array length is less than 16: %d, %d", start, len(pulses))
	}
	var b int = 0
	for i := 0; i < 8; i++ {
		pulseL := pulses[start+i*2]
		pulseH := pulses[start+i*2+1]
		if pulseL.Value != 0 {
			return 0, fmt.Errorf("Low edge value expected at index %d", start+i*2)
		}
		if pulseH.Value == 0 {
			return 0, fmt.Errorf("High edge value expected at index %d", start+i*2+1)
		}
		const HIGH_DUR_MAX = (70 + (70 + 54)) / 2 * time.Microsecond
		// Calc average value between 24us (bit 0) and 70us (bit 1).
		// Everything that less than this param is bit 0, bigger - bit 1.
		const HIGH_DUR_AVG = (24 + (70-24)/2) * time.Microsecond
		if pulseH.Duration > HIGH_DUR_MAX {
			return 0, fmt.Errorf("High edge value duration %v exceed "+
				"expected maximum amount %v", pulseH.Duration, HIGH_DUR_MAX)
		}
		if pulseH.Duration > HIGH_DUR_AVG {
			//fmt.Printf("bit %d is high\n", 7-i)
			b = b | (1 << uint(7-i))
		}
	}
	return byte(b), nil
}

// Decode bunch of pulse read from DHTxx sensors.
// Use pdf specifications from /docs folder to read 5 bytes and
// convert them to temperature and humidity.
func decodeDHT11Pulses(sensorType SensorType, pulses []Pulse) (temperature float32,
	humidity float32, err error) {
	if len(pulses) == 85 {
		pulses = pulses[3:]
	} else if len(pulses) == 84 {
		pulses = pulses[2:]
	} else if len(pulses) == 83 {
		pulses = pulses[1:]
	} else if len(pulses) != 82 {
		printPulseArrayForDebug(pulses)
		return -1, -1, fmt.Errorf("Can't decode pulse array received from "+
			"DHTxx sensor, since incorrect length: %d", len(pulses))
	}
	pulses = pulses[:80]
	// Decode 1st byte
	b0, err := decodeByte(pulses, 0)
	if err != nil {
		return -1, -1, err
	}
	// Decode 2nd byte
	b1, err := decodeByte(pulses, 16)
	if err != nil {
		return -1, -1, err
	}
	// Decode 3rd byte
	b2, err := decodeByte(pulses, 32)
	if err != nil {
		return -1, -1, err
	}
	// Decode 4th byte
	b3, err := decodeByte(pulses, 48)
	if err != nil {
		return -1, -1, err
	}
	// Decode 5th byte: control sum to verify all data received from sensor
	sum, err := decodeByte(pulses, 64)
	if err != nil {
		return -1, -1, err
	}
	// Produce data integrity check
	if sum != byte(b0+b1+b2+b3) {
		err := fmt.Errorf("Control sum %d doesn't match %d (%d+%d+%d+%d)",
			sum, byte(b0+b1+b2+b3), b0, b1, b2, b3)
		return -1, -1, err
	}
	// Debug output for 5 bytes
	log.Debug("Five bytes from DHTxx: [%d, %d, %d, %d, %d]", b0, b1, b2, b3, sum)
	// Extract temprature and humidity depending on sensor type
	temperature, humidity = 0.0, 0.0
	if sensorType == DHT11 {
		humidity = float32(b0)
		temperature = float32(b2)
	} else if sensorType == DHT22 {
		humidity = (float32(b0)*256 + float32(b1)) / 10.0
		temperature = (float32(b2&0x7F)*256 + float32(b3)) / 10.0
		if b2&0x80 != 0 {
			temperature *= -1.0
		}
	}
	if humidity > 100.0 {
		return -1, -1, fmt.Errorf("Humidity value exceed 100%%: %v", humidity)
	}
	// Success
	return temperature, humidity, nil
}

// Print bunch of pulses for debug purpose.
func printPulseArrayForDebug(pulses []Pulse) {
	var buf bytes.Buffer
	for i, pulse := range pulses {
		buf.WriteString(fmt.Sprintf("pulse %3d: %v, %v\n", i,
			pulse.Value, pulse.Duration))
	}
	log.Debug("Pulse count %d:\n%v", len(pulses), buf.String())
}

// Send activation request to DHTxx sensor via specific pin.
// Then decode pulses sent back with asynchronous
// protocol specific for DHTxx sensors.
//
// Input parameters:
// 1) sensor type: DHT11, DHT22 (aka AM2302);
// 2) pin number from GPIO connector to interract with sensor;
// 3) boost GPIO performance flag should be used for old devices
// such as Raspberry PI 1 (this will require root privileges).
//
// Return:
// 1) temperature in Celsius;
// 2) humidity in percent;
// 3) error if present.
func ReadDHTxx(sensorType SensorType, pin int,
	boostPerfFlag bool) (temperature float32, humidity float32, err error) {
	// Activate sensor and read data to pulses array
	pulses, err := dialDHTxxAndGetResponse(pin, boostPerfFlag)
	if err != nil {
		return -1, -1, err
	}
	// Output debug information
	printPulseArrayForDebug(pulses)
	// Decode pulses
	temp, hum, err := decodeDHT11Pulses(sensorType, pulses)
	if err != nil {
		return -1, -1, err
	}
	return temp, hum, nil
}

// Send activation request to DHTxx sensor via specific pin.
// Then decode pulses sent back with asynchronous
// protocol specific for DHTxx sensors. Retry n times in case of failure.
//
// Input parameters:
// 1) sensor type: DHT11, DHT22 (aka AM2302);
// 2) pin number from gadget GPIO to interract with sensor;
// 3) boost GPIO performance flag should be used for old devices
// such as Raspberry PI 1 (this will require root privileges);
// 4) how many times to retry until success either сounter is zeroed.
//
// Return:
// 1) temperature in Celsius;
// 2) humidity in percent;
// 3) number of extra retries data from sensor;
// 4) error if present.
func ReadDHTxxWithRetry(sensorType SensorType, pin int, boostPerfFlag bool,
	retry int) (temperature float32, humidity float32, retried int, err error) {
	retried = 0
	for {
		temp, hum, err := ReadDHTxx(sensorType, pin, boostPerfFlag)
		if err != nil {
			if retry > 0 {
				log.Warning("%v", err)
				retry--
				retried++
				// Sleep before new attempt
				time.Sleep(1500 * time.Millisecond)
				continue
			}
			return -1, -1, retried, err
		}
		return temp, hum, retried, nil
	}
}
<|endoftext|>