<commit_before>package docker

import (
	"context"
	"crypto/tls"
	"fmt"
	"log"
	"net"
	"net/http"
	"sync"
	"time"

	"github.com/docker/docker/api"
	"github.com/docker/docker/client"
	"github.com/docker/go-connections/tlsconfig"
	"github.com/play-with-docker/play-with-docker/router"
	"github.com/play-with-docker/play-with-docker/storage"
)

type localCachedFactory struct {
	rw              sync.Mutex
	sessionClient   DockerApi
	instanceClients map[string]DockerApi
	storage         storage.StorageApi
}

func (f *localCachedFactory) GetForSession(sessionId string) (DockerApi, error) {
	f.rw.Lock()
	defer f.rw.Unlock()

	if f.sessionClient != nil {
		return f.sessionClient, nil
	}

	c, err := client.NewEnvClient()
	if err != nil {
		return nil, err
	}
	err = f.check(c)
	if err != nil {
		return nil, err
	}
	d := NewDocker(c)
	f.sessionClient = d
	return f.sessionClient, nil
}

func (f *localCachedFactory) GetForInstance(sessionId, instanceName string) (DockerApi, error) {
	f.rw.Lock()
	defer f.rw.Unlock()

	c, found := f.instanceClients[sessionId+instanceName]
	if found {
		return c, nil
	}

	instance, err := f.storage.InstanceGet(sessionId, instanceName)
	if err != nil {
		return nil, err
	}
	// Need to create client to the DinD docker daemon
	// We check if the client needs to use TLS
	var tlsConfig *tls.Config
	if len(instance.Cert) > 0 && len(instance.Key) > 0 {
		tlsConfig = tlsconfig.ClientDefault()
		tlsConfig.InsecureSkipVerify = true
		tlsCert, err := tls.X509KeyPair(instance.Cert, instance.Key)
		if err != nil {
			return nil, fmt.Errorf("Could not load X509 key pair: %v. Make sure the key is not encrypted", err)
		}
		tlsConfig.Certificates = []tls.Certificate{tlsCert}
	}

	transport := &http.Transport{
		DialContext: (&net.Dialer{
			Timeout:   1 * time.Second,
			KeepAlive: 30 * time.Second,
		}).DialContext}
	if tlsConfig != nil {
		transport.TLSClientConfig = tlsConfig
	}
	cli := &http.Client{
		Transport: transport,
	}
	dc, err := client.NewClient("http://l2:443", api.DefaultVersion, cli, map[string]string{"X-Forwarded-Host": router.EncodeHost(instance.SessionId, instance.IP, router.HostOpts{EncodedPort: 2375})})
	if err != nil {
		return nil, fmt.Errorf("Could not connect to DinD docker daemon", err)
	}
	err = f.check(dc)
	if err != nil {
		return nil, err
	}
	dockerClient := NewDocker(dc)
	f.instanceClients[sessionId+instance.Name] = dockerClient

	return dockerClient, nil
}

func (f *localCachedFactory) check(c *client.Client) error {
	ok := false
	for i := 0; i < 5; i++ {
		_, err := c.Ping(context.Background())
		if err != nil {
			if client.IsErrConnectionFailed(err) {
				// connection has failed, maybe instance is not ready yet, sleep and retry
				log.Printf("Connection to [%s] has failed, maybe instance is not ready yet, sleeping and retrying in 1 second. Try #%d\n", c.DaemonHost(), i+1)
				time.Sleep(time.Second)
				continue
			}
			return err
		}
		ok = true
		break
	}
	if !ok {
		return fmt.Errorf("Connection to docker daemon was not established.")
	}
	return nil
}

func NewLocalCachedFactory(s storage.StorageApi) *localCachedFactory {
	return &localCachedFactory{
		instanceClients: make(map[string]DockerApi),
		storage:         s,
	}
}
<commit_msg>Make it multiple goroutines friendly<commit_after>package docker

import (
	"context"
	"crypto/tls"
	"fmt"
	"log"
	"net"
	"net/http"
	"sync"
	"time"

	"github.com/docker/docker/api"
	"github.com/docker/docker/client"
	"github.com/docker/go-connections/tlsconfig"
	"github.com/play-with-docker/play-with-docker/router"
	"github.com/play-with-docker/play-with-docker/storage"
)

type localCachedFactory struct {
	rw              sync.Mutex
	irw             sync.Mutex
	sessionClient   DockerApi
	instanceClients map[string]*instanceEntry
	storage         storage.StorageApi
}

type instanceEntry struct {
	rw     sync.Mutex
	client DockerApi
}

func (f *localCachedFactory) GetForSession(sessionId string) (DockerApi, error) {
	f.rw.Lock()
	defer f.rw.Unlock()

	if f.sessionClient != nil {
		return f.sessionClient, nil
	}

	c, err := client.NewEnvClient()
	if err != nil {
		return nil, err
	}
	err = f.check(c)
	if err != nil {
		return nil, err
	}
	d := NewDocker(c)
	f.sessionClient = d
	return f.sessionClient, nil
}

func (f *localCachedFactory) GetForInstance(sessionId, instanceName string) (DockerApi, error) {
	key := sessionId + instanceName

	f.irw.Lock()
	c, found := f.instanceClients[key]
	if !found {
		c := &instanceEntry{}
		f.instanceClients[key] = c
	}
	c = f.instanceClients[key]
	f.irw.Unlock()

	c.rw.Lock()
	defer c.rw.Unlock()

	if c.client != nil {
		return c.client, nil
	}

	instance, err := f.storage.InstanceGet(sessionId, instanceName)
	if err != nil {
		return nil, err
	}
	// Need to create client to the DinD docker daemon
	// We check if the client needs to use TLS
	var tlsConfig *tls.Config
	if len(instance.Cert) > 0 && len(instance.Key) > 0 {
		tlsConfig = tlsconfig.ClientDefault()
		tlsConfig.InsecureSkipVerify = true
		tlsCert, err := tls.X509KeyPair(instance.Cert, instance.Key)
		if err != nil {
			return nil, fmt.Errorf("Could not load X509 key pair: %v. Make sure the key is not encrypted", err)
		}
		tlsConfig.Certificates = []tls.Certificate{tlsCert}
	}

	transport := &http.Transport{
		DialContext: (&net.Dialer{
			Timeout:   1 * time.Second,
			KeepAlive: 30 * time.Second,
		}).DialContext}
	if tlsConfig != nil {
		transport.TLSClientConfig = tlsConfig
	}
	cli := &http.Client{
		Transport: transport,
	}
	dc, err := client.NewClient("http://l2:443", api.DefaultVersion, cli, map[string]string{"X-Forwarded-Host": router.EncodeHost(instance.SessionId, instance.IP, router.HostOpts{EncodedPort: 2375})})
	if err != nil {
		return nil, fmt.Errorf("Could not connect to DinD docker daemon", err)
	}
	err = f.check(dc)
	if err != nil {
		return nil, err
	}
	dockerClient := NewDocker(dc)
	c.client = dockerClient

	return dockerClient, nil
}

func (f *localCachedFactory) check(c *client.Client) error {
	ok := false
	for i := 0; i < 5; i++ {
		_, err := c.Ping(context.Background())
		if err != nil {
			if client.IsErrConnectionFailed(err) {
				// connection has failed, maybe instance is not ready yet, sleep and retry
				log.Printf("Connection to [%s] has failed, maybe instance is not ready yet, sleeping and retrying in 1 second. Try #%d\n", c.DaemonHost(), i+1)
				time.Sleep(time.Second)
				continue
			}
			return err
		}
		ok = true
		break
	}
	if !ok {
		return fmt.Errorf("Connection to docker daemon was not established.")
	}
	return nil
}

func NewLocalCachedFactory(s storage.StorageApi) *localCachedFactory {
	return &localCachedFactory{
		instanceClients: make(map[string]*instanceEntry),
		storage:         s,
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2018 The Jaeger Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package consumer

import (
	"runtime"
	"strconv"
	"sync/atomic"
	"time"

	"github.com/uber/jaeger-lib/metrics"
	"go.uber.org/zap"
)

// deadlockDetector monitors the messages consumed and wither signals for the partition to be closed by sending a
// message on closePartition, or triggers a panic if the close fails. It triggers a panic if there are no messages
// consumed across all partitions.
//
// Closing the partition should result in a rebalance, which alleviates the condition. This means that rebalances can
// happen frequently if there is no traffic on the Kafka topic. This shouldn't affect normal operations.
//
// If the message send isn't processed within the next check interval, a panic is issued.This hack relies on a
// container management system (k8s, aurora, marathon, etc) to reschedule
// the dead instance.
//
// This hack protects jaeger-ingester from issues described in  https://github.com/jaegertracing/jaeger/issues/1052
//
type deadlockDetector struct {
	metricsFactory                metrics.Factory
	logger                        *zap.Logger
	interval                      time.Duration
	allPartitionsDeadlockDetector *allPartitionsDeadlockDetector
	panicFunc                     func(int32)
}

type partitionDeadlockDetector struct {
	msgConsumed                   *uint64
	logger                        *zap.Logger
	partition                     int32
	closePartition                chan struct{}
	done                          chan struct{}
	incrementAllPartitionMsgCount func()
	disabled                      bool
}

type allPartitionsDeadlockDetector struct {
	msgConsumed *uint64
	logger      *zap.Logger
	done        chan struct{}
	disabled    bool
}

func newDeadlockDetector(metricsFactory metrics.Factory, logger *zap.Logger, interval time.Duration) deadlockDetector {
	panicFunc := func(partition int32) {
		metricsFactory.Counter(metrics.Options{Name: "deadlockdetector.panic-issued", Tags: map[string]string{"partition": strconv.Itoa(int(partition))}}).Inc(1)
		time.Sleep(time.Second) // Allow time to flush metric

		buf := make([]byte, 1<<20)
		logger.Panic("No messages processed in the last check interval",
			zap.Int32("partition", partition),
			zap.String("stack", string(buf[:runtime.Stack(buf, true)])))
	}

	return deadlockDetector{
		metricsFactory: metricsFactory,
		logger:         logger,
		interval:       interval,
		panicFunc:      panicFunc,
	}
}

func (s *deadlockDetector) startMonitoringForPartition(partition int32) *partitionDeadlockDetector {
	var msgConsumed uint64
	w := &partitionDeadlockDetector{
		msgConsumed:    &msgConsumed,
		partition:      partition,
		closePartition: make(chan struct{}, 1),
		done:           make(chan struct{}),
		logger:         s.logger,
		disabled:       s.interval == 0,

		incrementAllPartitionMsgCount: func() {
			s.allPartitionsDeadlockDetector.incrementMsgCount()
		},
	}

	if w.disabled {
		s.logger.Debug("Partition deadlock detector disabled")
	} else {
		go s.monitorForPartition(w, partition)
	}

	return w
}

func (s *deadlockDetector) monitorForPartition(w *partitionDeadlockDetector, partition int32) {
	ticker := time.NewTicker(s.interval)
	defer ticker.Stop()

	for {
		select {
		case <-w.done:
			s.logger.Info("Closing ticker routine", zap.Int32("partition", partition))
			return
		case <-ticker.C:
			if atomic.LoadUint64(w.msgConsumed) == 0 {
				select {
				case w.closePartition <- struct{}{}:
					s.metricsFactory.Counter(metrics.Options{Name: "deadlockdetector.close-signalled", Tags: map[string]string{"partition": strconv.Itoa(int(partition))}}).Inc(1)
					s.logger.Warn("Signalling partition close due to inactivity", zap.Int32("partition", partition))
				default:
					// If closePartition is blocked, the consumer might have deadlocked - kill the process
					s.panicFunc(partition)
					return // For tests
				}
			} else {
				atomic.StoreUint64(w.msgConsumed, 0)
			}
		}
	}
}

// start monitors that the sum of messages consumed across all partitions is non zero for the given interval
// If it is zero when there are producers producing messages on the topic, it means that sarama-cluster hasn't
// retrieved partition assignments. (This case will not be caught by startMonitoringForPartition because no partitions
// were retrieved).
func (s *deadlockDetector) start() {
	var msgConsumed uint64
	detector := &allPartitionsDeadlockDetector{
		msgConsumed: &msgConsumed,
		done:        make(chan struct{}),
		logger:      s.logger,
		disabled:    s.interval == 0,
	}

	if detector.disabled {
		s.logger.Debug("Global deadlock detector disabled")
	} else {
		s.logger.Debug("Starting global deadlock detector")
		go func() {
			ticker := time.NewTicker(s.interval)
			defer ticker.Stop()

			for {
				select {
				case <-detector.done:
					s.logger.Debug("Closing global ticker routine")
					return
				case <-ticker.C:
					if atomic.LoadUint64(detector.msgConsumed) == 0 {
						s.panicFunc(-1)
						return // For tests
					}
					atomic.StoreUint64(detector.msgConsumed, 0)
				}
			}
		}()
	}

	s.allPartitionsDeadlockDetector = detector
}

func (s *deadlockDetector) close() {
	if s.allPartitionsDeadlockDetector.disabled {
		return
	}
	s.logger.Debug("Closing all partitions deadlock detector")
	s.allPartitionsDeadlockDetector.done <- struct{}{}
}

func (s *allPartitionsDeadlockDetector) incrementMsgCount() {
	atomic.AddUint64(s.msgConsumed, 1)
}

func (w *partitionDeadlockDetector) closePartitionChannel() chan struct{} {
	return w.closePartition
}

func (w *partitionDeadlockDetector) close() {
	if w.disabled {
		return
	}
	w.logger.Debug("Closing deadlock detector", zap.Int32("partition", w.partition))
	w.done <- struct{}{}
}

func (w *partitionDeadlockDetector) incrementMsgCount() {
	w.incrementAllPartitionMsgCount()
	atomic.AddUint64(w.msgConsumed, 1)
}
<commit_msg>Clarify deadlock panic message (#2605)<commit_after>// Copyright (c) 2018 The Jaeger Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package consumer

import (
	"strconv"
	"sync/atomic"
	"time"

	"github.com/uber/jaeger-lib/metrics"
	"go.uber.org/zap"
)

// deadlockDetector monitors the messages consumed and wither signals for the partition to be closed by sending a
// message on closePartition, or triggers a panic if the close fails. It triggers a panic if there are no messages
// consumed across all partitions.
//
// Closing the partition should result in a rebalance, which alleviates the condition. This means that rebalances can
// happen frequently if there is no traffic on the Kafka topic. This shouldn't affect normal operations.
//
// If the message send isn't processed within the next check interval, a panic is issued.This hack relies on a
// container management system (k8s, aurora, marathon, etc) to reschedule
// the dead instance.
//
// This hack protects jaeger-ingester from issues described in  https://github.com/jaegertracing/jaeger/issues/1052
//
type deadlockDetector struct {
	metricsFactory                metrics.Factory
	logger                        *zap.Logger
	interval                      time.Duration
	allPartitionsDeadlockDetector *allPartitionsDeadlockDetector
	panicFunc                     func(int32)
}

type partitionDeadlockDetector struct {
	msgConsumed                   *uint64
	logger                        *zap.Logger
	partition                     int32
	closePartition                chan struct{}
	done                          chan struct{}
	incrementAllPartitionMsgCount func()
	disabled                      bool
}

type allPartitionsDeadlockDetector struct {
	msgConsumed *uint64
	logger      *zap.Logger
	done        chan struct{}
	disabled    bool
}

func newDeadlockDetector(metricsFactory metrics.Factory, logger *zap.Logger, interval time.Duration) deadlockDetector {
	panicFunc := func(partition int32) {
		metricsFactory.Counter(metrics.Options{Name: "deadlockdetector.panic-issued", Tags: map[string]string{"partition": strconv.Itoa(int(partition))}}).Inc(1)
		time.Sleep(time.Second) // Allow time to flush metric

		logger.Panic("No messages processed in the last check interval, possible deadlock, exiting. "+
			"This behavior can be disabled with --ingester.deadlockInterval=0 flag.",
			zap.Int32("partition", partition))
	}

	return deadlockDetector{
		metricsFactory: metricsFactory,
		logger:         logger,
		interval:       interval,
		panicFunc:      panicFunc,
	}
}

func (s *deadlockDetector) startMonitoringForPartition(partition int32) *partitionDeadlockDetector {
	var msgConsumed uint64
	w := &partitionDeadlockDetector{
		msgConsumed:    &msgConsumed,
		partition:      partition,
		closePartition: make(chan struct{}, 1),
		done:           make(chan struct{}),
		logger:         s.logger,
		disabled:       s.interval == 0,

		incrementAllPartitionMsgCount: func() {
			s.allPartitionsDeadlockDetector.incrementMsgCount()
		},
	}

	if w.disabled {
		s.logger.Debug("Partition deadlock detector disabled")
	} else {
		go s.monitorForPartition(w, partition)
	}

	return w
}

func (s *deadlockDetector) monitorForPartition(w *partitionDeadlockDetector, partition int32) {
	ticker := time.NewTicker(s.interval)
	defer ticker.Stop()

	for {
		select {
		case <-w.done:
			s.logger.Info("Closing ticker routine", zap.Int32("partition", partition))
			return
		case <-ticker.C:
			if atomic.LoadUint64(w.msgConsumed) == 0 {
				select {
				case w.closePartition <- struct{}{}:
					s.metricsFactory.Counter(metrics.Options{Name: "deadlockdetector.close-signalled", Tags: map[string]string{"partition": strconv.Itoa(int(partition))}}).Inc(1)
					s.logger.Warn("Signalling partition close due to inactivity", zap.Int32("partition", partition))
				default:
					// If closePartition is blocked, the consumer might have deadlocked - kill the process
					s.panicFunc(partition)
					return // For tests
				}
			} else {
				atomic.StoreUint64(w.msgConsumed, 0)
			}
		}
	}
}

// start monitors that the sum of messages consumed across all partitions is non zero for the given interval
// If it is zero when there are producers producing messages on the topic, it means that sarama-cluster hasn't
// retrieved partition assignments. (This case will not be caught by startMonitoringForPartition because no partitions
// were retrieved).
func (s *deadlockDetector) start() {
	var msgConsumed uint64
	detector := &allPartitionsDeadlockDetector{
		msgConsumed: &msgConsumed,
		done:        make(chan struct{}),
		logger:      s.logger,
		disabled:    s.interval == 0,
	}

	if detector.disabled {
		s.logger.Debug("Global deadlock detector disabled")
	} else {
		s.logger.Debug("Starting global deadlock detector")
		go func() {
			ticker := time.NewTicker(s.interval)
			defer ticker.Stop()

			for {
				select {
				case <-detector.done:
					s.logger.Debug("Closing global ticker routine")
					return
				case <-ticker.C:
					if atomic.LoadUint64(detector.msgConsumed) == 0 {
						s.panicFunc(-1)
						return // For tests
					}
					atomic.StoreUint64(detector.msgConsumed, 0)
				}
			}
		}()
	}

	s.allPartitionsDeadlockDetector = detector
}

func (s *deadlockDetector) close() {
	if s.allPartitionsDeadlockDetector.disabled {
		return
	}
	s.logger.Debug("Closing all partitions deadlock detector")
	s.allPartitionsDeadlockDetector.done <- struct{}{}
}

func (s *allPartitionsDeadlockDetector) incrementMsgCount() {
	atomic.AddUint64(s.msgConsumed, 1)
}

func (w *partitionDeadlockDetector) closePartitionChannel() chan struct{} {
	return w.closePartition
}

func (w *partitionDeadlockDetector) close() {
	if w.disabled {
		return
	}
	w.logger.Debug("Closing deadlock detector", zap.Int32("partition", w.partition))
	w.done <- struct{}{}
}

func (w *partitionDeadlockDetector) incrementMsgCount() {
	w.incrementAllPartitionMsgCount()
	atomic.AddUint64(w.msgConsumed, 1)
}
<|endoftext|>


<commit_before>package collectors

import (
	"github.com/StackExchange/tcollector/opentsdb"
	"github.com/StackExchange/wmi"
	"regexp"
	"strings"
)

func init() {
	collectors = append(collectors, &IntervalCollector{F: c_windows_processes})
}

// These are silly processes but exist on my machine, will need to update KMB
var processInclusions = regexp.MustCompile("chrome|powershell|tcollector")
var serviceInclusions = regexp.MustCompile("WinRM")

func c_windows_processes() opentsdb.MultiDataPoint {
	var dst []Win32_PerfRawData_PerfProc_Process
	var q = wmi.CreateQuery(&dst, `WHERE Name <> '_Total'`)
	err := queryWmi(q, &dst)
	if err != nil {
		l.Println("processes:", err)
		return nil
	}

	var svc_dst []Win32_Service
	var svc_q = wmi.CreateQuery(&svc_dst, `WHERE Name <> '_Total'`)
	err = queryWmi(svc_q, &svc_dst)
	if err != nil {
		l.Println("services:", err)
		return nil
	}

	var iis_dst []WorkerProcess
	iis_q := wmi.CreateQuery(&iis_dst, "")
	err = queryWmiNamespace(iis_q, &iis_dst, "root\\WebAdministration")
	if err != nil {
		l.Println("iis_worker:", err, "WQL Query: ", iis_q, "NameSpace", "root\\WebAdministration")
		return nil
	}

	var md opentsdb.MultiDataPoint
	for _, v := range dst {
		var name string
		service_match := false
		iis_match := false
		process_match := processInclusions.MatchString(v.Name)

		id := "0"

		if process_match {
			raw_name := strings.Split(v.Name, "#")
			name = raw_name[0]
			if len(raw_name) == 2 {
				id = raw_name[1]
			}
			// If you have a hash sign in your process name you don't deserve monitoring ;-)
			if len(raw_name) > 2 {
				continue
			}
		}

		// A Service match could "overwrite" a process match, but that is probably what we would want
		for _, svc := range svc_dst {
			if serviceInclusions.MatchString(svc.Name) {
				// It is possible the pid has gone and been reused, but I think this unlikely
				// And I'm not aware of an atomic join we could do anyways
				if svc.ProcessId == v.IDProcess {
					id = "0"
					service_match = true
					name = svc.Name
					break
				}
			}
		}

		for _, a_pool := range iis_dst {
			if a_pool.ProcessId == v.IDProcess {
				id = "0"
				iis_match = true
				name = strings.Join([]string{"iis", a_pool.AppPoolName}, "_")
				break
			}
		}

		if !(service_match || process_match || iis_match) {
			continue
		}

		Add(&md, "win.proc.elapsed_time", v.ElapsedTime, opentsdb.TagSet{"name": name, "id": id})
		Add(&md, "win.proc.handle_count", v.HandleCount, opentsdb.TagSet{"name": name, "id": id})
		Add(&md, "win.proc.io_bytes", v.IOOtherBytesPersec, opentsdb.TagSet{"name": name, "id": id, "type": "other"})
		Add(&md, "win.proc.io_operations", v.IOOtherOperationsPersec, opentsdb.TagSet{"name": name, "id": id, "type": "other"})
		Add(&md, "win.proc.io_bytes", v.IOReadBytesPersec, opentsdb.TagSet{"name": name, "id": id, "type": "read"})
		Add(&md, "win.proc.io_operations", v.IOReadOperationsPersec, opentsdb.TagSet{"name": name, "id": id, "type": "read"})
		Add(&md, "win.proc.io_bytes", v.IOWriteBytesPersec, opentsdb.TagSet{"name": name, "id": id, "type": "write"})
		Add(&md, "win.proc.io_operations", v.IOWriteOperationsPersec, opentsdb.TagSet{"name": name, "id": id, "type": "write"})
		Add(&md, "win.proc.mem.page_faults", v.PageFaultsPersec, opentsdb.TagSet{"name": name, "id": id})
		Add(&md, "win.proc.mem.pagefile_bytes", v.PageFileBytes, opentsdb.TagSet{"name": name, "id": id})
		Add(&md, "win.proc.mem.pagefile_bytes_peak", v.PageFileBytesPeak, opentsdb.TagSet{"name": name, "id": id})
		Add(&md, "win.proc.cpu", v.PercentPrivilegedTime, opentsdb.TagSet{"name": name, "id": id, "type": "privileged"})
		Add(&md, "win.proc.cpu_total", v.PercentProcessorTime, opentsdb.TagSet{"name": name, "id": id})
		Add(&md, "win.proc.cpu", v.PercentUserTime, opentsdb.TagSet{"name": name, "id": id, "type": "user"})
		Add(&md, "win.proc.mem.pool_nonpaged_bytes", v.PoolNonpagedBytes, opentsdb.TagSet{"name": name, "id": id})
		Add(&md, "win.proc.mem.pool_paged_bytes", v.PoolPagedBytes, opentsdb.TagSet{"name": name, "id": id})
		Add(&md, "win.proc.priority_base", v.PriorityBase, opentsdb.TagSet{"name": name, "id": id})
		Add(&md, "win.proc.private_bytes", v.PrivateBytes, opentsdb.TagSet{"name": name, "id": id})
		Add(&md, "win.proc.thread_count", v.ThreadCount, opentsdb.TagSet{"name": name, "id": id})
		Add(&md, "win.proc.mem.vm.bytes", v.VirtualBytes, opentsdb.TagSet{"name": name, "id": id})
		Add(&md, "win.proc.mem.vm.bytes_peak", v.VirtualBytesPeak, opentsdb.TagSet{"name": name, "id": id})
		Add(&md, "win.proc.mem.working_set", v.WorkingSet, opentsdb.TagSet{"name": name, "id": id})
		Add(&md, "win.proc.mem.working_set_peak", v.WorkingSetPeak, opentsdb.TagSet{"name": name, "id": id})
		Add(&md, "win.proc.mem.working_set_private", v.WorkingSetPrivate, opentsdb.TagSet{"name": name, "id": id})

	}
	return md
}

// Actually a CIM_StatisticalInformation Struct according to Reflection
type Win32_PerfRawData_PerfProc_Process struct {
	ElapsedTime             uint64
	HandleCount             uint32
	IDProcess               uint32
	IOOtherBytesPersec      uint64
	IOOtherOperationsPersec uint64
	IOReadBytesPersec       uint64
	IOReadOperationsPersec  uint64
	IOWriteBytesPersec      uint64
	IOWriteOperationsPersec uint64
	Name                    string
	PageFaultsPersec        uint32
	PageFileBytes           uint64
	PageFileBytesPeak       uint64
	PercentPrivilegedTime   uint64
	PercentProcessorTime    uint64
	PercentUserTime         uint64
	PoolNonpagedBytes       uint32
	PoolPagedBytes          uint32
	PriorityBase            uint32
	PrivateBytes            uint64
	ThreadCount             uint32
	VirtualBytes            uint64
	VirtualBytesPeak        uint64
	WorkingSet              uint64
	WorkingSetPeak          uint64
	WorkingSetPrivate       uint64
}

//Actually a Win32_BaseServce
type Win32_Service struct {
	Name      string
	ProcessId uint32
}

type WorkerProcess struct {
	AppPoolName string
	ProcessId   uint32
}
<commit_msg>cmd/scollector: Don't return when the webadministration namespace doesn't exist<commit_after>package collectors

import (
	"github.com/StackExchange/tcollector/opentsdb"
	"github.com/StackExchange/wmi"
	"regexp"
	"strings"
)

func init() {
	collectors = append(collectors, &IntervalCollector{F: c_windows_processes})
}

// These are silly processes but exist on my machine, will need to update KMB
var processInclusions = regexp.MustCompile("chrome|powershell|tcollector")
var serviceInclusions = regexp.MustCompile("WinRM")

func c_windows_processes() opentsdb.MultiDataPoint {
	var dst []Win32_PerfRawData_PerfProc_Process
	var q = wmi.CreateQuery(&dst, `WHERE Name <> '_Total'`)
	err := queryWmi(q, &dst)
	if err != nil {
		l.Println("processes:", err)
		return nil
	}

	var svc_dst []Win32_Service
	var svc_q = wmi.CreateQuery(&svc_dst, `WHERE Name <> '_Total'`)
	err = queryWmi(svc_q, &svc_dst)
	if err != nil {
		l.Println("services:", err)
		return nil
	}

	var iis_dst []WorkerProcess
	iis_q := wmi.CreateQuery(&iis_dst, "")
	err = queryWmiNamespace(iis_q, &iis_dst, "root\\WebAdministration")
	if err != nil {
		//Don't Return from this error since the name space might exist
		iis_dst = nil
	}

	var md opentsdb.MultiDataPoint
	for _, v := range dst {
		var name string
		service_match := false
		iis_match := false
		process_match := processInclusions.MatchString(v.Name)

		id := "0"

		if process_match {
			raw_name := strings.Split(v.Name, "#")
			name = raw_name[0]
			if len(raw_name) == 2 {
				id = raw_name[1]
			}
			// If you have a hash sign in your process name you don't deserve monitoring ;-)
			if len(raw_name) > 2 {
				continue
			}
		}

		// A Service match could "overwrite" a process match, but that is probably what we would want
		for _, svc := range svc_dst {
			if serviceInclusions.MatchString(svc.Name) {
				// It is possible the pid has gone and been reused, but I think this unlikely
				// And I'm not aware of an atomic join we could do anyways
				if svc.ProcessId == v.IDProcess {
					id = "0"
					service_match = true
					name = svc.Name
					break
				}
			}
		}

		for _, a_pool := range iis_dst {
			if a_pool.ProcessId == v.IDProcess {
				id = "0"
				iis_match = true
				name = strings.Join([]string{"iis", a_pool.AppPoolName}, "_")
				break
			}
		}

		if !(service_match || process_match || iis_match) {
			continue
		}

		Add(&md, "win.proc.elapsed_time", v.ElapsedTime, opentsdb.TagSet{"name": name, "id": id})
		Add(&md, "win.proc.handle_count", v.HandleCount, opentsdb.TagSet{"name": name, "id": id})
		Add(&md, "win.proc.io_bytes", v.IOOtherBytesPersec, opentsdb.TagSet{"name": name, "id": id, "type": "other"})
		Add(&md, "win.proc.io_operations", v.IOOtherOperationsPersec, opentsdb.TagSet{"name": name, "id": id, "type": "other"})
		Add(&md, "win.proc.io_bytes", v.IOReadBytesPersec, opentsdb.TagSet{"name": name, "id": id, "type": "read"})
		Add(&md, "win.proc.io_operations", v.IOReadOperationsPersec, opentsdb.TagSet{"name": name, "id": id, "type": "read"})
		Add(&md, "win.proc.io_bytes", v.IOWriteBytesPersec, opentsdb.TagSet{"name": name, "id": id, "type": "write"})
		Add(&md, "win.proc.io_operations", v.IOWriteOperationsPersec, opentsdb.TagSet{"name": name, "id": id, "type": "write"})
		Add(&md, "win.proc.mem.page_faults", v.PageFaultsPersec, opentsdb.TagSet{"name": name, "id": id})
		Add(&md, "win.proc.mem.pagefile_bytes", v.PageFileBytes, opentsdb.TagSet{"name": name, "id": id})
		Add(&md, "win.proc.mem.pagefile_bytes_peak", v.PageFileBytesPeak, opentsdb.TagSet{"name": name, "id": id})
		Add(&md, "win.proc.cpu", v.PercentPrivilegedTime, opentsdb.TagSet{"name": name, "id": id, "type": "privileged"})
		Add(&md, "win.proc.cpu_total", v.PercentProcessorTime, opentsdb.TagSet{"name": name, "id": id})
		Add(&md, "win.proc.cpu", v.PercentUserTime, opentsdb.TagSet{"name": name, "id": id, "type": "user"})
		Add(&md, "win.proc.mem.pool_nonpaged_bytes", v.PoolNonpagedBytes, opentsdb.TagSet{"name": name, "id": id})
		Add(&md, "win.proc.mem.pool_paged_bytes", v.PoolPagedBytes, opentsdb.TagSet{"name": name, "id": id})
		Add(&md, "win.proc.priority_base", v.PriorityBase, opentsdb.TagSet{"name": name, "id": id})
		Add(&md, "win.proc.private_bytes", v.PrivateBytes, opentsdb.TagSet{"name": name, "id": id})
		Add(&md, "win.proc.thread_count", v.ThreadCount, opentsdb.TagSet{"name": name, "id": id})
		Add(&md, "win.proc.mem.vm.bytes", v.VirtualBytes, opentsdb.TagSet{"name": name, "id": id})
		Add(&md, "win.proc.mem.vm.bytes_peak", v.VirtualBytesPeak, opentsdb.TagSet{"name": name, "id": id})
		Add(&md, "win.proc.mem.working_set", v.WorkingSet, opentsdb.TagSet{"name": name, "id": id})
		Add(&md, "win.proc.mem.working_set_peak", v.WorkingSetPeak, opentsdb.TagSet{"name": name, "id": id})
		Add(&md, "win.proc.mem.working_set_private", v.WorkingSetPrivate, opentsdb.TagSet{"name": name, "id": id})

	}
	return md
}

// Actually a CIM_StatisticalInformation Struct according to Reflection
type Win32_PerfRawData_PerfProc_Process struct {
	ElapsedTime             uint64
	HandleCount             uint32
	IDProcess               uint32
	IOOtherBytesPersec      uint64
	IOOtherOperationsPersec uint64
	IOReadBytesPersec       uint64
	IOReadOperationsPersec  uint64
	IOWriteBytesPersec      uint64
	IOWriteOperationsPersec uint64
	Name                    string
	PageFaultsPersec        uint32
	PageFileBytes           uint64
	PageFileBytesPeak       uint64
	PercentPrivilegedTime   uint64
	PercentProcessorTime    uint64
	PercentUserTime         uint64
	PoolNonpagedBytes       uint32
	PoolPagedBytes          uint32
	PriorityBase            uint32
	PrivateBytes            uint64
	ThreadCount             uint32
	VirtualBytes            uint64
	VirtualBytesPeak        uint64
	WorkingSet              uint64
	WorkingSetPeak          uint64
	WorkingSetPrivate       uint64
}

//Actually a Win32_BaseServce
type Win32_Service struct {
	Name      string
	ProcessId uint32
}

type WorkerProcess struct {
	AppPoolName string
	ProcessId   uint32
}
<|endoftext|>


<commit_before>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package core

import (
	"bytes"
	"crypto/rand"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/araddon/gou"
	"github.com/bmizerany/assert"
	"github.com/mattbaird/elastigo/api"
	"log"
	"strconv"
	"testing"
	"time"
)

//  go test -bench=".*"
//  go test -bench="Bulk"

var (
	buffers        = make([]*bytes.Buffer, 0)
	totalBytesSent int
	messageSets    int
)

func init() {
	flag.Parse()
	if testing.Verbose() {
		gou.SetupLogging("debug")
	}
}

// take two ints, compare, need to be within 5%
func CloseInt(a, b int) bool {
	c := float64(a) / float64(b)
	if c >= .95 && c <= 1.05 {
		return true
	}
	return false
}

func TestBulkIndexerBasic(t *testing.T) {
	InitTests(true)
	indexer := NewBulkIndexer(3)
	indexer.BulkSender = func(buf *bytes.Buffer) error {
		messageSets += 1
		totalBytesSent += buf.Len()
		buffers = append(buffers, buf)
		//		log.Printf("buffer:%s", string(buf.Bytes()))
		return BulkSend(buf)
	}
	done := make(chan bool)
	indexer.Run(done)

	date := time.Unix(1257894000, 0)
	data := map[string]interface{}{"name": "smurfs", "age": 22, "date": time.Unix(1257894000, 0).Format(time.RFC1123Z)}

	err := indexer.Index("users", "user", "1", "", &date, data, true)

	WaitFor(func() bool {
		return len(buffers) > 0
	}, 5)
	// part of request is url, so lets factor that in
	//totalBytesSent = totalBytesSent - len(*eshost)
	assert.T(t, len(buffers) == 1, fmt.Sprintf("Should have sent one operation but was %d", len(buffers)))
	assert.T(t, BulkErrorCt == 0 && err == nil, fmt.Sprintf("Should not have any errors. BulkErroCt: %v, err:%v", BulkErrorCt, err))
	expectedBytes := 167
	assert.T(t, totalBytesSent == expectedBytes, fmt.Sprintf("Should have sent %v bytes but was %v", expectedBytes, totalBytesSent))

	err = indexer.Index("users", "user", "2", "", nil, data, true)
	<-time.After(time.Millisecond * 10) // we need to wait for doc to hit send channel
	// this will test to ensure that Flush actually catches a doc
	indexer.Flush()
	totalBytesSent = totalBytesSent - len(*eshost)
	assert.T(t, err == nil, fmt.Sprintf("Should have nil error  =%v", err))
	assert.T(t, len(buffers) == 2, fmt.Sprintf("Should have another buffer ct=%d", len(buffers)))

	assert.T(t, BulkErrorCt == 0, fmt.Sprintf("Should not have any errors %d", BulkErrorCt))
	expectedBytes = 282 // with refresh
	assert.T(t, CloseInt(totalBytesSent, expectedBytes), fmt.Sprintf("Should have sent %v bytes but was %v", expectedBytes, totalBytesSent))

	done <- true
}

// currently broken in drone.io
func XXXTestBulkUpdate(t *testing.T) {
	InitTests(true)
	api.Port = "9200"
	indexer := NewBulkIndexer(3)
	indexer.BulkSender = func(buf *bytes.Buffer) error {
		messageSets += 1
		totalBytesSent += buf.Len()
		buffers = append(buffers, buf)
		return BulkSend(buf)
	}
	done := make(chan bool)
	indexer.Run(done)

	date := time.Unix(1257894000, 0)
	user := map[string]interface{}{
		"name": "smurfs", "age": 22, "date": time.Unix(1257894000, 0), "count": 1,
	}

	// Lets make sure the data is in the index ...
	_, err := Index("users", "user", "5", nil, user)

	// script and params
	data := map[string]interface{}{
		"script": "ctx._source.count += 2",
	}
	err = indexer.Update("users", "user", "5", "", &date, data, true)
	// So here's the deal. Flushing does seem to work, you just have to give the
	// channel a moment to recieve the message ...
	//	<- time.After(time.Millisecond * 20)
	//	indexer.Flush()
	done <- true

	WaitFor(func() bool {
		return len(buffers) > 0
	}, 5)

	assert.T(t, BulkErrorCt == 0 && err == nil, fmt.Sprintf("Should not have any errors, bulkErrorCt:%v, err:%v", BulkErrorCt, err))

	response, err := Get("users", "user", "5", nil)
	assert.T(t, err == nil, fmt.Sprintf("Should not have any errors  %v", err))
	newCount := response.Source.(map[string]interface{})["count"]
	assert.T(t, newCount.(float64) == 3,
		fmt.Sprintf("Should have update count: %#v ... %#v", response.Source.(map[string]interface{})["count"], response))
}

func TestBulkSmallBatch(t *testing.T) {
	InitTests(true)

	done := make(chan bool)

	date := time.Unix(1257894000, 0)
	data := map[string]interface{}{"name": "smurfs", "age": 22, "date": time.Unix(1257894000, 0)}

	// Now tests small batches
	indexersm := NewBulkIndexer(1)
	indexersm.BufferDelayMax = 100 * time.Millisecond
	indexersm.BulkMaxDocs = 2
	messageSets = 0
	indexersm.BulkSender = func(buf *bytes.Buffer) error {
		messageSets += 1
		return BulkSend(buf)
	}
	indexersm.Run(done)
	<-time.After(time.Millisecond * 20)

	indexersm.Index("users", "user", "2", "", &date, data, true)
	indexersm.Index("users", "user", "3", "", &date, data, true)
	indexersm.Index("users", "user", "4", "", &date, data, true)
	<-time.After(time.Millisecond * 200)
	//	indexersm.Flush()
	done <- true
	assert.T(t, messageSets == 2, fmt.Sprintf("Should have sent 2 message sets %d", messageSets))

}

func TestBulkErrors(t *testing.T) {
	// lets set a bad port, and hope we get a connection refused error?
	api.Port = "27845"
	defer func() {
		api.Port = "9200"
	}()
	BulkDelaySeconds = 1
	indexer := NewBulkIndexerErrors(10, 1)
	done := make(chan bool)
	indexer.Run(done)

	errorCt := 0
	go func() {
		for i := 0; i < 20; i++ {
			date := time.Unix(1257894000, 0)
			data := map[string]interface{}{"name": "smurfs", "age": 22, "date": time.Unix(1257894000, 0)}
			indexer.Index("users", "user", strconv.Itoa(i), "", &date, data, true)
		}
	}()
	var errBuf *ErrorBuffer
	for errBuf = range indexer.ErrorChannel {
		errorCt++
		break
	}
	if errBuf.Buf.Len() > 0 {
		gou.Debug(errBuf.Err)
	}
	assert.T(t, errorCt > 0, fmt.Sprintf("ErrorCt should be > 0 %d", errorCt))
	done <- true
}

/*
BenchmarkBulkSend	18:33:00 bulk_test.go:131: Sent 1 messages in 0 sets totaling 0 bytes
18:33:00 bulk_test.go:131: Sent 100 messages in 1 sets totaling 145889 bytes
18:33:01 bulk_test.go:131: Sent 10000 messages in 100 sets totaling 14608888 bytes
18:33:05 bulk_test.go:131: Sent 20000 messages in 99 sets totaling 14462790 bytes
   20000	    234526 ns/op

*/
func BenchmarkBulkSend(b *testing.B) {
	InitTests(true)
	b.StartTimer()
	totalBytes := 0
	sets := 0
	GlobalBulkIndexer.BulkSender = func(buf *bytes.Buffer) error {
		totalBytes += buf.Len()
		sets += 1
		//log.Println("got bulk")
		return BulkSend(buf)
	}
	for i := 0; i < b.N; i++ {
		about := make([]byte, 1000)
		rand.Read(about)
		data := map[string]interface{}{"name": "smurfs", "age": 22, "date": time.Unix(1257894000, 0), "about": about}
		IndexBulk("users", "user", strconv.Itoa(i), nil, data, true)
	}
	log.Printf("Sent %d messages in %d sets totaling %d bytes \n", b.N, sets, totalBytes)
	if BulkErrorCt != 0 {
		b.Fail()
	}
}

/*
TODO:  this should be faster than above

BenchmarkBulkSendBytes	18:33:05 bulk_test.go:169: Sent 1 messages in 0 sets totaling 0 bytes
18:33:05 bulk_test.go:169: Sent 100 messages in 2 sets totaling 292299 bytes
18:33:09 bulk_test.go:169: Sent 10000 messages in 99 sets totaling 14473800 bytes
   10000	    373529 ns/op

*/
func BenchmarkBulkSendBytes(b *testing.B) {
	InitTests(true)
	about := make([]byte, 1000)
	rand.Read(about)
	data := map[string]interface{}{"name": "smurfs", "age": 22, "date": time.Unix(1257894000, 0), "about": about}
	body, _ := json.Marshal(data)
	b.StartTimer()
	totalBytes := 0
	sets := 0
	GlobalBulkIndexer.BulkSender = func(buf *bytes.Buffer) error {
		totalBytes += buf.Len()
		sets += 1
		return BulkSend(buf)
	}
	for i := 0; i < b.N; i++ {
		IndexBulk("users", "user", strconv.Itoa(i), nil, body, true)
	}
	log.Printf("Sent %d messages in %d sets totaling %d bytes \n", b.N, sets, totalBytes)
	if BulkErrorCt != 0 {
		b.Fail()
	}
}
<commit_msg>comment out hanging test.<commit_after>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package core

import (
	"bytes"
	"crypto/rand"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/araddon/gou"
	"github.com/bmizerany/assert"
	"github.com/mattbaird/elastigo/api"
	"log"
	"strconv"
	"testing"
	"time"
)

//  go test -bench=".*"
//  go test -bench="Bulk"

var (
	buffers        = make([]*bytes.Buffer, 0)
	totalBytesSent int
	messageSets    int
)

func init() {
	flag.Parse()
	if testing.Verbose() {
		gou.SetupLogging("debug")
	}
}

// take two ints, compare, need to be within 5%
func CloseInt(a, b int) bool {
	c := float64(a) / float64(b)
	if c >= .95 && c <= 1.05 {
		return true
	}
	return false
}

func TestBulkIndexerBasic(t *testing.T) {
	InitTests(true)
	indexer := NewBulkIndexer(3)
	indexer.BulkSender = func(buf *bytes.Buffer) error {
		messageSets += 1
		totalBytesSent += buf.Len()
		buffers = append(buffers, buf)
		//		log.Printf("buffer:%s", string(buf.Bytes()))
		return BulkSend(buf)
	}
	done := make(chan bool)
	indexer.Run(done)

	date := time.Unix(1257894000, 0)
	data := map[string]interface{}{"name": "smurfs", "age": 22, "date": time.Unix(1257894000, 0).Format(time.RFC1123Z)}

	err := indexer.Index("users", "user", "1", "", &date, data, true)

	WaitFor(func() bool {
		return len(buffers) > 0
	}, 5)
	// part of request is url, so lets factor that in
	//totalBytesSent = totalBytesSent - len(*eshost)
	assert.T(t, len(buffers) == 1, fmt.Sprintf("Should have sent one operation but was %d", len(buffers)))
	assert.T(t, BulkErrorCt == 0 && err == nil, fmt.Sprintf("Should not have any errors. BulkErroCt: %v, err:%v", BulkErrorCt, err))
	expectedBytes := 167
	assert.T(t, totalBytesSent == expectedBytes, fmt.Sprintf("Should have sent %v bytes but was %v", expectedBytes, totalBytesSent))

	err = indexer.Index("users", "user", "2", "", nil, data, true)
	<-time.After(time.Millisecond * 10) // we need to wait for doc to hit send channel
	// this will test to ensure that Flush actually catches a doc
	indexer.Flush()
	totalBytesSent = totalBytesSent - len(*eshost)
	assert.T(t, err == nil, fmt.Sprintf("Should have nil error  =%v", err))
	assert.T(t, len(buffers) == 2, fmt.Sprintf("Should have another buffer ct=%d", len(buffers)))

	assert.T(t, BulkErrorCt == 0, fmt.Sprintf("Should not have any errors %d", BulkErrorCt))
	expectedBytes = 282 // with refresh
	assert.T(t, CloseInt(totalBytesSent, expectedBytes), fmt.Sprintf("Should have sent %v bytes but was %v", expectedBytes, totalBytesSent))

	done <- true
}

// currently broken in drone.io
func XXXTestBulkUpdate(t *testing.T) {
	InitTests(true)
	api.Port = "9200"
	indexer := NewBulkIndexer(3)
	indexer.BulkSender = func(buf *bytes.Buffer) error {
		messageSets += 1
		totalBytesSent += buf.Len()
		buffers = append(buffers, buf)
		return BulkSend(buf)
	}
	done := make(chan bool)
	indexer.Run(done)

	date := time.Unix(1257894000, 0)
	user := map[string]interface{}{
		"name": "smurfs", "age": 22, "date": time.Unix(1257894000, 0), "count": 1,
	}

	// Lets make sure the data is in the index ...
	_, err := Index("users", "user", "5", nil, user)

	// script and params
	data := map[string]interface{}{
		"script": "ctx._source.count += 2",
	}
	err = indexer.Update("users", "user", "5", "", &date, data, true)
	// So here's the deal. Flushing does seem to work, you just have to give the
	// channel a moment to recieve the message ...
	//	<- time.After(time.Millisecond * 20)
	//	indexer.Flush()
	done <- true

	WaitFor(func() bool {
		return len(buffers) > 0
	}, 5)

	assert.T(t, BulkErrorCt == 0 && err == nil, fmt.Sprintf("Should not have any errors, bulkErrorCt:%v, err:%v", BulkErrorCt, err))

	response, err := Get("users", "user", "5", nil)
	assert.T(t, err == nil, fmt.Sprintf("Should not have any errors  %v", err))
	newCount := response.Source.(map[string]interface{})["count"]
	assert.T(t, newCount.(float64) == 3,
		fmt.Sprintf("Should have update count: %#v ... %#v", response.Source.(map[string]interface{})["count"], response))
}

func TestBulkSmallBatch(t *testing.T) {
	InitTests(true)

	done := make(chan bool)

	date := time.Unix(1257894000, 0)
	data := map[string]interface{}{"name": "smurfs", "age": 22, "date": time.Unix(1257894000, 0)}

	// Now tests small batches
	indexersm := NewBulkIndexer(1)
	indexersm.BufferDelayMax = 100 * time.Millisecond
	indexersm.BulkMaxDocs = 2
	messageSets = 0
	indexersm.BulkSender = func(buf *bytes.Buffer) error {
		messageSets += 1
		return BulkSend(buf)
	}
	indexersm.Run(done)
	<-time.After(time.Millisecond * 20)

	indexersm.Index("users", "user", "2", "", &date, data, true)
	indexersm.Index("users", "user", "3", "", &date, data, true)
	indexersm.Index("users", "user", "4", "", &date, data, true)
	<-time.After(time.Millisecond * 200)
	//	indexersm.Flush()
	done <- true
	assert.T(t, messageSets == 2, fmt.Sprintf("Should have sent 2 message sets %d", messageSets))

}

func XXXTestBulkErrors(t *testing.T) {
	// lets set a bad port, and hope we get a connection refused error?
	api.Port = "27845"
	defer func() {
		api.Port = "9200"
	}()
	BulkDelaySeconds = 1
	indexer := NewBulkIndexerErrors(10, 1)
	done := make(chan bool)
	indexer.Run(done)
	errorCt := 0
	go func() {
		for i := 0; i < 20; i++ {
			date := time.Unix(1257894000, 0)
			data := map[string]interface{}{"name": "smurfs", "age": 22, "date": time.Unix(1257894000, 0)}
			indexer.Index("users", "user", strconv.Itoa(i), "", &date, data, true)
		}
	}()
	var errBuf *ErrorBuffer
	for errBuf = range indexer.ErrorChannel {
		errorCt++
		break
	}
	if errBuf.Buf.Len() > 0 {
		gou.Debug(errBuf.Err)
	}
	assert.T(t, errorCt > 0, fmt.Sprintf("ErrorCt should be > 0 %d", errorCt))
	done <- true
}

/*
BenchmarkBulkSend	18:33:00 bulk_test.go:131: Sent 1 messages in 0 sets totaling 0 bytes
18:33:00 bulk_test.go:131: Sent 100 messages in 1 sets totaling 145889 bytes
18:33:01 bulk_test.go:131: Sent 10000 messages in 100 sets totaling 14608888 bytes
18:33:05 bulk_test.go:131: Sent 20000 messages in 99 sets totaling 14462790 bytes
   20000	    234526 ns/op

*/
func BenchmarkBulkSend(b *testing.B) {
	InitTests(true)
	b.StartTimer()
	totalBytes := 0
	sets := 0
	GlobalBulkIndexer.BulkSender = func(buf *bytes.Buffer) error {
		totalBytes += buf.Len()
		sets += 1
		//log.Println("got bulk")
		return BulkSend(buf)
	}
	for i := 0; i < b.N; i++ {
		about := make([]byte, 1000)
		rand.Read(about)
		data := map[string]interface{}{"name": "smurfs", "age": 22, "date": time.Unix(1257894000, 0), "about": about}
		IndexBulk("users", "user", strconv.Itoa(i), nil, data, true)
	}
	log.Printf("Sent %d messages in %d sets totaling %d bytes \n", b.N, sets, totalBytes)
	if BulkErrorCt != 0 {
		b.Fail()
	}
}

/*
TODO:  this should be faster than above

BenchmarkBulkSendBytes	18:33:05 bulk_test.go:169: Sent 1 messages in 0 sets totaling 0 bytes
18:33:05 bulk_test.go:169: Sent 100 messages in 2 sets totaling 292299 bytes
18:33:09 bulk_test.go:169: Sent 10000 messages in 99 sets totaling 14473800 bytes
   10000	    373529 ns/op

*/
func BenchmarkBulkSendBytes(b *testing.B) {
	InitTests(true)
	about := make([]byte, 1000)
	rand.Read(about)
	data := map[string]interface{}{"name": "smurfs", "age": 22, "date": time.Unix(1257894000, 0), "about": about}
	body, _ := json.Marshal(data)
	b.StartTimer()
	totalBytes := 0
	sets := 0
	GlobalBulkIndexer.BulkSender = func(buf *bytes.Buffer) error {
		totalBytes += buf.Len()
		sets += 1
		return BulkSend(buf)
	}
	for i := 0; i < b.N; i++ {
		IndexBulk("users", "user", strconv.Itoa(i), nil, body, true)
	}
	log.Printf("Sent %d messages in %d sets totaling %d bytes \n", b.N, sets, totalBytes)
	if BulkErrorCt != 0 {
		b.Fail()
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package behaviors

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"testing"

	"gopkg.in/yaml.v2"
)

func TestValidate(t *testing.T) {
	var behaviorFiles []string

	err := filepath.Walk(".",
		func(path string, info os.FileInfo, err error) error {
			if err != nil {
				t.Errorf("%q", err.Error())
			}

			r, _ := regexp.Compile(".+.yaml$")
			if r.MatchString(path) {
				behaviorFiles = append(behaviorFiles, path)
			}
			return nil
		})
	if err != nil {
		t.Errorf("%q", err.Error())
	}

	for _, file := range behaviorFiles {
		validateSuite(file, t)
	}
}

func validateSuite(path string, t *testing.T) {
	var suite Suite
	yamlFile, err := ioutil.ReadFile(path)
	if err != nil {
		t.Errorf("%q", err.Error())
	}
	err = yaml.UnmarshalStrict(yamlFile, &suite)

	if err != nil {
		t.Errorf("%q", err.Error())
	}

	behaviorIDList := make(map[string]bool)

	for _, behavior := range suite.Behaviors {

		// Ensure no behavior IDs are duplicated
		if _, ok := behaviorIDList[behavior.ID]; ok {
			t.Errorf("Duplicate behavior ID: %s", behavior.ID)
		}
		behaviorIDList[behavior.ID] = true
	}
}
<commit_msg>Fix unmarshal for tests without behaviors<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package behaviors

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"testing"

	"gopkg.in/yaml.v2"
)

func TestValidate(t *testing.T) {
	var behaviorFiles []string

	err := filepath.Walk(".",
		func(path string, info os.FileInfo, err error) error {
			if err != nil {
				t.Errorf("%q", err.Error())
			}

			r, _ := regexp.Compile(".+.yaml$")
			if r.MatchString(path) {
				behaviorFiles = append(behaviorFiles, path)
			}
			return nil
		})
	if err != nil {
		t.Errorf("%q", err.Error())
	}

	for _, file := range behaviorFiles {
		validateSuite(file, t)
	}
}

func validateSuite(path string, t *testing.T) {
	var suite Suite
	yamlFile, err := ioutil.ReadFile(path)
	if err != nil {
		t.Errorf("%q", err.Error())
	}
	err = yaml.Unmarshal(yamlFile, &suite)

	if err != nil {
		t.Errorf("%q", err.Error())
	}

	behaviorIDList := make(map[string]bool)

	for _, behavior := range suite.Behaviors {

		// Ensure no behavior IDs are duplicated
		if _, ok := behaviorIDList[behavior.ID]; ok {
			t.Errorf("Duplicate behavior ID: %s", behavior.ID)
		}
		behaviorIDList[behavior.ID] = true
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package unversioned

const LabelZoneFailureDomain = "kubernetes.io/zone-failure-domain"
const LabelZoneRegion = "kubernetes.io/zone-region"
<commit_msg>Make zone and region node labels conformant. Fixes #17506<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package unversioned

const LabelZoneFailureDomain = "failure-domain.alpha.kubernetes.io/zone"
const LabelZoneRegion = "failure-domain.alpha.kubernetes.io/region"
<|endoftext|>


<commit_before>/*
Copyright AppsCode Inc. and Contributors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package statefulset

import (
	"fmt"

	"kubedb.dev/apimachinery/apis/kubedb"
	api "kubedb.dev/apimachinery/apis/kubedb/v1alpha2"
	db_cs "kubedb.dev/apimachinery/client/clientset/versioned"
	amc "kubedb.dev/apimachinery/pkg/controller"

	"github.com/appscode/go/log"
	apps "k8s.io/api/apps/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/dynamic"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/cache"
	core_util "kmodules.xyz/client-go/core/v1"
	"kmodules.xyz/client-go/tools/queue"
)

type Controller struct {
	*amc.Controller
	*amc.Config
}

func NewController(
	config *amc.Config,
	client kubernetes.Interface,
	dbClient db_cs.Interface,
	dmClient dynamic.Interface,
) *Controller {
	return &Controller{
		Controller: &amc.Controller{
			Client:        client,
			DBClient:      dbClient,
			DynamicClient: dmClient,
		},
		Config: config,
	}
}

func (c *Controller) InitStsWatcher() {
	log.Infoln("Initializing StatefulSet watcher.....")
	// Initialize RestoreSession Watcher
	c.StsInformer = c.KubeInformerFactory.Apps().V1().StatefulSets().Informer()
	c.StsQueue = queue.New(api.ResourceKindStatefulSet, c.MaxNumRequeues, c.NumThreads, c.processStatefulSet)
	c.StsLister = c.KubeInformerFactory.Apps().V1().StatefulSets().Lister()
	c.StsInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc: func(obj interface{}) {
			if sts, ok := obj.(*apps.StatefulSet); ok {
				c.enqueueOnlyKubeDBSts(sts)
			}
		},
		UpdateFunc: func(oldObj, newObj interface{}) {
			if sts, ok := newObj.(*apps.StatefulSet); ok {
				c.enqueueOnlyKubeDBSts(sts)
			}
		},
		DeleteFunc: func(obj interface{}) {
			if sts, ok := obj.(*apps.StatefulSet); ok {
				ok, _, err := core_util.IsOwnerOfGroup(metav1.GetControllerOf(sts), kubedb.GroupName)
				if err != nil || !ok {
					log.Warningln(err)
					return
				}
				dbInfo, err := c.extractDatabaseInfo(sts)
				if err != nil {
					log.Warningf("failed to extract database info from StatefulSet: %s/%s. Reason: %v", sts.Namespace, sts.Name, err)
					return
				}
				err = c.ensureReadyReplicasCond(dbInfo)
				if err != nil {
					log.Warningf("failed to update ReadyReplicas condition. Reason: %v", err)
					return
				}
			}
		},
	})
}

func (c *Controller) enqueueOnlyKubeDBSts(sts *apps.StatefulSet) {
	// only enqueue if the controlling owner is a KubeDB resource
	ok, _, err := core_util.IsOwnerOfGroup(metav1.GetControllerOf(sts), kubedb.GroupName)
	if err != nil {
		log.Warningln(err)
		return
	}
	if key, err := cache.MetaNamespaceKeyFunc(sts); ok && err == nil {
		queue.Enqueue(c.StsQueue.GetQueue(), key)
	}
}

func (c *Controller) processStatefulSet(key string) error {
	log.Infof("Started processing, key: %v", key)
	obj, exists, err := c.StsInformer.GetIndexer().GetByKey(key)
	if err != nil {
		log.Errorf("Fetching object with key %s from store failed with %v", key, err)
		return err
	}

	if !exists {
		log.Debugf("StatefulSet %s does not exist anymore", key)
	} else {
		sts := obj.(*apps.StatefulSet).DeepCopy()
		dbInfo, err := c.extractDatabaseInfo(sts)
		if err != nil {
			return fmt.Errorf("failed to extract database info from StatefulSet: %s/%s. Reason: %v", sts.Namespace, sts.Name, err)
		}
		return c.ensureReadyReplicasCond(dbInfo)
	}
	return nil
}
<commit_msg>Fix StatefulSet controller (#616)<commit_after>/*
Copyright AppsCode Inc. and Contributors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package statefulset

import (
	"fmt"

	"kubedb.dev/apimachinery/apis/kubedb"
	api "kubedb.dev/apimachinery/apis/kubedb/v1alpha2"
	db_cs "kubedb.dev/apimachinery/client/clientset/versioned"
	amc "kubedb.dev/apimachinery/pkg/controller"

	"github.com/appscode/go/log"
	apps "k8s.io/api/apps/v1"
	kerr "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/dynamic"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/cache"
	core_util "kmodules.xyz/client-go/core/v1"
	"kmodules.xyz/client-go/tools/queue"
)

type Controller struct {
	*amc.Controller
	*amc.Config
}

func NewController(
	config *amc.Config,
	client kubernetes.Interface,
	dbClient db_cs.Interface,
	dmClient dynamic.Interface,
) *Controller {
	return &Controller{
		Controller: &amc.Controller{
			Client:        client,
			DBClient:      dbClient,
			DynamicClient: dmClient,
		},
		Config: config,
	}
}

func (c *Controller) InitStsWatcher() {
	log.Infoln("Initializing StatefulSet watcher.....")
	// Initialize RestoreSession Watcher
	c.StsInformer = c.KubeInformerFactory.Apps().V1().StatefulSets().Informer()
	c.StsQueue = queue.New(api.ResourceKindStatefulSet, c.MaxNumRequeues, c.NumThreads, c.processStatefulSet)
	c.StsLister = c.KubeInformerFactory.Apps().V1().StatefulSets().Lister()
	c.StsInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc: func(obj interface{}) {
			if sts, ok := obj.(*apps.StatefulSet); ok {
				c.enqueueOnlyKubeDBSts(sts)
			}
		},
		UpdateFunc: func(oldObj, newObj interface{}) {
			if sts, ok := newObj.(*apps.StatefulSet); ok {
				c.enqueueOnlyKubeDBSts(sts)
			}
		},
		DeleteFunc: func(obj interface{}) {
			if sts, ok := obj.(*apps.StatefulSet); ok {
				ok, _, err := core_util.IsOwnerOfGroup(metav1.GetControllerOf(sts), kubedb.GroupName)
				if err != nil || !ok {
					log.Warningln(err)
					return
				}
				dbInfo, err := c.extractDatabaseInfo(sts)
				if err != nil {
					if !kerr.IsNotFound(err) {
						log.Warningf("failed to extract database info from StatefulSet: %s/%s. Reason: %v", sts.Namespace, sts.Name, err)
					}
					return
				}
				err = c.ensureReadyReplicasCond(dbInfo)
				if err != nil {
					log.Warningf("failed to update ReadyReplicas condition. Reason: %v", err)
					return
				}
			}
		},
	})
}

func (c *Controller) enqueueOnlyKubeDBSts(sts *apps.StatefulSet) {
	// only enqueue if the controlling owner is a KubeDB resource
	ok, _, err := core_util.IsOwnerOfGroup(metav1.GetControllerOf(sts), kubedb.GroupName)
	if err != nil {
		log.Warningf("failed to enqueue StatefulSet: %s/%s. Reason: %v", sts.Namespace, sts.Name, err)
		return
	}
	if ok {
		queue.Enqueue(c.StsQueue.GetQueue(), cache.ExplicitKey(sts.Namespace+"/"+sts.Name))
	}
}

func (c *Controller) processStatefulSet(key string) error {
	log.Infof("Started processing, key: %v", key)
	obj, exists, err := c.StsInformer.GetIndexer().GetByKey(key)
	if err != nil {
		log.Errorf("Fetching object with key %s from store failed with %v", key, err)
		return err
	}

	if !exists {
		log.Debugf("StatefulSet %s does not exist anymore", key)
	} else {
		sts := obj.(*apps.StatefulSet).DeepCopy()
		dbInfo, err := c.extractDatabaseInfo(sts)
		if err != nil {
			return fmt.Errorf("failed to extract database info from StatefulSet: %s/%s. Reason: %v", sts.Namespace, sts.Name, err)
		}
		return c.ensureReadyReplicasCond(dbInfo)
	}
	return nil
}
<|endoftext|>


<commit_before>package config

import (
	"context"
	"encoding/json"
	"fmt"
	"path"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/bmatcuk/doublestar"
)

func NewConfig(ctx context.Context, fileReader fileReader) (c Config, err error) {
	exist, err := fileReader.IsGiterminismConfigExistAnywhere(ctx)
	if err != nil {
		return c, err
	}

	if !exist {
		return Config{}, nil
	}

	data, err := fileReader.ReadGiterminismConfig(ctx)
	if err != nil {
		return c, err
	}

	err = processWithOpenAPISchema(&data)
	if err != nil {
		return c, fmt.Errorf("the giterminism config validation failed: %s", err)
	}

	if err := json.Unmarshal(data, &c); err != nil {
		panic(fmt.Sprint("unexpected error: ", err))
	}

	return c, err
}

type fileReader interface {
	IsGiterminismConfigExistAnywhere(ctx context.Context) (bool, error)
	ReadGiterminismConfig(ctx context.Context) ([]byte, error)
}

type Config struct {
	Config config `json:"config"`
	Helm   helm   `json:"helm"`
}

func (c Config) IsUncommittedConfigAccepted() bool {
	return c.Config.AllowUncommitted
}

func (c Config) IsUncommittedConfigTemplateFileAccepted(path string) (bool, error) {
	return c.Config.IsUncommittedTemplateFileAccepted(path)
}

func (c Config) IsUncommittedConfigGoTemplateRenderingFileAccepted(path string) (bool, error) {
	return c.Config.GoTemplateRendering.IsUncommittedFileAccepted(path)
}

func (c Config) IsConfigGoTemplateRenderingEnvNameAccepted(envName string) (bool, error) {
	return c.Config.GoTemplateRendering.IsEnvNameAccepted(envName)
}

func (c Config) IsConfigStapelFromLatestAccepted() bool {
	return c.Config.Stapel.AllowFromLatest
}

func (c Config) IsConfigStapelGitBranchAccepted() bool {
	return c.Config.Stapel.Git.AllowBranch
}

func (c Config) IsConfigStapelMountBuildDirAccepted() bool {
	return c.Config.Stapel.Mount.AllowBuildDir
}

func (c Config) IsConfigStapelMountFromPathAccepted(fromPath string) (bool, error) {
	return c.Config.Stapel.Mount.IsFromPathAccepted(fromPath)
}

func (c Config) IsConfigDockerfileContextAddFileAccepted(relPath string) (bool, error) {
	return c.Config.Dockerfile.IsContextAddFileAccepted(relPath)
}

func (c Config) IsUncommittedDockerfileAccepted(relPath string) (bool, error) {
	return c.Config.Dockerfile.IsUncommittedAccepted(relPath)
}

func (c Config) IsUncommittedDockerignoreAccepted(relPath string) (bool, error) {
	return c.Config.Dockerfile.IsUncommittedDockerignoreAccepted(relPath)
}

func (c Config) IsUncommittedHelmFileAccepted(relPath string) (bool, error) {
	return c.Helm.IsUncommittedHelmFileAccepted(relPath)
}

type config struct {
	AllowUncommitted          bool                `json:"allowUncommitted"`
	AllowUncommittedTemplates []string            `json:"allowUncommittedTemplates"`
	GoTemplateRendering       goTemplateRendering `json:"goTemplateRendering"`
	Stapel                    stapel              `json:"stapel"`
	Dockerfile                dockerfile          `json:"dockerfile"`
}

func (c config) IsUncommittedTemplateFileAccepted(path string) (bool, error) {
	return isPathMatched(c.AllowUncommittedTemplates, path)
}

type goTemplateRendering struct {
	AllowEnvVariables     []string `json:"allowEnvVariables"`
	AllowUncommittedFiles []string `json:"allowUncommittedFiles"`
}

func (r goTemplateRendering) IsEnvNameAccepted(name string) (bool, error) {
	for _, pattern := range r.AllowEnvVariables {
		if strings.HasPrefix(pattern, "/") && strings.HasSuffix(pattern, "/") {
			expr := fmt.Sprintf("^%s$", pattern[1:len(pattern)-1])
			r, err := regexp.Compile(expr)
			if err != nil {
				return false, err
			}

			return r.MatchString(name), nil
		} else {
			return pattern == name, nil
		}
	}

	return false, nil
}

func (r goTemplateRendering) IsUncommittedFileAccepted(path string) (bool, error) {
	return isPathMatched(r.AllowUncommittedFiles, path)
}

type stapel struct {
	AllowFromLatest bool  `json:"allowFromLatest"`
	Git             git   `json:"git"`
	Mount           mount `json:"mount"`
}

type git struct {
	AllowBranch bool `json:"allowBranch"`
}

type mount struct {
	AllowBuildDir  bool     `json:"allowBuildDir"`
	AllowFromPaths []string `json:"allowFromPaths"`
}

func (m mount) IsFromPathAccepted(path string) (bool, error) {
	return isPathMatched(m.AllowFromPaths, path)
}

type dockerfile struct {
	AllowUncommitted                  []string `json:"allowUncommitted"`
	AllowUncommittedDockerignoreFiles []string `json:"allowUncommittedDockerignoreFiles"`
	AllowContextAddFiles              []string `json:"allowContextAddFiles"`
}

func (d dockerfile) IsContextAddFileAccepted(path string) (bool, error) {
	return isPathMatched(d.AllowContextAddFiles, path)
}

func (d dockerfile) IsUncommittedAccepted(path string) (bool, error) {
	return isPathMatched(d.AllowUncommitted, path)
}

func (d dockerfile) IsUncommittedDockerignoreAccepted(path string) (bool, error) {
	return isPathMatched(d.AllowUncommittedDockerignoreFiles, path)
}

type helm struct {
	AllowUncommittedFiles []string `json:"allowUncommittedFiles"`
}

func (h helm) IsUncommittedHelmFileAccepted(path string) (bool, error) {
	return isPathMatched(h.AllowUncommittedFiles, path)
}

func isPathMatched(patterns []string, p string) (bool, error) {
	p = filepath.ToSlash(p)
	for _, pattern := range patterns {
		pattern = filepath.ToSlash(pattern)

		matchFunc := func() (bool, error) {
			exist, err := doublestar.Match(pattern, p)
			if err != nil {
				return false, err
			}

			if exist {
				return true, nil
			}

			return doublestar.Match(path.Join(pattern, "**", "*"), p)
		}

		if matched, err := matchFunc(); err != nil {
			return false, fmt.Errorf("unable to match path (pattern: %q, path %q): %s", pattern, p, err)
		} else if matched {
			return true, nil
		}
	}

	return false, nil
}
<commit_msg>[giterminism] Fix env allowance check<commit_after>package config

import (
	"context"
	"encoding/json"
	"fmt"
	"path"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/bmatcuk/doublestar"
)

func NewConfig(ctx context.Context, fileReader fileReader) (c Config, err error) {
	exist, err := fileReader.IsGiterminismConfigExistAnywhere(ctx)
	if err != nil {
		return c, err
	}

	if !exist {
		return Config{}, nil
	}

	data, err := fileReader.ReadGiterminismConfig(ctx)
	if err != nil {
		return c, err
	}

	err = processWithOpenAPISchema(&data)
	if err != nil {
		return c, fmt.Errorf("the giterminism config validation failed: %s", err)
	}

	if err := json.Unmarshal(data, &c); err != nil {
		panic(fmt.Sprint("unexpected error: ", err))
	}

	return c, err
}

type fileReader interface {
	IsGiterminismConfigExistAnywhere(ctx context.Context) (bool, error)
	ReadGiterminismConfig(ctx context.Context) ([]byte, error)
}

type Config struct {
	Config config `json:"config"`
	Helm   helm   `json:"helm"`
}

func (c Config) IsUncommittedConfigAccepted() bool {
	return c.Config.AllowUncommitted
}

func (c Config) IsUncommittedConfigTemplateFileAccepted(path string) (bool, error) {
	return c.Config.IsUncommittedTemplateFileAccepted(path)
}

func (c Config) IsUncommittedConfigGoTemplateRenderingFileAccepted(path string) (bool, error) {
	return c.Config.GoTemplateRendering.IsUncommittedFileAccepted(path)
}

func (c Config) IsConfigGoTemplateRenderingEnvNameAccepted(envName string) (bool, error) {
	return c.Config.GoTemplateRendering.IsEnvNameAccepted(envName)
}

func (c Config) IsConfigStapelFromLatestAccepted() bool {
	return c.Config.Stapel.AllowFromLatest
}

func (c Config) IsConfigStapelGitBranchAccepted() bool {
	return c.Config.Stapel.Git.AllowBranch
}

func (c Config) IsConfigStapelMountBuildDirAccepted() bool {
	return c.Config.Stapel.Mount.AllowBuildDir
}

func (c Config) IsConfigStapelMountFromPathAccepted(fromPath string) (bool, error) {
	return c.Config.Stapel.Mount.IsFromPathAccepted(fromPath)
}

func (c Config) IsConfigDockerfileContextAddFileAccepted(relPath string) (bool, error) {
	return c.Config.Dockerfile.IsContextAddFileAccepted(relPath)
}

func (c Config) IsUncommittedDockerfileAccepted(relPath string) (bool, error) {
	return c.Config.Dockerfile.IsUncommittedAccepted(relPath)
}

func (c Config) IsUncommittedDockerignoreAccepted(relPath string) (bool, error) {
	return c.Config.Dockerfile.IsUncommittedDockerignoreAccepted(relPath)
}

func (c Config) IsUncommittedHelmFileAccepted(relPath string) (bool, error) {
	return c.Helm.IsUncommittedHelmFileAccepted(relPath)
}

type config struct {
	AllowUncommitted          bool                `json:"allowUncommitted"`
	AllowUncommittedTemplates []string            `json:"allowUncommittedTemplates"`
	GoTemplateRendering       goTemplateRendering `json:"goTemplateRendering"`
	Stapel                    stapel              `json:"stapel"`
	Dockerfile                dockerfile          `json:"dockerfile"`
}

func (c config) IsUncommittedTemplateFileAccepted(path string) (bool, error) {
	return isPathMatched(c.AllowUncommittedTemplates, path)
}

type goTemplateRendering struct {
	AllowEnvVariables     []string `json:"allowEnvVariables"`
	AllowUncommittedFiles []string `json:"allowUncommittedFiles"`
}

func (r goTemplateRendering) IsEnvNameAccepted(name string) (bool, error) {
	for _, pattern := range r.AllowEnvVariables {
		match, err := func() (bool, error) {
			if strings.HasPrefix(pattern, "/") && strings.HasSuffix(pattern, "/") {
				expr := fmt.Sprintf("^%s$", pattern[1:len(pattern)-1])
				r, err := regexp.Compile(expr)
				if err != nil {
					return false, err
				}

				return r.MatchString(name), nil
			} else {
				return pattern == name, nil
			}
		}()

		if err != nil {
			return false, err
		}

		if match {
			return true, nil
		}
	}

	return false, nil
}

func (r goTemplateRendering) IsUncommittedFileAccepted(path string) (bool, error) {
	return isPathMatched(r.AllowUncommittedFiles, path)
}

type stapel struct {
	AllowFromLatest bool  `json:"allowFromLatest"`
	Git             git   `json:"git"`
	Mount           mount `json:"mount"`
}

type git struct {
	AllowBranch bool `json:"allowBranch"`
}

type mount struct {
	AllowBuildDir  bool     `json:"allowBuildDir"`
	AllowFromPaths []string `json:"allowFromPaths"`
}

func (m mount) IsFromPathAccepted(path string) (bool, error) {
	return isPathMatched(m.AllowFromPaths, path)
}

type dockerfile struct {
	AllowUncommitted                  []string `json:"allowUncommitted"`
	AllowUncommittedDockerignoreFiles []string `json:"allowUncommittedDockerignoreFiles"`
	AllowContextAddFiles              []string `json:"allowContextAddFiles"`
}

func (d dockerfile) IsContextAddFileAccepted(path string) (bool, error) {
	return isPathMatched(d.AllowContextAddFiles, path)
}

func (d dockerfile) IsUncommittedAccepted(path string) (bool, error) {
	return isPathMatched(d.AllowUncommitted, path)
}

func (d dockerfile) IsUncommittedDockerignoreAccepted(path string) (bool, error) {
	return isPathMatched(d.AllowUncommittedDockerignoreFiles, path)
}

type helm struct {
	AllowUncommittedFiles []string `json:"allowUncommittedFiles"`
}

func (h helm) IsUncommittedHelmFileAccepted(path string) (bool, error) {
	return isPathMatched(h.AllowUncommittedFiles, path)
}

func isPathMatched(patterns []string, p string) (bool, error) {
	p = filepath.ToSlash(p)
	for _, pattern := range patterns {
		pattern = filepath.ToSlash(pattern)

		matchFunc := func() (bool, error) {
			exist, err := doublestar.Match(pattern, p)
			if err != nil {
				return false, err
			}

			if exist {
				return true, nil
			}

			return doublestar.Match(path.Join(pattern, "**", "*"), p)
		}

		if matched, err := matchFunc(); err != nil {
			return false, fmt.Errorf("unable to match path (pattern: %q, path %q): %s", pattern, p, err)
		} else if matched {
			return true, nil
		}
	}

	return false, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2012-2019 Eli Janssen
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

package htrie

import (
	"strings"

	"github.com/xlab/treeprint"
)

func (gpn *globPathNode) printTree(stree treeprint.Tree) {
	for i, x := range gpn.subtrees {
		if x == nil {
			continue
		}
		c := "*"
		if i != 0 {
			c = string(i)
		}

		subTree := stree.AddBranch(c)
		meta := make([]string, 0)
		if x.isGlob {
			meta = append(meta, "glob")
		}
		if x.hasGlobChild {
			meta = append(meta, "glob-child")
		}
		if x.canMatch {
			meta = append(meta, "$")
		}
		if len(meta) > 0 {
			subTree.SetMetaValue(strings.Join(meta, ","))
		}

		x.printTree(subTree)
	}
}

func (gpn *globPathNode) RenderTree() string {
	tree := treeprint.New()

	meta := make([]string, 0)
	if gpn.isGlob {
		meta = append(meta, "glob")
	}
	if gpn.hasGlobChild {
		meta = append(meta, "glob-child")
	}
	if gpn.canMatch {
		meta = append(meta, "$")
	}
	if len(meta) > 0 {
		tree.SetMetaValue(strings.Join(meta, ","))
	}

	gpn.printTree(tree)
	return tree.String()
}
<commit_msg>update pretty printing to pass new vet check<commit_after>// Copyright (c) 2012-2019 Eli Janssen
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

package htrie

import (
	"strings"

	"github.com/xlab/treeprint"
)

func (gpn *globPathNode) printTree(stree treeprint.Tree) {
	for i, x := range gpn.subtrees {
		if x == nil {
			continue
		}
		c := "*"
		if i != 0 {
			// we use uint32 for performance, and don't care about
			// truncation at all here (just printing anyway), so
			// just convert.
			c = string(uint8(i))
		}

		subTree := stree.AddBranch(c)
		meta := make([]string, 0)
		if x.isGlob {
			meta = append(meta, "glob")
		}
		if x.hasGlobChild {
			meta = append(meta, "glob-child")
		}
		if x.canMatch {
			meta = append(meta, "$")
		}
		if len(meta) > 0 {
			subTree.SetMetaValue(strings.Join(meta, ","))
		}

		x.printTree(subTree)
	}
}

func (gpn *globPathNode) RenderTree() string {
	tree := treeprint.New()

	meta := make([]string, 0)
	if gpn.isGlob {
		meta = append(meta, "glob")
	}
	if gpn.hasGlobChild {
		meta = append(meta, "glob-child")
	}
	if gpn.canMatch {
		meta = append(meta, "$")
	}
	if len(meta) > 0 {
		tree.SetMetaValue(strings.Join(meta, ","))
	}

	gpn.printTree(tree)
	return tree.String()
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package options

import (
	"fmt"
	"net"
	"time"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/serializer"
	"k8s.io/apiserver/pkg/server"
	utilfeature "k8s.io/apiserver/pkg/util/feature"

	"github.com/spf13/pflag"
)

// ServerRunOptions contains the options while running a generic api server.
type ServerRunOptions struct {
	AdvertiseAddress net.IP

	CorsAllowedOriginList       []string
	ExternalHost                string
	MaxRequestsInFlight         int
	MaxMutatingRequestsInFlight int
	RequestTimeout              time.Duration
	GoawayChance                float64
	LivezGracePeriod            time.Duration
	MinRequestTimeout           int
	ShutdownDelayDuration       time.Duration
	// We intentionally did not add a flag for this option. Users of the
	// apiserver library can wire it to a flag.
	JSONPatchMaxCopyBytes int64
	// The limit on the request body size that would be accepted and
	// decoded in a write request. 0 means no limit.
	// We intentionally did not add a flag for this option. Users of the
	// apiserver library can wire it to a flag.
	MaxRequestBodyBytes       int64
	TargetRAMMB               int
	EnablePriorityAndFairness bool
}

func NewServerRunOptions() *ServerRunOptions {
	defaults := server.NewConfig(serializer.CodecFactory{})
	return &ServerRunOptions{
		MaxRequestsInFlight:         defaults.MaxRequestsInFlight,
		MaxMutatingRequestsInFlight: defaults.MaxMutatingRequestsInFlight,
		RequestTimeout:              defaults.RequestTimeout,
		LivezGracePeriod:            defaults.LivezGracePeriod,
		MinRequestTimeout:           defaults.MinRequestTimeout,
		ShutdownDelayDuration:       defaults.ShutdownDelayDuration,
		JSONPatchMaxCopyBytes:       defaults.JSONPatchMaxCopyBytes,
		MaxRequestBodyBytes:         defaults.MaxRequestBodyBytes,
		EnablePriorityAndFairness:   true,
	}
}

// ApplyOptions applies the run options to the method receiver and returns self
func (s *ServerRunOptions) ApplyTo(c *server.Config) error {
	c.CorsAllowedOriginList = s.CorsAllowedOriginList
	c.ExternalAddress = s.ExternalHost
	c.MaxRequestsInFlight = s.MaxRequestsInFlight
	c.MaxMutatingRequestsInFlight = s.MaxMutatingRequestsInFlight
	c.LivezGracePeriod = s.LivezGracePeriod
	c.RequestTimeout = s.RequestTimeout
	c.GoawayChance = s.GoawayChance
	c.MinRequestTimeout = s.MinRequestTimeout
	c.ShutdownDelayDuration = s.ShutdownDelayDuration
	c.JSONPatchMaxCopyBytes = s.JSONPatchMaxCopyBytes
	c.MaxRequestBodyBytes = s.MaxRequestBodyBytes
	c.PublicAddress = s.AdvertiseAddress

	return nil
}

// DefaultAdvertiseAddress sets the field AdvertiseAddress if unset. The field will be set based on the SecureServingOptions.
func (s *ServerRunOptions) DefaultAdvertiseAddress(secure *SecureServingOptions) error {
	if secure == nil {
		return nil
	}

	if s.AdvertiseAddress == nil || s.AdvertiseAddress.IsUnspecified() {
		hostIP, err := secure.DefaultExternalAddress()
		if err != nil {
			return fmt.Errorf("Unable to find suitable network address.error='%v'. "+
				"Try to set the AdvertiseAddress directly or provide a valid BindAddress to fix this.", err)
		}
		s.AdvertiseAddress = hostIP
	}

	return nil
}

// Validate checks validation of ServerRunOptions
func (s *ServerRunOptions) Validate() []error {
	errors := []error{}
	if s.TargetRAMMB < 0 {
		errors = append(errors, fmt.Errorf("--target-ram-mb can not be negative value"))
	}

	if s.LivezGracePeriod < 0 {
		errors = append(errors, fmt.Errorf("--livez-grace-period can not be a negative value"))
	}

	if s.MaxRequestsInFlight < 0 {
		errors = append(errors, fmt.Errorf("--max-requests-inflight can not be negative value"))
	}
	if s.MaxMutatingRequestsInFlight < 0 {
		errors = append(errors, fmt.Errorf("--max-mutating-requests-inflight can not be negative value"))
	}

	if s.RequestTimeout.Nanoseconds() < 0 {
		errors = append(errors, fmt.Errorf("--request-timeout can not be negative value"))
	}

	if s.GoawayChance < 0 || s.GoawayChance > 0.02 {
		errors = append(errors, fmt.Errorf("--goaway-chance can not be less than 0 or greater than 0.02"))
	}

	if s.MinRequestTimeout < 0 {
		errors = append(errors, fmt.Errorf("--min-request-timeout can not be negative value"))
	}

	if s.ShutdownDelayDuration < 0 {
		errors = append(errors, fmt.Errorf("--shutdown-delay-duration can not be negative value"))
	}

	if s.JSONPatchMaxCopyBytes < 0 {
		errors = append(errors, fmt.Errorf("--json-patch-max-copy-bytes can not be negative value"))
	}

	if s.MaxRequestBodyBytes < 0 {
		errors = append(errors, fmt.Errorf("--max-resource-write-bytes can not be negative value"))
	}

	return errors
}

// AddUniversalFlags adds flags for a specific APIServer to the specified FlagSet
func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
	// Note: the weird ""+ in below lines seems to be the only way to get gofmt to
	// arrange these text blocks sensibly. Grrr.

	fs.IPVar(&s.AdvertiseAddress, "advertise-address", s.AdvertiseAddress, ""+
		"The IP address on which to advertise the apiserver to members of the cluster. This "+
		"address must be reachable by the rest of the cluster. If blank, the --bind-address "+
		"will be used. If --bind-address is unspecified, the host's default interface will "+
		"be used.")

	fs.StringSliceVar(&s.CorsAllowedOriginList, "cors-allowed-origins", s.CorsAllowedOriginList, ""+
		"List of allowed origins for CORS, comma separated.  An allowed origin can be a regular "+
		"expression to support subdomain matching. If this list is empty CORS will not be enabled.")

	fs.IntVar(&s.TargetRAMMB, "target-ram-mb", s.TargetRAMMB,
		"Memory limit for apiserver in MB (used to configure sizes of caches, etc.)")

	fs.StringVar(&s.ExternalHost, "external-hostname", s.ExternalHost,
		"The hostname to use when generating externalized URLs for this master (e.g. Swagger API Docs or OpenID Discovery).")

	deprecatedMasterServiceNamespace := metav1.NamespaceDefault
	fs.StringVar(&deprecatedMasterServiceNamespace, "master-service-namespace", deprecatedMasterServiceNamespace, ""+
		"DEPRECATED: the namespace from which the Kubernetes master services should be injected into pods.")

	fs.IntVar(&s.MaxRequestsInFlight, "max-requests-inflight", s.MaxRequestsInFlight, ""+
		"The maximum number of non-mutating requests in flight at a given time. When the server exceeds this, "+
		"it rejects requests. Zero for no limit.")

	fs.IntVar(&s.MaxMutatingRequestsInFlight, "max-mutating-requests-inflight", s.MaxMutatingRequestsInFlight, ""+
		"The maximum number of mutating requests in flight at a given time. When the server exceeds this, "+
		"it rejects requests. Zero for no limit.")

	fs.DurationVar(&s.RequestTimeout, "request-timeout", s.RequestTimeout, ""+
		"An optional field indicating the duration a handler must keep a request open before timing "+
		"it out. This is the default request timeout for requests but may be overridden by flags such as "+
		"--min-request-timeout for specific types of requests.")

	fs.Float64Var(&s.GoawayChance, "goaway-chance", s.GoawayChance, ""+
		"To prevent HTTP/2 clients from getting stuck on a single apiserver, randomly close a connection (GOAWAY). "+
		"The client's other in-flight requests won't be affected, and the client will reconnect, likely landing on a different apiserver after going through the load balancer again. "+
		"This argument sets the fraction of requests that will be sent a GOAWAY. Clusters with single apiservers, or which don't use a load balancer, should NOT enable this. "+
		"Min is 0 (off), Max is .02 (1/50 requests); .001 (1/1000) is a recommended starting point.")

	fs.DurationVar(&s.LivezGracePeriod, "livez-grace-period", s.LivezGracePeriod, ""+
		"This option represents the maximum amount of time it should take for apiserver to complete its startup sequence "+
		"and become live. From apiserver's start time to when this amount of time has elapsed, /livez will assume "+
		"that unfinished post-start hooks will complete successfully and therefore return true.")

	fs.IntVar(&s.MinRequestTimeout, "min-request-timeout", s.MinRequestTimeout, ""+
		"An optional field indicating the minimum number of seconds a handler must keep "+
		"a request open before timing it out. Currently only honored by the watch request "+
		"handler, which picks a randomized value above this number as the connection timeout, "+
		"to spread out load.")

	fs.BoolVar(&s.EnablePriorityAndFairness, "enable-priority-and-fairness", s.EnablePriorityAndFairness, ""+
		"If true and the APIPriorityAndFairness feature gate is enabled, replace the max-in-flight handler with an enhanced one that queues and dispatches with priority and fairness")

	fs.DurationVar(&s.ShutdownDelayDuration, "shutdown-delay-duration", s.ShutdownDelayDuration, ""+
		"Time to delay the termination. During that time the server keeps serving requests normally and /healthz "+
		"returns success, but /readyz immediately returns failure. Graceful termination starts after this delay "+
		"has elapsed. This can be used to allow load balancer to stop sending traffic to this server.")

	utilfeature.DefaultMutableFeatureGate.AddFlag(fs)
}
<commit_msg>fix the wrong function description<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package options

import (
	"fmt"
	"net"
	"time"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/serializer"
	"k8s.io/apiserver/pkg/server"
	utilfeature "k8s.io/apiserver/pkg/util/feature"

	"github.com/spf13/pflag"
)

// ServerRunOptions contains the options while running a generic api server.
type ServerRunOptions struct {
	AdvertiseAddress net.IP

	CorsAllowedOriginList       []string
	ExternalHost                string
	MaxRequestsInFlight         int
	MaxMutatingRequestsInFlight int
	RequestTimeout              time.Duration
	GoawayChance                float64
	LivezGracePeriod            time.Duration
	MinRequestTimeout           int
	ShutdownDelayDuration       time.Duration
	// We intentionally did not add a flag for this option. Users of the
	// apiserver library can wire it to a flag.
	JSONPatchMaxCopyBytes int64
	// The limit on the request body size that would be accepted and
	// decoded in a write request. 0 means no limit.
	// We intentionally did not add a flag for this option. Users of the
	// apiserver library can wire it to a flag.
	MaxRequestBodyBytes       int64
	TargetRAMMB               int
	EnablePriorityAndFairness bool
}

func NewServerRunOptions() *ServerRunOptions {
	defaults := server.NewConfig(serializer.CodecFactory{})
	return &ServerRunOptions{
		MaxRequestsInFlight:         defaults.MaxRequestsInFlight,
		MaxMutatingRequestsInFlight: defaults.MaxMutatingRequestsInFlight,
		RequestTimeout:              defaults.RequestTimeout,
		LivezGracePeriod:            defaults.LivezGracePeriod,
		MinRequestTimeout:           defaults.MinRequestTimeout,
		ShutdownDelayDuration:       defaults.ShutdownDelayDuration,
		JSONPatchMaxCopyBytes:       defaults.JSONPatchMaxCopyBytes,
		MaxRequestBodyBytes:         defaults.MaxRequestBodyBytes,
		EnablePriorityAndFairness:   true,
	}
}

// ApplyTo applies the run options to the method receiver and returns self
func (s *ServerRunOptions) ApplyTo(c *server.Config) error {
	c.CorsAllowedOriginList = s.CorsAllowedOriginList
	c.ExternalAddress = s.ExternalHost
	c.MaxRequestsInFlight = s.MaxRequestsInFlight
	c.MaxMutatingRequestsInFlight = s.MaxMutatingRequestsInFlight
	c.LivezGracePeriod = s.LivezGracePeriod
	c.RequestTimeout = s.RequestTimeout
	c.GoawayChance = s.GoawayChance
	c.MinRequestTimeout = s.MinRequestTimeout
	c.ShutdownDelayDuration = s.ShutdownDelayDuration
	c.JSONPatchMaxCopyBytes = s.JSONPatchMaxCopyBytes
	c.MaxRequestBodyBytes = s.MaxRequestBodyBytes
	c.PublicAddress = s.AdvertiseAddress

	return nil
}

// DefaultAdvertiseAddress sets the field AdvertiseAddress if unset. The field will be set based on the SecureServingOptions.
func (s *ServerRunOptions) DefaultAdvertiseAddress(secure *SecureServingOptions) error {
	if secure == nil {
		return nil
	}

	if s.AdvertiseAddress == nil || s.AdvertiseAddress.IsUnspecified() {
		hostIP, err := secure.DefaultExternalAddress()
		if err != nil {
			return fmt.Errorf("Unable to find suitable network address.error='%v'. "+
				"Try to set the AdvertiseAddress directly or provide a valid BindAddress to fix this.", err)
		}
		s.AdvertiseAddress = hostIP
	}

	return nil
}

// Validate checks validation of ServerRunOptions
func (s *ServerRunOptions) Validate() []error {
	errors := []error{}
	if s.TargetRAMMB < 0 {
		errors = append(errors, fmt.Errorf("--target-ram-mb can not be negative value"))
	}

	if s.LivezGracePeriod < 0 {
		errors = append(errors, fmt.Errorf("--livez-grace-period can not be a negative value"))
	}

	if s.MaxRequestsInFlight < 0 {
		errors = append(errors, fmt.Errorf("--max-requests-inflight can not be negative value"))
	}
	if s.MaxMutatingRequestsInFlight < 0 {
		errors = append(errors, fmt.Errorf("--max-mutating-requests-inflight can not be negative value"))
	}

	if s.RequestTimeout.Nanoseconds() < 0 {
		errors = append(errors, fmt.Errorf("--request-timeout can not be negative value"))
	}

	if s.GoawayChance < 0 || s.GoawayChance > 0.02 {
		errors = append(errors, fmt.Errorf("--goaway-chance can not be less than 0 or greater than 0.02"))
	}

	if s.MinRequestTimeout < 0 {
		errors = append(errors, fmt.Errorf("--min-request-timeout can not be negative value"))
	}

	if s.ShutdownDelayDuration < 0 {
		errors = append(errors, fmt.Errorf("--shutdown-delay-duration can not be negative value"))
	}

	if s.JSONPatchMaxCopyBytes < 0 {
		errors = append(errors, fmt.Errorf("--json-patch-max-copy-bytes can not be negative value"))
	}

	if s.MaxRequestBodyBytes < 0 {
		errors = append(errors, fmt.Errorf("--max-resource-write-bytes can not be negative value"))
	}

	return errors
}

// AddUniversalFlags adds flags for a specific APIServer to the specified FlagSet
func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
	// Note: the weird ""+ in below lines seems to be the only way to get gofmt to
	// arrange these text blocks sensibly. Grrr.

	fs.IPVar(&s.AdvertiseAddress, "advertise-address", s.AdvertiseAddress, ""+
		"The IP address on which to advertise the apiserver to members of the cluster. This "+
		"address must be reachable by the rest of the cluster. If blank, the --bind-address "+
		"will be used. If --bind-address is unspecified, the host's default interface will "+
		"be used.")

	fs.StringSliceVar(&s.CorsAllowedOriginList, "cors-allowed-origins", s.CorsAllowedOriginList, ""+
		"List of allowed origins for CORS, comma separated.  An allowed origin can be a regular "+
		"expression to support subdomain matching. If this list is empty CORS will not be enabled.")

	fs.IntVar(&s.TargetRAMMB, "target-ram-mb", s.TargetRAMMB,
		"Memory limit for apiserver in MB (used to configure sizes of caches, etc.)")

	fs.StringVar(&s.ExternalHost, "external-hostname", s.ExternalHost,
		"The hostname to use when generating externalized URLs for this master (e.g. Swagger API Docs or OpenID Discovery).")

	deprecatedMasterServiceNamespace := metav1.NamespaceDefault
	fs.StringVar(&deprecatedMasterServiceNamespace, "master-service-namespace", deprecatedMasterServiceNamespace, ""+
		"DEPRECATED: the namespace from which the Kubernetes master services should be injected into pods.")

	fs.IntVar(&s.MaxRequestsInFlight, "max-requests-inflight", s.MaxRequestsInFlight, ""+
		"The maximum number of non-mutating requests in flight at a given time. When the server exceeds this, "+
		"it rejects requests. Zero for no limit.")

	fs.IntVar(&s.MaxMutatingRequestsInFlight, "max-mutating-requests-inflight", s.MaxMutatingRequestsInFlight, ""+
		"The maximum number of mutating requests in flight at a given time. When the server exceeds this, "+
		"it rejects requests. Zero for no limit.")

	fs.DurationVar(&s.RequestTimeout, "request-timeout", s.RequestTimeout, ""+
		"An optional field indicating the duration a handler must keep a request open before timing "+
		"it out. This is the default request timeout for requests but may be overridden by flags such as "+
		"--min-request-timeout for specific types of requests.")

	fs.Float64Var(&s.GoawayChance, "goaway-chance", s.GoawayChance, ""+
		"To prevent HTTP/2 clients from getting stuck on a single apiserver, randomly close a connection (GOAWAY). "+
		"The client's other in-flight requests won't be affected, and the client will reconnect, likely landing on a different apiserver after going through the load balancer again. "+
		"This argument sets the fraction of requests that will be sent a GOAWAY. Clusters with single apiservers, or which don't use a load balancer, should NOT enable this. "+
		"Min is 0 (off), Max is .02 (1/50 requests); .001 (1/1000) is a recommended starting point.")

	fs.DurationVar(&s.LivezGracePeriod, "livez-grace-period", s.LivezGracePeriod, ""+
		"This option represents the maximum amount of time it should take for apiserver to complete its startup sequence "+
		"and become live. From apiserver's start time to when this amount of time has elapsed, /livez will assume "+
		"that unfinished post-start hooks will complete successfully and therefore return true.")

	fs.IntVar(&s.MinRequestTimeout, "min-request-timeout", s.MinRequestTimeout, ""+
		"An optional field indicating the minimum number of seconds a handler must keep "+
		"a request open before timing it out. Currently only honored by the watch request "+
		"handler, which picks a randomized value above this number as the connection timeout, "+
		"to spread out load.")

	fs.BoolVar(&s.EnablePriorityAndFairness, "enable-priority-and-fairness", s.EnablePriorityAndFairness, ""+
		"If true and the APIPriorityAndFairness feature gate is enabled, replace the max-in-flight handler with an enhanced one that queues and dispatches with priority and fairness")

	fs.DurationVar(&s.ShutdownDelayDuration, "shutdown-delay-duration", s.ShutdownDelayDuration, ""+
		"Time to delay the termination. During that time the server keeps serving requests normally and /healthz "+
		"returns success, but /readyz immediately returns failure. Graceful termination starts after this delay "+
		"has elapsed. This can be used to allow load balancer to stop sending traffic to this server.")

	utilfeature.DefaultMutableFeatureGate.AddFlag(fs)
}
<|endoftext|>