<commit_before>// Copyright 2014 Richard Lehane. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Core siegfried defaults
package config

import (
	"io"
	"net/http"
	"path/filepath"
	"time"
)

var siegfried = struct {
	version   [3]int // Siegfried version (i.e. of the sf tool)
	home      string // Home directory used by both sf and roy tools
	signature string // Name of signature file
	conf      string // Name of the conf file
	magic     []byte // Magic bytes to ID signature file
	// Defaults for processing bytematcher signatures. These control the segmentation.
	distance   int // The acceptable distance between two frames before they will be segmented (default is 8192)
	rng        int // The acceptable range between two frames before they will be segmented (default is 0-2049)
	choices    int // The acceptable number of plain sequences generated from a single segment
	cost       int // The acceptable cost of a signature segement, in terms of the times that it might match in a worst case
	repetition int // The acceptable repetition within a signature segment, used in combination with cost to determine segmentation.
	// Config for using the update service.
	updateURL       string // URL for the update service (a JSON file that indicates whether update necessary and where can be found)
	updateTimeout   time.Duration
	updateTransport *http.Transport
	// Archivematica format policy registry service
	fpr string
	// DEBUG and SLOW modes
	debug      bool
	slow       bool
	out        io.Writer
	checkpoint int64
	userAgent  string
}{
	version:         [3]int{1, 9, 1},
	signature:       "default.sig",
	conf:            "sf.conf",
	magic:           []byte{'s', 'f', 0x00, 0xFF},
	distance:        8192,
	rng:             4096,
	choices:         128,
	cost:            25600000,
	repetition:      4,
	updateURL:       "https://www.itforarchivists.com/siegfried/update", // "http://localhost:8081/siegfried/update",
	updateTimeout:   30 * time.Second,
	updateTransport: &http.Transport{Proxy: http.ProxyFromEnvironment},
	fpr:             "/tmp/siegfried",
	checkpoint:      524288, // point at which to report slow signatures (must be power of two)
	userAgent:       "siegfried/siegbot (+https://github.com/richardlehane/siegfried)",
}

// GETTERS

// Version reports the siegfried version.
func Version() [3]int {
	return siegfried.version
}

// Home reports the siegfried HOME location (e.g. /usr/home/siegfried).
func Home() string {
	return siegfried.home
}

// Home makes a path local to Home() if it is relative
func Local(base string) string {
	if filepath.Dir(base) == "." {
		return filepath.Join(siegfried.home, base)
	}
	return base
}

// Signature returns the path to the siegfried signature file.
func Signature() string {
	return Local(siegfried.signature)
}

// SignatureBase returns the filename of the siegfried signature file.
func SignatureBase() string {
	return siegfried.signature
}

// Conf returns the path to the siegfried configuration file.
func Conf() string {
	return Local(siegfried.conf)
}

// Magic returns the magic string encoded at the start of a siegfried signature file.
func Magic() []byte {
	return siegfried.magic
}

// Distance is a bytematcher setting. It controls the absolute widths at which segments in signatures are split.
// E.g. if segments are separated by a minimum of 50 and maximum of 100 bytes, the distance is 100.
// A short distance means a smaller Aho Corasick search tree and more patterns to follow-up.
// A long distance means a larger Aho Corasick search tree and more signatures immediately satisfied without follow-up pattern matching.
func Distance() int {
	return siegfried.distance
}

// Range is a bytematcher setting. It controls the relative widths at which segments in signatures are split.
// E.g. if segments are separated by a minimum of 50 and maximum of 100 bytes, the range is 50.
// A small range means a smaller Aho Corasick search tree and more patterns to follow-up.
// A large range means a larger Aho Corasick search tree and more signatures immediately satisfied without follow-up pattern matching.
func Range() int {
	return siegfried.rng
}

// Choices is a bytematcher setting. It controls the number of tolerable strings produced by processing signature segments.
// E.g. signature has two adjoining frames ("PDF") and ("1.1" OR "1.2") it can be processed into two search strings: "PDF1.1" and "PDF1.2".
// A low number of choices means a smaller Aho Corasick search tree and more patterns to follow-up.
// A large of choices means a larger Aho Corasick search tree and more signatures immediately satisfied without follow-up pattern matching.
func Choices() int {
	return siegfried.choices
}

// Choices is a bytematcher setting. It controls the number of tolerable matches in a worst case scenario for a signature segement.
// If this cost is exceeded, then segmentation won't happen and the choices/range/distance preferences will be ignored.
func Cost() int {
	return siegfried.cost
}

// Repetitition is a bytematcher setting. It is used in combination with Cost to determine segmentation.
func Repetition() int {
	return siegfried.repetition
}

// UpdateOptions returns the update URL, timeout and transport for the sf -update command.
func UpdateOptions() (string, time.Duration, *http.Transport) {
	return siegfried.updateURL, siegfried.updateTimeout, siegfried.updateTransport
}

// Fpr reports whether sf is being run in -fpr (Archivematica format policy registry) mode.
func Fpr() string {
	return siegfried.fpr
}

// Debug reports whether debug logging is activated.
func Debug() bool {
	return siegfried.debug
}

// Slow reports whether slow logging is activated.
func Slow() bool {
	return siegfried.slow
}

// Out reports the target for logging messages (STDOUT or STDIN).
func Out() io.Writer {
	return siegfried.out
}

// Checkpoint reports the offset at which slow logging should trigger.
func Checkpoint(i int64) bool {
	return i == siegfried.checkpoint
}

// UserAgent returns the siegbot User-Agent string for http requests.
func UserAgent() string {
	return siegfried.userAgent
}

// SETTERS

// SetHome sets the siegfried HOME location (e.g. /usr/home/siegfried).
func SetHome(h string) {
	siegfried.home = h
}

// SetSignature sets the signature filename or filepath.
func SetSignature(s string) {
	siegfried.signature = s
}

// SetConf sets the configuration filename or filepath.
func SetConf(s string) {
	siegfried.conf = s
}

// SetDistance sets the distance variable for the bytematcher.
func SetDistance(i int) func() private {
	return func() private {
		siegfried.distance = i
		return private{}
	}
}

// SetRange sets the range variable for the bytematcher.
func SetRange(i int) func() private {
	return func() private {
		siegfried.rng = i
		return private{}
	}
}

// SetChoices sets the choices variable for the bytematcher.
func SetChoices(i int) func() private {
	return func() private {
		siegfried.choices = i
		return private{}
	}
}

// SetCost sets the cost variable for the bytematcher.
func SetCost(i int) func() private {
	return func() private {
		siegfried.cost = i
		return private{}
	}
}

// SetRepetition sets the repetitition variable for the bytematcher.
func SetRepetition(i int) func() private {
	return func() private {
		siegfried.repetition = i
		return private{}
	}
}

// SetDebug sets degub logging on.
func SetDebug() {
	siegfried.debug = true
}

// SetSlow sets slow logging on.
func SetSlow() {
	siegfried.slow = true
}

// SetOut sets the target for logging.
func SetOut(o io.Writer) {
	siegfried.out = o
}
<commit_msg>prep for 1.9.2 update<commit_after>// Copyright 2014 Richard Lehane. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Core siegfried defaults
package config

import (
	"io"
	"net/http"
	"path/filepath"
	"time"
)

var siegfried = struct {
	version   [3]int // Siegfried version (i.e. of the sf tool)
	home      string // Home directory used by both sf and roy tools
	signature string // Name of signature file
	conf      string // Name of the conf file
	magic     []byte // Magic bytes to ID signature file
	// Defaults for processing bytematcher signatures. These control the segmentation.
	distance   int // The acceptable distance between two frames before they will be segmented (default is 8192)
	rng        int // The acceptable range between two frames before they will be segmented (default is 0-2049)
	choices    int // The acceptable number of plain sequences generated from a single segment
	cost       int // The acceptable cost of a signature segement, in terms of the times that it might match in a worst case
	repetition int // The acceptable repetition within a signature segment, used in combination with cost to determine segmentation.
	// Config for using the update service.
	updateURL       string // URL for the update service (a JSON file that indicates whether update necessary and where can be found)
	updateTimeout   time.Duration
	updateTransport *http.Transport
	// Archivematica format policy registry service
	fpr string
	// DEBUG and SLOW modes
	debug      bool
	slow       bool
	out        io.Writer
	checkpoint int64
	userAgent  string
}{
	version:         [3]int{1, 9, 2},
	signature:       "default.sig",
	conf:            "sf.conf",
	magic:           []byte{'s', 'f', 0x00, 0xFF},
	distance:        8192,
	rng:             4096,
	choices:         128,
	cost:            25600000,
	repetition:      4,
	updateURL:       "https://www.itforarchivists.com/siegfried/update", // "http://localhost:8081/siegfried/update",
	updateTimeout:   30 * time.Second,
	updateTransport: &http.Transport{Proxy: http.ProxyFromEnvironment},
	fpr:             "/tmp/siegfried",
	checkpoint:      524288, // point at which to report slow signatures (must be power of two)
	userAgent:       "siegfried/siegbot (+https://github.com/richardlehane/siegfried)",
}

// GETTERS

// Version reports the siegfried version.
func Version() [3]int {
	return siegfried.version
}

// Home reports the siegfried HOME location (e.g. /usr/home/siegfried).
func Home() string {
	return siegfried.home
}

// Home makes a path local to Home() if it is relative
func Local(base string) string {
	if filepath.Dir(base) == "." {
		return filepath.Join(siegfried.home, base)
	}
	return base
}

// Signature returns the path to the siegfried signature file.
func Signature() string {
	return Local(siegfried.signature)
}

// SignatureBase returns the filename of the siegfried signature file.
func SignatureBase() string {
	return siegfried.signature
}

// Conf returns the path to the siegfried configuration file.
func Conf() string {
	return Local(siegfried.conf)
}

// Magic returns the magic string encoded at the start of a siegfried signature file.
func Magic() []byte {
	return siegfried.magic
}

// Distance is a bytematcher setting. It controls the absolute widths at which segments in signatures are split.
// E.g. if segments are separated by a minimum of 50 and maximum of 100 bytes, the distance is 100.
// A short distance means a smaller Aho Corasick search tree and more patterns to follow-up.
// A long distance means a larger Aho Corasick search tree and more signatures immediately satisfied without follow-up pattern matching.
func Distance() int {
	return siegfried.distance
}

// Range is a bytematcher setting. It controls the relative widths at which segments in signatures are split.
// E.g. if segments are separated by a minimum of 50 and maximum of 100 bytes, the range is 50.
// A small range means a smaller Aho Corasick search tree and more patterns to follow-up.
// A large range means a larger Aho Corasick search tree and more signatures immediately satisfied without follow-up pattern matching.
func Range() int {
	return siegfried.rng
}

// Choices is a bytematcher setting. It controls the number of tolerable strings produced by processing signature segments.
// E.g. signature has two adjoining frames ("PDF") and ("1.1" OR "1.2") it can be processed into two search strings: "PDF1.1" and "PDF1.2".
// A low number of choices means a smaller Aho Corasick search tree and more patterns to follow-up.
// A large of choices means a larger Aho Corasick search tree and more signatures immediately satisfied without follow-up pattern matching.
func Choices() int {
	return siegfried.choices
}

// Choices is a bytematcher setting. It controls the number of tolerable matches in a worst case scenario for a signature segement.
// If this cost is exceeded, then segmentation won't happen and the choices/range/distance preferences will be ignored.
func Cost() int {
	return siegfried.cost
}

// Repetitition is a bytematcher setting. It is used in combination with Cost to determine segmentation.
func Repetition() int {
	return siegfried.repetition
}

// UpdateOptions returns the update URL, timeout and transport for the sf -update command.
func UpdateOptions() (string, time.Duration, *http.Transport) {
	return siegfried.updateURL, siegfried.updateTimeout, siegfried.updateTransport
}

// Fpr reports whether sf is being run in -fpr (Archivematica format policy registry) mode.
func Fpr() string {
	return siegfried.fpr
}

// Debug reports whether debug logging is activated.
func Debug() bool {
	return siegfried.debug
}

// Slow reports whether slow logging is activated.
func Slow() bool {
	return siegfried.slow
}

// Out reports the target for logging messages (STDOUT or STDIN).
func Out() io.Writer {
	return siegfried.out
}

// Checkpoint reports the offset at which slow logging should trigger.
func Checkpoint(i int64) bool {
	return i == siegfried.checkpoint
}

// UserAgent returns the siegbot User-Agent string for http requests.
func UserAgent() string {
	return siegfried.userAgent
}

// SETTERS

// SetHome sets the siegfried HOME location (e.g. /usr/home/siegfried).
func SetHome(h string) {
	siegfried.home = h
}

// SetSignature sets the signature filename or filepath.
func SetSignature(s string) {
	siegfried.signature = s
}

// SetConf sets the configuration filename or filepath.
func SetConf(s string) {
	siegfried.conf = s
}

// SetDistance sets the distance variable for the bytematcher.
func SetDistance(i int) func() private {
	return func() private {
		siegfried.distance = i
		return private{}
	}
}

// SetRange sets the range variable for the bytematcher.
func SetRange(i int) func() private {
	return func() private {
		siegfried.rng = i
		return private{}
	}
}

// SetChoices sets the choices variable for the bytematcher.
func SetChoices(i int) func() private {
	return func() private {
		siegfried.choices = i
		return private{}
	}
}

// SetCost sets the cost variable for the bytematcher.
func SetCost(i int) func() private {
	return func() private {
		siegfried.cost = i
		return private{}
	}
}

// SetRepetition sets the repetitition variable for the bytematcher.
func SetRepetition(i int) func() private {
	return func() private {
		siegfried.repetition = i
		return private{}
	}
}

// SetDebug sets degub logging on.
func SetDebug() {
	siegfried.debug = true
}

// SetSlow sets slow logging on.
func SetSlow() {
	siegfried.slow = true
}

// SetOut sets the target for logging.
func SetOut(o io.Writer) {
	siegfried.out = o
}
<|endoftext|>


<commit_before>// Copyright 2016-2019 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package endpoint

import (
	"fmt"

	"github.com/cilium/cilium/pkg/addressing"
	"github.com/cilium/cilium/pkg/endpoint/id"
	"github.com/cilium/cilium/pkg/eventqueue"
	"github.com/cilium/cilium/pkg/logging/logfields"
	"github.com/cilium/cilium/pkg/option"
)

type endpointManager interface {
	AllocateID(id uint16) (uint16, error)
	RunK8sCiliumEndpointSync(*Endpoint, EndpointStatusConfiguration)
	UpdateReferences(map[id.PrefixType]string, *Endpoint)
	UpdateIDReference(*Endpoint)
	RemoveReferences(map[id.PrefixType]string)
	RemoveID(uint16)
	ReleaseID(*Endpoint) error
	AddIPv6Address(addressing.CiliumIPv6)
	RemoveIPv6Address(addressing.CiliumIPv6)
}

// Expose exposes the endpoint to the endpointmanager. After this function
// is called, the endpoint may be accessed by any lookup in the endpointmanager.
func (e *Endpoint) Expose(mgr endpointManager) error {
	// No need to check liveness as an endpoint can only be deleted via the
	// API after it has been inserted into the manager.
	// 'e.ID' written below, read lock is not enough.
	e.unconditionalLock()

	newID, err := mgr.AllocateID(e.ID)
	if err != nil {
		e.unlock()
		return err
	}

	e.ID = newID
	e.UpdateLogger(map[string]interface{}{
		logfields.EndpointID: e.ID,
	})

	e.startRegenerationFailureHandler()
	// Now that the endpoint has its ID, it can be created with a name based on
	// its ID, and its eventqueue can be safely started. Ensure that it is only
	// started once it is exposed to the endpointmanager so that it will be
	// stopped when the endpoint is removed from the endpointmanager.
	e.eventQueue = eventqueue.NewEventQueueBuffered(fmt.Sprintf("endpoint-%d", e.ID), option.Config.EndpointQueueSize)
	e.eventQueue.Run()

	mgr.AddIPv6Address(e.IPv6)

	// No need to check liveness as an endpoint can only be deleted via the
	// API after it has been inserted into the manager.
	mgr.UpdateIDReference(e)
	e.updateReferences(mgr)
	e.getLogger().Info("New endpoint")
	e.unlock()

	mgr.RunK8sCiliumEndpointSync(e, option.Config)
	return nil
}

// UpdateReferences updates the endpointmanager mappings of identifiers to
// endpoints for the given endpoint. Returns an error if the endpoint is
// being deleted.
func (e *Endpoint) UpdateReferences(mgr endpointManager) error {
	if err := e.rlockAlive(); err != nil {
		return err
	}
	defer e.runlock()
	e.updateReferences(mgr)
	return nil
}

func (e *Endpoint) updateReferences(mgr endpointManager) {
	refs := e.generateReferences()
	mgr.UpdateReferences(refs, e)
}

func (e *Endpoint) generateReferences() map[id.PrefixType]string {
	refs := make(map[id.PrefixType]string, 6)
	if e.containerID != "" {
		refs[id.ContainerIdPrefix] = e.containerID
	}

	if e.dockerEndpointID != "" {
		refs[id.DockerEndpointPrefix] = e.dockerEndpointID
	}

	if e.IPv4.IsSet() {
		refs[id.IPv4Prefix] = e.IPv4.String()
	}

	if e.IPv6.IsSet() {
		refs[id.IPv6Prefix] = e.IPv6.String()
	}

	if e.containerName != "" {
		refs[id.ContainerNamePrefix] = e.containerName
	}

	if podName := e.getK8sNamespaceAndPodName(); podName != "" {
		refs[id.PodNamePrefix] = podName
	}
	return refs
}

func (e *Endpoint) removeReferences(mgr endpointManager) {
	refs := e.generateReferences()
	mgr.RemoveReferences(refs)
}

// Unexpose removes the endpoint from being globally acccessible via other
// packages.
func (e *Endpoint) Unexpose(mgr endpointManager) <-chan struct{} {
	epRemoved := make(chan struct{})

	// This must be done before the ID is released for the endpoint!
	mgr.RemoveID(e.ID)

	mgr.RemoveIPv6Address(e.IPv6)

	go func(ep *Endpoint) {

		// The endpoint's EventQueue may not be stopped yet (depending on whether
		// the caller of the EventQueue has stopped it or not). Call it here
		// to be safe so that ep.WaitToBeDrained() does not hang forever.
		ep.eventQueue.Stop()

		// Wait for no more events (primarily regenerations) to be occurring for
		// this endpoint.
		ep.eventQueue.WaitToBeDrained()

		err := mgr.ReleaseID(ep)
		if err != nil {
			// While restoring, endpoint IDs may not have been reused yet.
			// Failure to release means that the endpoint ID was not reused
			// yet.
			//
			// While endpoint is disconnecting, ID is already available in ID cache.
			//
			// Avoid irritating warning messages.
			state := ep.GetState()
			if state != StateRestoring && state != StateDisconnecting && state != StateDisconnected {
				log.WithError(err).WithField("state", state).Warning("Unable to release endpoint ID")
			}
		}

		close(epRemoved)
	}(e)
	e.removeReferences(mgr)
	return epRemoved
}
<commit_msg>endpoint: Add function to initialize event queue<commit_after>// Copyright 2016-2019 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package endpoint

import (
	"fmt"

	"github.com/cilium/cilium/pkg/addressing"
	"github.com/cilium/cilium/pkg/endpoint/id"
	"github.com/cilium/cilium/pkg/eventqueue"
	"github.com/cilium/cilium/pkg/logging/logfields"
	"github.com/cilium/cilium/pkg/option"
)

type endpointManager interface {
	AllocateID(id uint16) (uint16, error)
	RunK8sCiliumEndpointSync(*Endpoint, EndpointStatusConfiguration)
	UpdateReferences(map[id.PrefixType]string, *Endpoint)
	UpdateIDReference(*Endpoint)
	RemoveReferences(map[id.PrefixType]string)
	RemoveID(uint16)
	ReleaseID(*Endpoint) error
	AddIPv6Address(addressing.CiliumIPv6)
	RemoveIPv6Address(addressing.CiliumIPv6)
}

// Expose exposes the endpoint to the endpointmanager. After this function
// is called, the endpoint may be accessed by any lookup in the endpointmanager.
func (e *Endpoint) Expose(mgr endpointManager) error {
	// No need to check liveness as an endpoint can only be deleted via the
	// API after it has been inserted into the manager.
	// 'e.ID' written below, read lock is not enough.
	e.unconditionalLock()

	newID, err := mgr.AllocateID(e.ID)
	if err != nil {
		e.unlock()
		return err
	}

	e.ID = newID
	e.UpdateLogger(map[string]interface{}{
		logfields.EndpointID: e.ID,
	})

	e.startRegenerationFailureHandler()
	// Now that the endpoint has its ID, it can be created with a name based on
	// its ID, and its eventqueue can be safely started. Ensure that it is only
	// started once it is exposed to the endpointmanager so that it will be
	// stopped when the endpoint is removed from the endpointmanager.
	if e.eventQueue == nil {
		e.InitEventQueue()
	}
	e.eventQueue.Run()

	mgr.AddIPv6Address(e.IPv6)

	// No need to check liveness as an endpoint can only be deleted via the
	// API after it has been inserted into the manager.
	mgr.UpdateIDReference(e)
	e.updateReferences(mgr)
	e.getLogger().Info("New endpoint")
	e.unlock()

	mgr.RunK8sCiliumEndpointSync(e, option.Config)
	return nil
}

// UpdateReferences updates the endpointmanager mappings of identifiers to
// endpoints for the given endpoint. Returns an error if the endpoint is
// being deleted.
func (e *Endpoint) UpdateReferences(mgr endpointManager) error {
	if err := e.rlockAlive(); err != nil {
		return err
	}
	defer e.runlock()
	e.updateReferences(mgr)
	return nil
}

func (e *Endpoint) updateReferences(mgr endpointManager) {
	refs := e.generateReferences()
	mgr.UpdateReferences(refs, e)
}

func (e *Endpoint) generateReferences() map[id.PrefixType]string {
	refs := make(map[id.PrefixType]string, 6)
	if e.containerID != "" {
		refs[id.ContainerIdPrefix] = e.containerID
	}

	if e.dockerEndpointID != "" {
		refs[id.DockerEndpointPrefix] = e.dockerEndpointID
	}

	if e.IPv4.IsSet() {
		refs[id.IPv4Prefix] = e.IPv4.String()
	}

	if e.IPv6.IsSet() {
		refs[id.IPv6Prefix] = e.IPv6.String()
	}

	if e.containerName != "" {
		refs[id.ContainerNamePrefix] = e.containerName
	}

	if podName := e.getK8sNamespaceAndPodName(); podName != "" {
		refs[id.PodNamePrefix] = podName
	}
	return refs
}

func (e *Endpoint) removeReferences(mgr endpointManager) {
	refs := e.generateReferences()
	mgr.RemoveReferences(refs)
}

// Unexpose removes the endpoint from being globally acccessible via other
// packages.
func (e *Endpoint) Unexpose(mgr endpointManager) <-chan struct{} {
	epRemoved := make(chan struct{})

	// This must be done before the ID is released for the endpoint!
	mgr.RemoveID(e.ID)

	mgr.RemoveIPv6Address(e.IPv6)

	go func(ep *Endpoint) {

		// The endpoint's EventQueue may not be stopped yet (depending on whether
		// the caller of the EventQueue has stopped it or not). Call it here
		// to be safe so that ep.WaitToBeDrained() does not hang forever.
		ep.eventQueue.Stop()

		// Wait for no more events (primarily regenerations) to be occurring for
		// this endpoint.
		ep.eventQueue.WaitToBeDrained()

		err := mgr.ReleaseID(ep)
		if err != nil {
			// While restoring, endpoint IDs may not have been reused yet.
			// Failure to release means that the endpoint ID was not reused
			// yet.
			//
			// While endpoint is disconnecting, ID is already available in ID cache.
			//
			// Avoid irritating warning messages.
			state := ep.GetState()
			if state != StateRestoring && state != StateDisconnecting && state != StateDisconnected {
				log.WithError(err).WithField("state", state).Warning("Unable to release endpoint ID")
			}
		}

		close(epRemoved)
	}(e)
	e.removeReferences(mgr)
	return epRemoved
}

// InitEventQueue initializes the endpoint's event queue. Note that this
// function does not begin processing events off the queue, as that's left up
// to the caller to call Expose in order to allow other subsystems to access
// the endpoint. This function assumes that the endpoint ID has already been
// allocated!
//
// Having this be a separate function allows us to prepare
// the event queue while the endpoint is being validated (during restoration)
// so that when its metadata is resolved, events can be enqueued (such as
// visibility policy and bandwidth policy).
func (e *Endpoint) InitEventQueue() {
	e.eventQueue = eventqueue.NewEventQueueBuffered(fmt.Sprintf("endpoint-%d", e.ID), option.Config.EndpointQueueSize)
}
<|endoftext|>


<commit_before>package fuzzy

import (
	"github.com/skatteetaten/ao/pkg/client"
	"github.com/stretchr/testify/assert"
	"testing"
)

var fileNames = client.FileNames{
	"about.json",
	"console.json",
	"boober.json",
	"utv/about.json",
	"utv/about-template.json",
	"utv/boober.json",
	"utv/console.json",
	"utv-relay/about.json",
	"utv-relay/boober.json",
	"test/about.json",
	"test/boober.json",
	"test/console.json",
	"test-relay/about.json",
	"test-relay/boober.json",
}

func TestFilterFileNamesForDeploy(t *testing.T) {
	var expected = []string{
		"utv/boober",
		"utv/console",
		"utv-relay/boober",
		"test/boober",
		"test/console",
		"test-relay/boober",
	}

	actual := fileNames.GetApplicationIds()
	assert.Equal(t, expected, actual)
}

func TestFindMatches(t *testing.T) {

	tests := []struct {
		Search     string
		WithSuffix bool
		Expected   []string
	}{
		{"asdlfkja", true, []string{}},
		{"about", true, []string{"about.json"}},
		{"console", true, []string{"console.json"}},
		{"test/boober", true, []string{"test/boober.json"}},
		{"con", true, []string{"console.json", "utv/console.json", "test/console.json"}},
		{"con", false, []string{"console", "utv/console", "test/console"}},
		{"utv/ab", true, []string{"utv/about.json", "utv-relay/about.json", "utv/about-template.json"}},
		{"utv/o", true, []string{"utv/about.json", "utv/boober.json", "utv/console.json",
			"utv-relay/about.json", "utv-relay/boober.json", "utv/about-template.json"}},
	}

	for _, test := range tests {
		matches := FindMatches(test.Search, fileNames, test.WithSuffix)
		assert.Equal(t, test.Expected, matches, test.Search+" returned unexpected matches than expected.")
	}
}

func TestFindFileToEdit(t *testing.T) {
	tests := []struct {
		Search   string
		Expected []string
	}{
		{"about", []string{"about.json"}},
		{"console", []string{"console.json"}},
		{"utv/ab", []string{"utv/about.json", "utv-relay/about.json", "utv/about-template.json"}},
	}

	for _, test := range tests {
		filename := SearchForFile(test.Search, fileNames)
		assert.Equal(t, test.Expected, filename)
	}
}

func TestFindApplicationsToDeploy(t *testing.T) {
	tests := []struct {
		Search   string
		Expected []string
	}{
		{"aosdfkja", []string{}},
		{"utv", []string{"utv/boober", "utv/console"}},
		{"console", []string{"test/console", "utv/console"}},
		{"test", []string{"test/boober", "test/console"}},
		{"test-r", []string{"test-relay/boober"}},
		{"boober", []string{"test/boober", "test-relay/boober", "utv/boober", "utv-relay/boober"}},
		{"boo", []string{"utv/boober", "test/boober", "utv-relay/boober", "test-relay/boober"}},
	}

	filteredFiles := fileNames.GetApplicationIds()

	for _, test := range tests {
		deploys := SearchForApplications(test.Search, filteredFiles)
		assert.Equal(t, test.Expected, deploys, "Searching for "+test.Search)
	}
}

func TestFindAllFor(t *testing.T) {
	tests := []struct {
		Search   string
		Mode     FilterMode
		Expected []string
	}{
		{"utv", ENV_FILTER, []string{"utv/boober", "utv/console"}},
		{"console", APP_FILTER, []string{"test/console", "utv/console"}},
		{"test", ENV_FILTER, []string{"test/boober", "test/console"}},
		{"test-r", ENV_FILTER, []string{}},
		{"boober", APP_FILTER, []string{"test/boober", "test-relay/boober", "utv/boober", "utv-relay/boober"}},
		{"boo", APP_FILTER, []string{}},
	}

	filteredFiles := fileNames.GetApplicationIds()

	filteredFiles = append(filteredFiles, "illegalfile")

	for _, test := range tests {
		deploys := FindAllDeploysFor(test.Mode, test.Search, filteredFiles)
		assert.Equal(t, test.Expected, deploys)
	}
}
<commit_msg>fixed broken test<commit_after>package fuzzy

import (
	"testing"

	"github.com/skatteetaten/ao/pkg/client"
	"github.com/stretchr/testify/assert"
)

var fileNames = client.FileNames{
	"about.json",
	"console.json",
	"boober.json",
	"utv/about.json",
	"utv/about-template.json",
	"utv/boober.json",
	"utv/console.json",
	"utv-relay/about.json",
	"utv-relay/boober.json",
	"test/about.json",
	"test/boober.json",
	"test/console.json",
	"test-relay/about.json",
	"test-relay/boober.json",
}

func TestFilterFileNamesForDeploy(t *testing.T) {
	var expected = []string{
		"test-relay/boober",
		"test/boober",
		"test/console",
		"utv-relay/boober",
		"utv/boober",
		"utv/console",
	}

	actual := fileNames.GetApplicationIds()
	assert.Equal(t, expected, actual)
}

func TestFindMatches(t *testing.T) {

	tests := []struct {
		Search     string
		WithSuffix bool
		Expected   []string
	}{
		{"asdlfkja", true, []string{}},
		{"about", true, []string{"about.json"}},
		{"console", true, []string{"console.json"}},
		{"test/boober", true, []string{"test/boober.json"}},
		{"con", true, []string{"console.json", "utv/console.json", "test/console.json"}},
		{"con", false, []string{"console", "utv/console", "test/console"}},
		{"utv/ab", true, []string{"utv/about.json", "utv-relay/about.json", "utv/about-template.json"}},
		{"utv/o", true, []string{"utv/about.json", "utv/boober.json", "utv/console.json",
			"utv-relay/about.json", "utv-relay/boober.json", "utv/about-template.json"}},
	}

	for _, test := range tests {
		matches := FindMatches(test.Search, fileNames, test.WithSuffix)
		assert.Equal(t, test.Expected, matches, test.Search+" returned unexpected matches than expected.")
	}
}

func TestFindFileToEdit(t *testing.T) {
	tests := []struct {
		Search   string
		Expected []string
	}{
		{"about", []string{"about.json"}},
		{"console", []string{"console.json"}},
		{"utv/ab", []string{"utv/about.json", "utv-relay/about.json", "utv/about-template.json"}},
	}

	for _, test := range tests {
		filename := SearchForFile(test.Search, fileNames)
		assert.Equal(t, test.Expected, filename)
	}
}

func TestFindApplicationsToDeploy(t *testing.T) {
	tests := []struct {
		Search   string
		Expected []string
	}{
		{"aosdfkja", []string{}},
		{"utv", []string{"utv/boober", "utv/console"}},
		{"console", []string{"test/console", "utv/console"}},
		{"test", []string{"test/boober", "test/console"}},
		{"test-r", []string{"test-relay/boober"}},
		{"boober", []string{"test/boober", "test-relay/boober", "utv/boober", "utv-relay/boober"}},
		{"boo", []string{"utv/boober", "test/boober", "utv-relay/boober", "test-relay/boober"}},
	}

	filteredFiles := fileNames.GetApplicationIds()

	for _, test := range tests {
		deploys := SearchForApplications(test.Search, filteredFiles)
		assert.Equal(t, test.Expected, deploys, "Searching for "+test.Search)
	}
}

func TestFindAllFor(t *testing.T) {
	tests := []struct {
		Search   string
		Mode     FilterMode
		Expected []string
	}{
		{"utv", ENV_FILTER, []string{"utv/boober", "utv/console"}},
		{"console", APP_FILTER, []string{"test/console", "utv/console"}},
		{"test", ENV_FILTER, []string{"test/boober", "test/console"}},
		{"test-r", ENV_FILTER, []string{}},
		{"boober", APP_FILTER, []string{"test/boober", "test-relay/boober", "utv/boober", "utv-relay/boober"}},
		{"boo", APP_FILTER, []string{}},
	}

	filteredFiles := fileNames.GetApplicationIds()

	filteredFiles = append(filteredFiles, "illegalfile")

	for _, test := range tests {
		deploys := FindAllDeploysFor(test.Mode, test.Search, filteredFiles)
		assert.Equal(t, test.Expected, deploys)
	}
}
<|endoftext|>


<commit_before>package instance

import (
	"context"
	"fmt"
	"net/url"
	"sync"
	"time"

	"github.com/cozy/cozy-stack/pkg/apps"
	"github.com/cozy/cozy-stack/pkg/config"
	"github.com/cozy/cozy-stack/pkg/logger"
	"github.com/cozy/cozy-stack/pkg/utils"
	multierror "github.com/hashicorp/go-multierror"
	"github.com/robfig/cron"
)

const numUpdaters = 50

var log = logger.WithNamespace("updates")
var globalUpdating = make(chan struct{}, 1)

func init() {
	globalUpdating <- struct{}{}
}

type updateCron struct {
	stopped  chan struct{}
	finished chan struct{}
}

// StartUpdateCron starts the auto update process which launche a full auto
// updates of all the instances existing.
func StartUpdateCron() (utils.Shutdowner, error) {
	autoUpdates := config.GetConfig().AutoUpdates
	if !autoUpdates.Activated {
		return utils.NopShutdown, nil
	}

	u := &updateCron{
		stopped:  make(chan struct{}),
		finished: make(chan struct{}),
	}

	scheduleValue := autoUpdates.Schedule
	if scheduleValue == "" {
		scheduleValue = "@midnight"
	}
	schedule, err := cron.Parse(scheduleValue)
	if err != nil {
		return nil, err
	}

	go func() {
		next := time.Now()
		defer func() { u.finished <- struct{}{} }()
		for {
			next = schedule.Next(next)
			select {
			case <-time.After(-time.Since(next)):
				if err := UpdateAll(false); err != nil {
					log.Error("Could not update all:", err)
				}
			case <-u.stopped:
				return
			}
		}
	}()

	return u, nil
}

func (u *updateCron) Shutdown(ctx context.Context) error {
	fmt.Print("  shutting down updaters...")
	u.stopped <- struct{}{}
	select {
	case <-ctx.Done():
		fmt.Println("timeouted.")
	case <-u.finished:
		fmt.Println("ok.")
	}
	return nil
}

// UpdateAll starts the auto-updates process for all instances. The slugs
// parameters can be used optionnaly to filter (whitelist) the applications'
// slug to update.
func UpdateAll(force bool, slugs ...string) error {
	<-globalUpdating
	defer func() {
		globalUpdating <- struct{}{}
	}()

	insc := make(chan *apps.Installer)
	errc := make(chan error)

	var g sync.WaitGroup
	g.Add(numUpdaters)

	for i := 0; i < numUpdaters; i++ {
		go func() {
			for installer := range insc {
				_, err := installer.RunSync()
				errc <- err
			}
			g.Done()
		}()
	}

	go func() {
		// TODO: filter instances that are AutoUpdate only
		ForeachInstances(func(inst *Instance) error {
			if force || !inst.NoAutoUpdate {
				installerPush(inst, insc, errc, slugs...)
			}
			return nil
		})
		close(insc)
	}()

	var errm error
	go func() {
		for err := range errc {
			if err != nil {
				errm = multierror.Append(errm, err)
			}
		}
	}()

	g.Wait()
	close(errc)

	return errm
}

// UpdateInstance starts the auto-update process on the given instance. The
// slugs parameters can be used to filter (whitelist) the applications' slug
func UpdateInstance(inst *Instance, slugs ...string) error {
	insc := make(chan *apps.Installer)
	errc := make(chan error)

	var g sync.WaitGroup
	g.Add(4)

	for i := 0; i < 4; i++ {
		go func() {
			for installer := range insc {
				_, err := installer.RunSync()
				errc <- err
			}
			g.Done()
		}()
	}

	go func() {
		installerPush(inst, insc, errc, slugs...)
		close(insc)
	}()

	var errm error
	go func() {
		for err := range errc {
			if err != nil {
				errm = multierror.Append(errm, err)
			}
		}
	}()

	g.Wait()
	close(errc)

	return errm
}

func installerPush(inst *Instance, insc chan *apps.Installer, errc chan error, slugs ...string) {
	registries, err := inst.Registries()
	if err != nil {
		errc <- err
		return
	}

	var g sync.WaitGroup
	g.Add(2)

	go func() {
		defer g.Done()
		webapps, err := apps.ListWebapps(inst)
		if err != nil {
			errc <- err
			return
		}
		for _, app := range webapps {
			if filterSlug(app.Slug(), slugs) {
				continue
			}
			installer, err := createInstaller(inst, registries, app)
			if err != nil {
				errc <- err
			} else {
				insc <- installer
			}
		}
	}()

	go func() {
		defer g.Done()
		konnectors, err := apps.ListKonnectors(inst)
		if err != nil {
			errc <- err
			return
		}
		for _, app := range konnectors {
			if filterSlug(app.Slug(), slugs) {
				continue
			}
			installer, err := createInstaller(inst, registries, app)
			if err != nil {
				errc <- err
			} else {
				insc <- installer
			}
		}
	}()

	g.Wait()
}

func filterSlug(slug string, slugs []string) bool {
	if len(slugs) == 0 {
		return false
	}
	for _, s := range slugs {
		if s == slug {
			return false
		}
	}
	return true
}

func createInstaller(inst *Instance, registries []*url.URL, man apps.Manifest) (*apps.Installer, error) {
	return apps.NewInstaller(inst, inst.AppsCopier(man.AppType()),
		&apps.InstallerOptions{
			Operation:  apps.Update,
			Manifest:   man,
			Registries: registries,
		},
	)
}
<commit_msg>No default for auto-updates<commit_after>package instance

import (
	"context"
	"fmt"
	"net/url"
	"sync"
	"time"

	"github.com/cozy/cozy-stack/pkg/apps"
	"github.com/cozy/cozy-stack/pkg/config"
	"github.com/cozy/cozy-stack/pkg/logger"
	"github.com/cozy/cozy-stack/pkg/utils"
	multierror "github.com/hashicorp/go-multierror"
	"github.com/robfig/cron"
)

const numUpdaters = 50

var log = logger.WithNamespace("updates")
var globalUpdating = make(chan struct{}, 1)

func init() {
	globalUpdating <- struct{}{}
}

type updateCron struct {
	stopped  chan struct{}
	finished chan struct{}
}

// StartUpdateCron starts the auto update process which launche a full auto
// updates of all the instances existing.
func StartUpdateCron() (utils.Shutdowner, error) {
	autoUpdates := config.GetConfig().AutoUpdates
	if !autoUpdates.Activated {
		return utils.NopShutdown, nil
	}

	u := &updateCron{
		stopped:  make(chan struct{}),
		finished: make(chan struct{}),
	}

	schedule, err := cron.Parse(autoUpdates.Schedule)
	if err != nil {
		return nil, err
	}

	go func() {
		next := time.Now()
		defer func() { u.finished <- struct{}{} }()
		for {
			next = schedule.Next(next)
			select {
			case <-time.After(-time.Since(next)):
				if err := UpdateAll(false); err != nil {
					log.Error("Could not update all:", err)
				}
			case <-u.stopped:
				return
			}
		}
	}()

	return u, nil
}

func (u *updateCron) Shutdown(ctx context.Context) error {
	fmt.Print("  shutting down updaters...")
	u.stopped <- struct{}{}
	select {
	case <-ctx.Done():
		fmt.Println("timeouted.")
	case <-u.finished:
		fmt.Println("ok.")
	}
	return nil
}

// UpdateAll starts the auto-updates process for all instances. The slugs
// parameters can be used optionnaly to filter (whitelist) the applications'
// slug to update.
func UpdateAll(force bool, slugs ...string) error {
	<-globalUpdating
	defer func() {
		globalUpdating <- struct{}{}
	}()

	insc := make(chan *apps.Installer)
	errc := make(chan error)

	var g sync.WaitGroup
	g.Add(numUpdaters)

	for i := 0; i < numUpdaters; i++ {
		go func() {
			for installer := range insc {
				_, err := installer.RunSync()
				errc <- err
			}
			g.Done()
		}()
	}

	go func() {
		// TODO: filter instances that are AutoUpdate only
		ForeachInstances(func(inst *Instance) error {
			if force || !inst.NoAutoUpdate {
				installerPush(inst, insc, errc, slugs...)
			}
			return nil
		})
		close(insc)
	}()

	var errm error
	go func() {
		for err := range errc {
			if err != nil {
				errm = multierror.Append(errm, err)
			}
		}
	}()

	g.Wait()
	close(errc)

	return errm
}

// UpdateInstance starts the auto-update process on the given instance. The
// slugs parameters can be used to filter (whitelist) the applications' slug
func UpdateInstance(inst *Instance, slugs ...string) error {
	insc := make(chan *apps.Installer)
	errc := make(chan error)

	var g sync.WaitGroup
	g.Add(4)

	for i := 0; i < 4; i++ {
		go func() {
			for installer := range insc {
				_, err := installer.RunSync()
				errc <- err
			}
			g.Done()
		}()
	}

	go func() {
		installerPush(inst, insc, errc, slugs...)
		close(insc)
	}()

	var errm error
	go func() {
		for err := range errc {
			if err != nil {
				errm = multierror.Append(errm, err)
			}
		}
	}()

	g.Wait()
	close(errc)

	return errm
}

func installerPush(inst *Instance, insc chan *apps.Installer, errc chan error, slugs ...string) {
	registries, err := inst.Registries()
	if err != nil {
		errc <- err
		return
	}

	var g sync.WaitGroup
	g.Add(2)

	go func() {
		defer g.Done()
		webapps, err := apps.ListWebapps(inst)
		if err != nil {
			errc <- err
			return
		}
		for _, app := range webapps {
			if filterSlug(app.Slug(), slugs) {
				continue
			}
			installer, err := createInstaller(inst, registries, app)
			if err != nil {
				errc <- err
			} else {
				insc <- installer
			}
		}
	}()

	go func() {
		defer g.Done()
		konnectors, err := apps.ListKonnectors(inst)
		if err != nil {
			errc <- err
			return
		}
		for _, app := range konnectors {
			if filterSlug(app.Slug(), slugs) {
				continue
			}
			installer, err := createInstaller(inst, registries, app)
			if err != nil {
				errc <- err
			} else {
				insc <- installer
			}
		}
	}()

	g.Wait()
}

func filterSlug(slug string, slugs []string) bool {
	if len(slugs) == 0 {
		return false
	}
	for _, s := range slugs {
		if s == slug {
			return false
		}
	}
	return true
}

func createInstaller(inst *Instance, registries []*url.URL, man apps.Manifest) (*apps.Installer, error) {
	return apps.NewInstaller(inst, inst.AppsCopier(man.AppType()),
		&apps.InstallerOptions{
			Operation:  apps.Update,
			Manifest:   man,
			Registries: registries,
		},
	)
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package out provides a mechanism for sending localized, stylized output to the console.
package out

import (
	"bytes"
	"fmt"
	"html"
	"html/template"
	"io"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/briandowns/spinner"
	isatty "github.com/mattn/go-isatty"

	"k8s.io/klog/v2"
	"k8s.io/minikube/pkg/minikube/out/register"
	"k8s.io/minikube/pkg/minikube/style"
	"k8s.io/minikube/pkg/minikube/translate"
)

// By design, this package uses global references to language and output objects, in preference
// to passing a console object throughout the code base. Typical usage is:
//
// out.SetOutFile(os.Stdout)
// out.String("Starting up!")
// out.Step(style.StatusChange, "Configuring things")

// out.SetErrFile(os.Stderr)
// out.Fatal("Oh no, everything failed.")

// NOTE: If you do not want colorized output, set MINIKUBE_IN_STYLE=false in your environment.

var (
	// silent will disable all output, if called from a script. Set using SetSilent()
	silent bool
	// outFile is where Out* functions send output to. Set using SetOutFile()
	outFile fdWriter
	// errFile is where Err* functions send output to. Set using SetErrFile()
	errFile fdWriter
	// useColor is whether or not color output should be used, updated by Set*Writer.
	useColor = false
	// OverrideEnv is the environment variable used to override color/emoji usage
	OverrideEnv = "MINIKUBE_IN_STYLE"
	// JSON is whether or not we should output stdout in JSON format. Set using SetJSON()
	JSON = false
	// spin is spinner showed at starting minikube
	spin = spinner.New(spinner.CharSets[style.SpinnerCharacter], 100*time.Millisecond)
)

// MaxLogEntries controls the number of log entries to show for each source
const (
	MaxLogEntries = 3
)

// fdWriter is the subset of file.File that implements io.Writer and Fd()
type fdWriter interface {
	io.Writer
	Fd() uintptr
}

// V is a convenience wrapper for templating, it represents the variable key/value pair.
type V map[string]interface{}

// Step writes a stylized and templated message to stdout
func Step(st style.Enum, format string, a ...V) {
	if st == style.Option {
		Infof(format, a...)
		return
	}
	outStyled, _ := stylized(st, useColor, format, a...)
	if JSON {
		register.PrintStep(outStyled)
		return
	}
	register.RecordStep(outStyled)
	Styled(st, format, a...)
}

// Styled writes a stylized and templated message to stdout
func Styled(st style.Enum, format string, a ...V) {
	if JSON || st == style.Option {
		Infof(format, a...)
		return
	}
	outStyled, spinner := stylized(st, useColor, format, a...)
	if spinner {
		spinnerString(outStyled)
	} else {
		String(outStyled)
	}
}

// Infof is used for informational logs (options, env variables, etc)
func Infof(format string, a ...V) {
	outStyled, _ := stylized(style.Option, useColor, format, a...)
	if JSON {
		register.PrintInfo(outStyled)
		return
	}
	String(outStyled)
}

// String writes a basic formatted string to stdout
func String(format string, a ...interface{}) {
	// Flush log buffer so that output order makes sense
	klog.Flush()

	if silent {
		klog.Infof(format, a...)
		return
	}

	if outFile == nil {
		klog.Warningf("[unset outFile]: %s", fmt.Sprintf(format, a...))
		return
	}
	klog.Infof(format, a...)
	// if spin is active from a previous step, it will stop spinner displaying
	if spin.Active() {
		spin.Stop()
	}

	Output(outFile, format, a...)
}

// Output writes a basic formatted string
func Output(file fdWriter, format string, a ...interface{}) {
	_, err := fmt.Fprintf(file, format, a...)
	if err != nil {
		klog.Errorf("Fprintf failed: %v", err)
	}
}

// spinnerString writes a basic formatted string to stdout with spinner character
func spinnerString(format string, a ...interface{}) {
	// Flush log buffer so that output order makes sense
	klog.Flush()

	if outFile == nil {
		klog.Warningf("[unset outFile]: %s", fmt.Sprintf(format, a...))
		return
	}

	klog.Infof(format, a...)
	// if spin is active from a previous step, it will stop spinner displaying
	if spin.Active() {
		spin.Stop()
	}
	Output(outFile, format, a...)
	// Start spinning at the end of the printed line
	spin.Start()
}

// Ln writes a basic formatted string with a newline to stdout
func Ln(format string, a ...interface{}) {
	if JSON {
		klog.Warningf("please use out.T to log steps in JSON")
		return
	}
	String(format+"\n", a...)
}

// ErrT writes a stylized and templated error message to stderr
func ErrT(st style.Enum, format string, a ...V) {
	errStyled, _ := stylized(st, useColor, format, a...)
	Err(errStyled)
}

// Err writes a basic formatted string to stderr
func Err(format string, a ...interface{}) {
	if JSON {
		register.PrintError(format)
		return
	}
	register.RecordError(format)

	if errFile == nil {
		klog.Errorf("[unset errFile]: %s", fmt.Sprintf(format, a...))
		return
	}

	klog.Warningf(format, a...)

	// if spin is active from a previous step, it will stop spinner displaying
	if spin.Active() {
		spin.Stop()
	}
	Output(errFile, format, a...)
}

// ErrLn writes a basic formatted string with a newline to stderr
func ErrLn(format string, a ...interface{}) {
	Err(format+"\n", a...)
}

// SuccessT is a shortcut for writing a templated success message to stdout
func SuccessT(format string, a ...V) {
	Step(style.Success, format, a...)
}

// FatalT is a shortcut for writing a templated fatal message to stderr
func FatalT(format string, a ...V) {
	ErrT(style.Fatal, format, a...)
}

// WarningT is a shortcut for writing a templated warning message to stderr
func WarningT(format string, a ...V) {
	if JSON {
		// if spin is active from a previous step, it will stop spinner displaying
		if spin.Active() {
			spin.Stop()
		}
		st, _ := stylized(style.Warning, useColor, format, a...)
		register.PrintWarning(st)
		return
	}
	ErrT(style.Warning, format, a...)
}

// FailureT is a shortcut for writing a templated failure message to stderr
func FailureT(format string, a ...V) {
	ErrT(style.Failure, format, a...)
}

// IsTerminal returns whether we have a terminal or not
func IsTerminal(w fdWriter) bool {
	fd := w.Fd()
	isT := isatty.IsTerminal(fd)
	klog.Infof("isatty.IsTerminal(%d) = %v\n", fd, isT)
	return isT
}

// SetSilent configures whether output is disabled or not
func SetSilent(q bool) {
	klog.Infof("Setting silent to %v", q)
	silent = q
}

// SetOutFile configures which writer standard output goes to.
func SetOutFile(w fdWriter) {
	klog.Infof("Setting OutFile to fd %d ...", w.Fd())
	outFile = w
	useColor = wantsColor(w.Fd())
}

// SetJSON configures printing to STDOUT in JSON
func SetJSON(j bool) {
	klog.Infof("Setting JSON to %v", j)
	JSON = j
}

// SetErrFile configures which writer error output goes to.
func SetErrFile(w fdWriter) {
	klog.Infof("Setting ErrFile to fd %d...", w.Fd())
	errFile = w
	useColor = wantsColor(w.Fd())
}

// wantsColor determines if the user might want colorized output.
func wantsColor(fd uintptr) bool {
	// First process the environment: we allow users to force colors on or off.
	//
	// MINIKUBE_IN_STYLE=[1, T, true, TRUE]
	// MINIKUBE_IN_STYLE=[0, f, false, FALSE]
	//
	// If unset, we try to automatically determine suitability from the environment.
	val := os.Getenv(OverrideEnv)
	if val != "" {
		klog.Infof("%s=%q\n", OverrideEnv, os.Getenv(OverrideEnv))
		override, err := strconv.ParseBool(val)
		if err != nil {
			// That's OK, we will just fall-back to automatic detection.
			klog.Errorf("ParseBool(%s): %v", OverrideEnv, err)
		} else {
			return override
		}
	}

	// New Windows Terminal
	if os.Getenv("WT_SESSION") != "" {
		return true
	}

	term := os.Getenv("TERM")
	colorTerm := os.Getenv("COLORTERM")
	// Example: term-256color
	if !strings.Contains(term, "color") && !strings.Contains(colorTerm, "truecolor") && !strings.Contains(colorTerm, "24bit") && !strings.Contains(colorTerm, "yes") {
		klog.Infof("TERM=%s,COLORTERM=%s, which probably does not support color", term, colorTerm)
		return false
	}

	isT := isatty.IsTerminal(fd)
	klog.Infof("isatty.IsTerminal(%d) = %v\n", fd, isT)
	return isT
}

// LogEntries outputs an error along with any important log entries.
func LogEntries(msg string, err error, entries map[string][]string) {
	DisplayError(msg, err)

	for name, lines := range entries {
		Step(style.Failure, "Problems detected in {{.entry}}:", V{"entry": name})
		if len(lines) > MaxLogEntries {
			lines = lines[:MaxLogEntries]
		}
		for _, l := range lines {
			Step(style.LogEntry, l)
		}
	}
}

// DisplayError prints the error and displays the standard minikube error messaging
func DisplayError(msg string, err error) {
	klog.Warningf(fmt.Sprintf("%s: %v", msg, err))
	if JSON {
		FatalT("{{.msg}}: {{.err}}", V{"msg": translate.T(msg), "err": err})
		return
	}
	// use Warning because Error will display a duplicate message to stderr
	ErrT(style.Empty, "")
	FatalT("{{.msg}}: {{.err}}", V{"msg": translate.T(msg), "err": err})
	ErrT(style.Empty, "")
	ErrT(style.Sad, "minikube is exiting due to an error. If the above message is not useful, open an issue:")
	ErrT(style.URL, "https://github.com/kubernetes/minikube/issues/new/choose")
}

// applyTmpl applies formatting
func applyTmpl(format string, a ...V) string {
	if len(a) == 0 {
		klog.Warningf("no arguments passed for %q - returning raw string", format)
		return format
	}

	var buf bytes.Buffer
	t, err := template.New(format).Parse(format)
	if err != nil {
		klog.Errorf("unable to parse %q: %v - returning raw string.", format, err)
		return format
	}
	err = t.Execute(&buf, a[0])
	if err != nil {
		klog.Errorf("unable to execute %s: %v - returning raw string.", format, err)
		return format
	}
	out := buf.String()

	// Return quotes back to normal
	out = html.UnescapeString(out)

	// escape any outstanding '%' signs so that they don't get interpreted
	// as a formatting directive down the line
	out = strings.Replace(out, "%", "%%", -1)
	// avoid doubling up in case this function is called multiple times
	out = strings.Replace(out, "%%%%", "%%", -1)
	return out
}

// Fmt applies formatting and translation
func Fmt(format string, a ...V) string {
	format = translate.T(format)
	return applyTmpl(format, a...)
}
<commit_msg>Use helper for IsTerminal also in wantsColor<commit_after>/*
Copyright 2019 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package out provides a mechanism for sending localized, stylized output to the console.
package out

import (
	"bytes"
	"fmt"
	"html"
	"html/template"
	"io"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/briandowns/spinner"
	isatty "github.com/mattn/go-isatty"

	"k8s.io/klog/v2"
	"k8s.io/minikube/pkg/minikube/out/register"
	"k8s.io/minikube/pkg/minikube/style"
	"k8s.io/minikube/pkg/minikube/translate"
)

// By design, this package uses global references to language and output objects, in preference
// to passing a console object throughout the code base. Typical usage is:
//
// out.SetOutFile(os.Stdout)
// out.String("Starting up!")
// out.Step(style.StatusChange, "Configuring things")

// out.SetErrFile(os.Stderr)
// out.Fatal("Oh no, everything failed.")

// NOTE: If you do not want colorized output, set MINIKUBE_IN_STYLE=false in your environment.

var (
	// silent will disable all output, if called from a script. Set using SetSilent()
	silent bool
	// outFile is where Out* functions send output to. Set using SetOutFile()
	outFile fdWriter
	// errFile is where Err* functions send output to. Set using SetErrFile()
	errFile fdWriter
	// useColor is whether or not color output should be used, updated by Set*Writer.
	useColor = false
	// OverrideEnv is the environment variable used to override color/emoji usage
	OverrideEnv = "MINIKUBE_IN_STYLE"
	// JSON is whether or not we should output stdout in JSON format. Set using SetJSON()
	JSON = false
	// spin is spinner showed at starting minikube
	spin = spinner.New(spinner.CharSets[style.SpinnerCharacter], 100*time.Millisecond)
)

// MaxLogEntries controls the number of log entries to show for each source
const (
	MaxLogEntries = 3
)

// fdWriter is the subset of file.File that implements io.Writer and Fd()
type fdWriter interface {
	io.Writer
	Fd() uintptr
}

// V is a convenience wrapper for templating, it represents the variable key/value pair.
type V map[string]interface{}

// Step writes a stylized and templated message to stdout
func Step(st style.Enum, format string, a ...V) {
	if st == style.Option {
		Infof(format, a...)
		return
	}
	outStyled, _ := stylized(st, useColor, format, a...)
	if JSON {
		register.PrintStep(outStyled)
		return
	}
	register.RecordStep(outStyled)
	Styled(st, format, a...)
}

// Styled writes a stylized and templated message to stdout
func Styled(st style.Enum, format string, a ...V) {
	if JSON || st == style.Option {
		Infof(format, a...)
		return
	}
	outStyled, spinner := stylized(st, useColor, format, a...)
	if spinner {
		spinnerString(outStyled)
	} else {
		String(outStyled)
	}
}

// Infof is used for informational logs (options, env variables, etc)
func Infof(format string, a ...V) {
	outStyled, _ := stylized(style.Option, useColor, format, a...)
	if JSON {
		register.PrintInfo(outStyled)
		return
	}
	String(outStyled)
}

// String writes a basic formatted string to stdout
func String(format string, a ...interface{}) {
	// Flush log buffer so that output order makes sense
	klog.Flush()

	if silent {
		klog.Infof(format, a...)
		return
	}

	if outFile == nil {
		klog.Warningf("[unset outFile]: %s", fmt.Sprintf(format, a...))
		return
	}
	klog.Infof(format, a...)
	// if spin is active from a previous step, it will stop spinner displaying
	if spin.Active() {
		spin.Stop()
	}

	Output(outFile, format, a...)
}

// Output writes a basic formatted string
func Output(file fdWriter, format string, a ...interface{}) {
	_, err := fmt.Fprintf(file, format, a...)
	if err != nil {
		klog.Errorf("Fprintf failed: %v", err)
	}
}

// spinnerString writes a basic formatted string to stdout with spinner character
func spinnerString(format string, a ...interface{}) {
	// Flush log buffer so that output order makes sense
	klog.Flush()

	if outFile == nil {
		klog.Warningf("[unset outFile]: %s", fmt.Sprintf(format, a...))
		return
	}

	klog.Infof(format, a...)
	// if spin is active from a previous step, it will stop spinner displaying
	if spin.Active() {
		spin.Stop()
	}
	Output(outFile, format, a...)
	// Start spinning at the end of the printed line
	spin.Start()
}

// Ln writes a basic formatted string with a newline to stdout
func Ln(format string, a ...interface{}) {
	if JSON {
		klog.Warningf("please use out.T to log steps in JSON")
		return
	}
	String(format+"\n", a...)
}

// ErrT writes a stylized and templated error message to stderr
func ErrT(st style.Enum, format string, a ...V) {
	errStyled, _ := stylized(st, useColor, format, a...)
	Err(errStyled)
}

// Err writes a basic formatted string to stderr
func Err(format string, a ...interface{}) {
	if JSON {
		register.PrintError(format)
		return
	}
	register.RecordError(format)

	if errFile == nil {
		klog.Errorf("[unset errFile]: %s", fmt.Sprintf(format, a...))
		return
	}

	klog.Warningf(format, a...)

	// if spin is active from a previous step, it will stop spinner displaying
	if spin.Active() {
		spin.Stop()
	}
	Output(errFile, format, a...)
}

// ErrLn writes a basic formatted string with a newline to stderr
func ErrLn(format string, a ...interface{}) {
	Err(format+"\n", a...)
}

// SuccessT is a shortcut for writing a templated success message to stdout
func SuccessT(format string, a ...V) {
	Step(style.Success, format, a...)
}

// FatalT is a shortcut for writing a templated fatal message to stderr
func FatalT(format string, a ...V) {
	ErrT(style.Fatal, format, a...)
}

// WarningT is a shortcut for writing a templated warning message to stderr
func WarningT(format string, a ...V) {
	if JSON {
		// if spin is active from a previous step, it will stop spinner displaying
		if spin.Active() {
			spin.Stop()
		}
		st, _ := stylized(style.Warning, useColor, format, a...)
		register.PrintWarning(st)
		return
	}
	ErrT(style.Warning, format, a...)
}

// FailureT is a shortcut for writing a templated failure message to stderr
func FailureT(format string, a ...V) {
	ErrT(style.Failure, format, a...)
}

// IsTerminal returns whether we have a terminal or not
func IsTerminal(w fdWriter) bool {
	return isatty.IsTerminal(w.Fd())
}

// SetSilent configures whether output is disabled or not
func SetSilent(q bool) {
	klog.Infof("Setting silent to %v", q)
	silent = q
}

// SetOutFile configures which writer standard output goes to.
func SetOutFile(w fdWriter) {
	klog.Infof("Setting OutFile to fd %d ...", w.Fd())
	outFile = w
	useColor = wantsColor(w)
}

// SetJSON configures printing to STDOUT in JSON
func SetJSON(j bool) {
	klog.Infof("Setting JSON to %v", j)
	JSON = j
}

// SetErrFile configures which writer error output goes to.
func SetErrFile(w fdWriter) {
	klog.Infof("Setting ErrFile to fd %d...", w.Fd())
	errFile = w
	useColor = wantsColor(w)
}

// wantsColor determines if the user might want colorized output.
func wantsColor(w fdWriter) bool {
	// First process the environment: we allow users to force colors on or off.
	//
	// MINIKUBE_IN_STYLE=[1, T, true, TRUE]
	// MINIKUBE_IN_STYLE=[0, f, false, FALSE]
	//
	// If unset, we try to automatically determine suitability from the environment.
	val := os.Getenv(OverrideEnv)
	if val != "" {
		klog.Infof("%s=%q\n", OverrideEnv, os.Getenv(OverrideEnv))
		override, err := strconv.ParseBool(val)
		if err != nil {
			// That's OK, we will just fall-back to automatic detection.
			klog.Errorf("ParseBool(%s): %v", OverrideEnv, err)
		} else {
			return override
		}
	}

	// New Windows Terminal
	if os.Getenv("WT_SESSION") != "" {
		return true
	}

	term := os.Getenv("TERM")
	colorTerm := os.Getenv("COLORTERM")
	// Example: term-256color
	if !strings.Contains(term, "color") && !strings.Contains(colorTerm, "truecolor") && !strings.Contains(colorTerm, "24bit") && !strings.Contains(colorTerm, "yes") {
		klog.Infof("TERM=%s,COLORTERM=%s, which probably does not support color", term, colorTerm)
		return false
	}

	isT := IsTerminal(w)
	klog.Infof("isatty.IsTerminal(%d) = %v\n", w.Fd(), isT)
	return isT
}

// LogEntries outputs an error along with any important log entries.
func LogEntries(msg string, err error, entries map[string][]string) {
	DisplayError(msg, err)

	for name, lines := range entries {
		Step(style.Failure, "Problems detected in {{.entry}}:", V{"entry": name})
		if len(lines) > MaxLogEntries {
			lines = lines[:MaxLogEntries]
		}
		for _, l := range lines {
			Step(style.LogEntry, l)
		}
	}
}

// DisplayError prints the error and displays the standard minikube error messaging
func DisplayError(msg string, err error) {
	klog.Warningf(fmt.Sprintf("%s: %v", msg, err))
	if JSON {
		FatalT("{{.msg}}: {{.err}}", V{"msg": translate.T(msg), "err": err})
		return
	}
	// use Warning because Error will display a duplicate message to stderr
	ErrT(style.Empty, "")
	FatalT("{{.msg}}: {{.err}}", V{"msg": translate.T(msg), "err": err})
	ErrT(style.Empty, "")
	ErrT(style.Sad, "minikube is exiting due to an error. If the above message is not useful, open an issue:")
	ErrT(style.URL, "https://github.com/kubernetes/minikube/issues/new/choose")
}

// applyTmpl applies formatting
func applyTmpl(format string, a ...V) string {
	if len(a) == 0 {
		klog.Warningf("no arguments passed for %q - returning raw string", format)
		return format
	}

	var buf bytes.Buffer
	t, err := template.New(format).Parse(format)
	if err != nil {
		klog.Errorf("unable to parse %q: %v - returning raw string.", format, err)
		return format
	}
	err = t.Execute(&buf, a[0])
	if err != nil {
		klog.Errorf("unable to execute %s: %v - returning raw string.", format, err)
		return format
	}
	out := buf.String()

	// Return quotes back to normal
	out = html.UnescapeString(out)

	// escape any outstanding '%' signs so that they don't get interpreted
	// as a formatting directive down the line
	out = strings.Replace(out, "%", "%%", -1)
	// avoid doubling up in case this function is called multiple times
	out = strings.Replace(out, "%%%%", "%%", -1)
	return out
}

// Fmt applies formatting and translation
func Fmt(format string, a ...V) string {
	format = translate.T(format)
	return applyTmpl(format, a...)
}
<|endoftext|>


<commit_before>// Copyright 2016 DeepFabric, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package pdserver

import (
	"errors"
	"sync"
	"time"

	"github.com/deepfabric/elasticell/pkg/codec"
	"github.com/deepfabric/elasticell/pkg/log"
	"github.com/deepfabric/elasticell/pkg/pb/pdpb"
	"github.com/deepfabric/elasticell/pkg/util"
	"github.com/fagongzi/goetty"
)

var (
	errConnect = errors.New("not connected")
)

const (
	ready = iota
	paused
)

func cmp(a, b interface{}) bool {
	return a.(*notify).watcher == b.(*notify).watcher
}

type notify struct {
	watcher string
	offset  uint64
}

func newWatcherState(watcher string) *watcherState {
	return &watcherState{
		watcher: watcher,
		state:   ready,
		q:       util.NewOffsetQueue(),
	}
}

type watcherState struct {
	watcher          string
	state            int
	hbTimeout        *goetty.Timeout
	q                *util.OffsetQueue
	lastNotifyOffset uint64
}

func (state *watcherState) reset() {
	state.cancelTimeout()
	state.state = ready
	state.q = util.NewOffsetQueue()
	state.lastNotifyOffset = 0

	log.Debugf("notify: %s reseted", state.watcher)
}

func (state *watcherState) pause() {
	state.reset()
	state.state = paused
	log.Debugf("notify: %s reset to pause", state.watcher)
}

func (state *watcherState) isReady() bool {
	return state.state == ready
}

func (state *watcherState) isPause() bool {
	return state.state == paused
}

func (state *watcherState) addNotify(info *pdpb.Range) uint64 {
	return state.q.Add(info)
}

func (state *watcherState) cancelTimeout() {
	if state.hbTimeout != nil {
		state.hbTimeout.Stop()
		state.hbTimeout = nil
	}
}

func (state *watcherState) resetTimeout(timeout time.Duration, fn func(interface{})) {
	state.cancelTimeout()

	t, _ := util.DefaultTimeoutWheel().Schedule(timeout, fn, state.watcher)
	state.hbTimeout = &t
}

// watcherNotifier is used for notify the newest cell info to all watchers
type watcherNotifier struct {
	sync.RWMutex

	notifies *util.Queue
	pool     *goetty.AddressBasedPool
	watchers map[string]*watcherState
	timeout  time.Duration
}

func newWatcherNotifier(timeout time.Duration) *watcherNotifier {
	wn := &watcherNotifier{
		notifies: util.New(1024),
		watchers: make(map[string]*watcherState),
		timeout:  timeout,
	}

	wn.pool = goetty.NewAddressBasedPool(createConn, wn)
	return wn
}

func (wn *watcherNotifier) start() {
	go func() {
		batch := int64(64)
		items := make([]interface{}, batch, batch)

		for {
			n, err := wn.notifies.Get(int64(batch), items)
			if err != nil {
				log.Infof("stop: watcher notifier stopped")
				return
			}

			for i := int64(0); i < n; i++ {
				nt := items[i].(*notify)

				if !wn.allowNotify(nt.watcher) {
					continue
				}

				if !wn.allowSend(nt) {
					continue
				}

				conn, err := wn.pool.GetConn(nt.watcher)
				if err != nil {
					wn.pause(nt.watcher, "get conn error")
					log.Warnf("notify: %d to %s failed, errors:\n%+v",
						nt.offset,
						nt.watcher,
						err)
					continue
				}

				req := &pdpb.WatcherNotify{
					Offset: nt.offset,
				}
				err = conn.Write(req)
				if err != nil {
					wn.pause(nt.watcher, "write error")
					log.Warnf("notify: %d to %s failed, errors:\n%+v",
						nt.offset,
						nt.watcher,
						err)
					continue
				}
			}
		}
	}()
}

func (wn *watcherNotifier) stop() {
	wn.notifies.Dispose()
}

func (wn *watcherNotifier) removedAllWatcher() {
	wn.Lock()
	for watcher := range wn.watchers {
		wn.removeWatcher(watcher)
	}
	wn.Unlock()
}

// addWatcher add a new watcher for notify the newest cells info.
func (wn *watcherNotifier) addWatcher(addr string) {
	wn.Lock()

	if state, ok := wn.watchers[addr]; ok {
		state.cancelTimeout()
		state.reset()
	} else {
		wn.watchers[addr] = newWatcherState(addr)
	}

	wn.resetTimeout(addr)
	wn.Unlock()

	log.Warnf("notify: %s added", addr)
}

func (wn *watcherNotifier) resetTimeout(addr string) {
	wn.watchers[addr].resetTimeout(wn.timeout, wn.watcherTimeout)
}

// removeWatcher remove a watcher
func (wn *watcherNotifier) removeWatcher(addr string) {
	wn.Lock()
	delete(wn.watchers, addr)
	wn.pool.RemoveConn(addr)
	wn.Unlock()
}

// watcherHeartbeat return true if the watcher resume from pause.
func (wn *watcherNotifier) watcherHeartbeat(addr string, offset uint64) bool {
	wn.Lock()
	value := wn.resume(addr)
	if state, ok := wn.watchers[addr]; ok && !value {
		max := state.q.GetMaxOffset()
		if offset < max {
			wn.notifies.Put(&notify{
				watcher: addr,
				offset:  max,
			})
		}
	}
	wn.Unlock()
	return value
}

func (wn *watcherNotifier) watcherTimeout(arg interface{}) {
	wn.pause(arg.(string), "hb timeout")
}

func (wn *watcherNotifier) allowNotify(addr string) bool {
	wn.Lock()
	if state, ok := wn.watchers[addr]; ok {
		allow := state.isReady()
		wn.Unlock()
		return allow
	}
	wn.Unlock()
	return false
}

func (wn *watcherNotifier) allowSend(nt *notify) bool {
	wn.Lock()
	if state, ok := wn.watchers[nt.watcher]; ok {
		allow := state.isReady() && state.lastNotifyOffset == 0 || state.lastNotifyOffset < nt.offset
		if allow {
			state.lastNotifyOffset = nt.offset
		}
		wn.Unlock()
		return allow
	}
	wn.Unlock()
	return false
}

func (wn *watcherNotifier) pause(addr string, reason string) {
	wn.Lock()
	wn.pool.RemoveConn(addr)
	if state, ok := wn.watchers[addr]; ok {
		state.pause()
	}
	wn.Unlock()
}

func (wn *watcherNotifier) resume(addr string) bool {
	if state, ok := wn.watchers[addr]; ok {
		resumeFromPause := state.isPause()

		if state.isPause() {
			state.state = ready
		}

		wn.resetTimeout(addr)
		return resumeFromPause
	}

	return false
}

func (wn *watcherNotifier) notify(info *pdpb.Range) {
	wn.Lock()

	for _, state := range wn.watchers {
		if state.isReady() {
			nt := &notify{
				watcher: state.watcher,
				offset:  state.addNotify(info),
			}
			wn.notifies.PutOrUpdate(cmp, nt)
		}
	}

	wn.Unlock()
}

func (wn *watcherNotifier) sync(addr string, offset uint64) *pdpb.WatcherNotifyRsp {
	wn.Lock()

	if state, ok := wn.watchers[addr]; ok && state.isReady() {
		items, max := state.q.Get(offset)
		rsp := new(pdpb.WatcherNotifyRsp)
		for _, item := range items {
			rsp.Ranges = append(rsp.Ranges, item.(*pdpb.Range))
		}
		rsp.Offset = max
		wn.Unlock()
		return rsp
	}

	wn.Unlock()
	return nil
}

// ConnectFailed pool status handler
func (wn *watcherNotifier) ConnectFailed(addr string, err error) {
	wn.pause(addr, "connect failed")
}

// Connected pool status handler
func (wn *watcherNotifier) Connected(addr string, conn goetty.IOSession) {
	go func() {
		for {
			msg, err := conn.Read()
			if err != nil {
				log.Errorf("notify: read from %s failed, errors:\n%+v", addr, err)
				wn.pause(addr, "read error")
				return
			}

			if s, ok := msg.(*pdpb.WatcherNotifySync); ok {
				err := conn.Write(wn.sync(addr, s.Offset))
				if err != nil {
					wn.pause(addr, "write sync")
					return
				}
			}
		}
	}()
}

func createConn(addr string) goetty.IOSession {
	c := &goetty.Conf{
		Addr: addr,
		TimeoutConnectToServer: time.Second * 30,
	}
	return goetty.NewConnector(c, &codec.ProxyDecoder{}, &codec.ProxyEncoder{})
}
<commit_msg>fix: fix test failed<commit_after>// Copyright 2016 DeepFabric, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package pdserver

import (
	"errors"
	"sync"
	"time"

	"github.com/deepfabric/elasticell/pkg/codec"
	"github.com/deepfabric/elasticell/pkg/log"
	"github.com/deepfabric/elasticell/pkg/pb/pdpb"
	"github.com/deepfabric/elasticell/pkg/util"
	"github.com/fagongzi/goetty"
)

var (
	errConnect = errors.New("not connected")
)

const (
	ready = iota
	paused
)

func cmp(a, b interface{}) bool {
	return a.(*notify).watcher == b.(*notify).watcher
}

type notify struct {
	watcher string
	offset  uint64
}

func newWatcherState(watcher string) *watcherState {
	return &watcherState{
		watcher: watcher,
		state:   ready,
		q:       util.NewOffsetQueue(),
	}
}

type watcherState struct {
	watcher          string
	state            int
	hbTimeout        *goetty.Timeout
	q                *util.OffsetQueue
	lastNotifyOffset uint64
}

func (state *watcherState) reset() {
	state.cancelTimeout()
	state.state = ready
	state.q = util.NewOffsetQueue()
	state.lastNotifyOffset = 0

	log.Debugf("notify: %s reseted", state.watcher)
}

func (state *watcherState) pause() {
	state.reset()
	state.state = paused
	log.Debugf("notify: %s reset to pause", state.watcher)
}

func (state *watcherState) isReady() bool {
	return state.state == ready
}

func (state *watcherState) isPause() bool {
	return state.state == paused
}

func (state *watcherState) addNotify(info *pdpb.Range) uint64 {
	return state.q.Add(info)
}

func (state *watcherState) cancelTimeout() {
	if state.hbTimeout != nil {
		state.hbTimeout.Stop()
		state.hbTimeout = nil
	}
}

func (state *watcherState) resetTimeout(timeout time.Duration, fn func(interface{})) {
	state.cancelTimeout()

	t, _ := util.DefaultTimeoutWheel().Schedule(timeout, fn, state.watcher)
	state.hbTimeout = &t
}

// watcherNotifier is used for notify the newest cell info to all watchers
type watcherNotifier struct {
	sync.RWMutex

	notifies *util.Queue
	pool     *goetty.AddressBasedPool
	watchers map[string]*watcherState
	timeout  time.Duration
}

func newWatcherNotifier(timeout time.Duration) *watcherNotifier {
	wn := &watcherNotifier{
		notifies: util.New(1024),
		watchers: make(map[string]*watcherState),
		timeout:  timeout,
	}

	wn.pool = goetty.NewAddressBasedPool(createConn, wn)
	return wn
}

func (wn *watcherNotifier) start() {
	go func() {
		batch := int64(64)
		items := make([]interface{}, batch, batch)

		for {
			n, err := wn.notifies.Get(int64(batch), items)
			if err != nil {
				log.Infof("stop: watcher notifier stopped")
				return
			}

			for i := int64(0); i < n; i++ {
				nt := items[i].(*notify)

				if !wn.allowNotify(nt.watcher) {
					continue
				}

				if !wn.allowSend(nt) {
					continue
				}

				conn, err := wn.pool.GetConn(nt.watcher)
				if err != nil {
					wn.pause(nt.watcher)
					log.Warnf("notify: %d to %s failed, errors:\n%+v",
						nt.offset,
						nt.watcher,
						err)
					continue
				}

				req := &pdpb.WatcherNotify{
					Offset: nt.offset,
				}
				err = conn.Write(req)
				if err != nil {
					wn.pause(nt.watcher)
					log.Warnf("notify: %d to %s failed, errors:\n%+v",
						nt.offset,
						nt.watcher,
						err)
					continue
				}
			}
		}
	}()
}

func (wn *watcherNotifier) stop() {
	wn.notifies.Dispose()
}

func (wn *watcherNotifier) removedAllWatcher() {
	wn.Lock()
	for watcher := range wn.watchers {
		wn.removeWatcher(watcher)
	}
	wn.Unlock()
}

// addWatcher add a new watcher for notify the newest cells info.
func (wn *watcherNotifier) addWatcher(addr string) {
	wn.Lock()

	if state, ok := wn.watchers[addr]; ok {
		state.cancelTimeout()
		state.reset()
	} else {
		wn.watchers[addr] = newWatcherState(addr)
	}

	wn.resetTimeout(addr)
	wn.Unlock()

	log.Warnf("notify: %s added", addr)
}

func (wn *watcherNotifier) resetTimeout(addr string) {
	wn.watchers[addr].resetTimeout(wn.timeout, wn.watcherTimeout)
}

// removeWatcher remove a watcher
func (wn *watcherNotifier) removeWatcher(addr string) {
	wn.Lock()
	delete(wn.watchers, addr)
	wn.pool.RemoveConn(addr)
	wn.Unlock()
}

// watcherHeartbeat return true if the watcher resume from pause.
func (wn *watcherNotifier) watcherHeartbeat(addr string, offset uint64) bool {
	wn.Lock()
	value := wn.resume(addr)
	if state, ok := wn.watchers[addr]; ok && !value {
		max := state.q.GetMaxOffset()
		if offset < max {
			wn.notifies.Put(&notify{
				watcher: addr,
				offset:  max,
			})
		}
	}
	wn.Unlock()
	return value
}

func (wn *watcherNotifier) watcherTimeout(arg interface{}) {
	wn.pause(arg.(string))
}

func (wn *watcherNotifier) allowNotify(addr string) bool {
	wn.Lock()
	if state, ok := wn.watchers[addr]; ok {
		allow := state.isReady()
		wn.Unlock()
		return allow
	}
	wn.Unlock()
	return false
}

func (wn *watcherNotifier) allowSend(nt *notify) bool {
	wn.Lock()
	if state, ok := wn.watchers[nt.watcher]; ok {
		allow := state.isReady() && state.lastNotifyOffset == 0 || state.lastNotifyOffset < nt.offset
		if allow {
			state.lastNotifyOffset = nt.offset
		}
		wn.Unlock()
		return allow
	}
	wn.Unlock()
	return false
}

func (wn *watcherNotifier) pause(addr string) {
	wn.Lock()
	wn.pool.RemoveConn(addr)
	if state, ok := wn.watchers[addr]; ok {
		state.pause()
	}
	wn.Unlock()
}

func (wn *watcherNotifier) resume(addr string) bool {
	if state, ok := wn.watchers[addr]; ok {
		resumeFromPause := state.isPause()

		if state.isPause() {
			state.state = ready
		}

		wn.resetTimeout(addr)
		return resumeFromPause
	}

	return false
}

func (wn *watcherNotifier) notify(info *pdpb.Range) {
	wn.Lock()

	for _, state := range wn.watchers {
		if state.isReady() {
			nt := &notify{
				watcher: state.watcher,
				offset:  state.addNotify(info),
			}
			wn.notifies.PutOrUpdate(cmp, nt)
		}
	}

	wn.Unlock()
}

func (wn *watcherNotifier) sync(addr string, offset uint64) *pdpb.WatcherNotifyRsp {
	wn.Lock()

	if state, ok := wn.watchers[addr]; ok && state.isReady() {
		items, max := state.q.Get(offset)
		rsp := new(pdpb.WatcherNotifyRsp)
		for _, item := range items {
			rsp.Ranges = append(rsp.Ranges, item.(*pdpb.Range))
		}
		rsp.Offset = max
		wn.Unlock()
		return rsp
	}

	wn.Unlock()
	return nil
}

// ConnectFailed pool status handler
func (wn *watcherNotifier) ConnectFailed(addr string, err error) {
	wn.pause(addr)
}

// Connected pool status handler
func (wn *watcherNotifier) Connected(addr string, conn goetty.IOSession) {
	go func() {
		for {
			msg, err := conn.Read()
			if err != nil {
				log.Errorf("notify: read from %s failed, errors:\n%+v", addr, err)
				wn.pause(addr)
				return
			}

			if s, ok := msg.(*pdpb.WatcherNotifySync); ok {
				err := conn.Write(wn.sync(addr, s.Offset))
				if err != nil {
					wn.pause(addr)
					return
				}
			}
		}
	}()
}

func createConn(addr string) goetty.IOSession {
	c := &goetty.Conf{
		Addr: addr,
		TimeoutConnectToServer: time.Second * 30,
	}
	return goetty.NewConnector(c, &codec.ProxyDecoder{}, &codec.ProxyEncoder{})
}
<|endoftext|>


<commit_before>package promclient

import (
	"context"
	"time"

	"github.com/prometheus/client_golang/api"
	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
	"github.com/prometheus/common/model"
	"github.com/prometheus/prometheus/pkg/labels"
	"github.com/sirupsen/logrus"
)

// DebugAPI simply logs debug lines for the given API with the given prefix
type DebugAPI struct {
	API
	PrefixMessage string
}

// LabelValues performs a query for the values of the given label.
func (d *DebugAPI) LabelValues(ctx context.Context, label string) (model.LabelValues, api.Warnings, error) {
	logrus.WithFields(logrus.Fields{
		"api":   "LabelValues",
		"label": label,
	}).Debug(d.PrefixMessage)

	v, w, err := d.API.LabelValues(ctx, label)

	logrus.WithFields(logrus.Fields{
		"api":      "LabelValues",
		"label":    label,
		"value":    v,
		"warnings": w,
		"error":    err,
	}).Trace(d.PrefixMessage)

	return v, w, err
}

// Query performs a query for the given time.
func (d *DebugAPI) Query(ctx context.Context, query string, ts time.Time) (model.Value, api.Warnings, error) {
	logrus.WithFields(logrus.Fields{
		"api":   "Query",
		"query": query,
		"ts":    ts,
	}).Debug(d.PrefixMessage)

	v, w, err := d.API.Query(ctx, query, ts)

	logrus.WithFields(logrus.Fields{
		"api":      "Query",
		"query":    query,
		"ts":       ts,
		"value":    v,
		"warnings": w,
		"error":    err,
	}).Trace(d.PrefixMessage)

	return v, w, err
}

// QueryRange performs a query for the given range.
func (d *DebugAPI) QueryRange(ctx context.Context, query string, r v1.Range) (model.Value, api.Warnings, error) {
	logrus.WithFields(logrus.Fields{
		"api":   "QueryRange",
		"query": query,
		"r":     r,
	}).Debug(d.PrefixMessage)

	v, w, err := d.API.QueryRange(ctx, query, r)

	logrus.WithFields(logrus.Fields{
		"api":      "QueryRange",
		"query":    query,
		"r":        r,
		"value":    v,
		"warnings": w,
		"error":    err,
	}).Trace(d.PrefixMessage)

	return v, w, err
}

// Series finds series by label matchers.
func (d *DebugAPI) Series(ctx context.Context, matches []string, startTime time.Time, endTime time.Time) ([]model.LabelSet, api.Warnings, error) {
	logrus.WithFields(logrus.Fields{
		"api":       "Series",
		"matches":   matches,
		"startTime": startTime,
		"endTime":   endTime,
	}).Debug(d.PrefixMessage)

	v, w, err := d.API.Series(ctx, matches, startTime, endTime)

	logrus.WithFields(logrus.Fields{
		"api":       "Series",
		"matches":   matches,
		"startTime": startTime,
		"endTime":   endTime,
		"value":     v,
		"warnings":  w,
		"error":     err,
	}).Trace(d.PrefixMessage)

	return v, w, err
}

// GetValue loads the raw data for a given set of matchers in the time range
func (d *DebugAPI) GetValue(ctx context.Context, start, end time.Time, matchers []*labels.Matcher) (model.Value, api.Warnings, error) {
	logrus.WithFields(logrus.Fields{
		"api":      "GetValue",
		"start":    start,
		"end":      end,
		"matchers": matchers,
	}).Debug(d.PrefixMessage)

	v, w, err := d.API.GetValue(ctx, start, end, matchers)

	logrus.WithFields(logrus.Fields{
		"api":      "GetValue",
		"start":    start,
		"end":      end,
		"matchers": matchers,
		"value":    v,
		"warnings": w,
		"error":    err,
	}).Trace(d.PrefixMessage)

	return v, w, err
}
<commit_msg>Add timings to debug logging of servergroups<commit_after>package promclient

import (
	"context"
	"time"

	"github.com/prometheus/client_golang/api"
	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
	"github.com/prometheus/common/model"
	"github.com/prometheus/prometheus/pkg/labels"
	"github.com/sirupsen/logrus"
)

// DebugAPI simply logs debug lines for the given API with the given prefix
type DebugAPI struct {
	API
	PrefixMessage string
}

// LabelValues performs a query for the values of the given label.
func (d *DebugAPI) LabelValues(ctx context.Context, label string) (model.LabelValues, api.Warnings, error) {
	fields := logrus.Fields{
		"api":   "LabelValues",
		"label": label,
	}
	logrus.WithFields(fields).Debug(d.PrefixMessage)

	s := time.Now()
	v, w, err := d.API.LabelValues(ctx, label)
	fields["took"] = time.Now().Sub(s)

	if logrus.GetLevel() > logrus.DebugLevel {
		fields["value"] = v
		fields["warnings"] = w
		fields["error"] = err
		logrus.WithFields(fields).Trace(d.PrefixMessage)
	} else {
		logrus.WithFields(fields).Debug(d.PrefixMessage)
	}

	return v, w, err
}

// Query performs a query for the given time.
func (d *DebugAPI) Query(ctx context.Context, query string, ts time.Time) (model.Value, api.Warnings, error) {
	fields := logrus.Fields{
		"api":   "Query",
		"query": query,
		"ts":    ts,
	}
	logrus.WithFields(fields).Debug(d.PrefixMessage)

	s := time.Now()
	v, w, err := d.API.Query(ctx, query, ts)
	fields["took"] = time.Now().Sub(s)

	if logrus.GetLevel() > logrus.DebugLevel {
		fields["value"] = v
		fields["warnings"] = w
		fields["error"] = err
		logrus.WithFields(fields).Trace(d.PrefixMessage)
	} else {
		logrus.WithFields(fields).Debug(d.PrefixMessage)
	}

	return v, w, err
}

// QueryRange performs a query for the given range.
func (d *DebugAPI) QueryRange(ctx context.Context, query string, r v1.Range) (model.Value, api.Warnings, error) {
	fields := logrus.Fields{
		"api":   "QueryRange",
		"query": query,
		"r":     r,
	}
	logrus.WithFields(fields).Debug(d.PrefixMessage)

	s := time.Now()
	v, w, err := d.API.QueryRange(ctx, query, r)
	fields["took"] = time.Now().Sub(s)

	if logrus.GetLevel() > logrus.DebugLevel {
		fields["value"] = v
		fields["warnings"] = w
		fields["error"] = err
		logrus.WithFields(fields).Trace(d.PrefixMessage)
	} else {
		logrus.WithFields(fields).Debug(d.PrefixMessage)
	}

	return v, w, err
}

// Series finds series by label matchers.
func (d *DebugAPI) Series(ctx context.Context, matches []string, startTime time.Time, endTime time.Time) ([]model.LabelSet, api.Warnings, error) {
	fields := logrus.Fields{
		"api":       "Series",
		"matches":   matches,
		"startTime": startTime,
		"endTime":   endTime,
	}
	logrus.WithFields(fields).Debug(d.PrefixMessage)

	s := time.Now()
	v, w, err := d.API.Series(ctx, matches, startTime, endTime)
	fields["took"] = time.Now().Sub(s)

	if logrus.GetLevel() > logrus.DebugLevel {
		fields["value"] = v
		fields["warnings"] = w
		fields["error"] = err
		logrus.WithFields(fields).Trace(d.PrefixMessage)
	} else {
		logrus.WithFields(fields).Debug(d.PrefixMessage)
	}
	return v, w, err
}

// GetValue loads the raw data for a given set of matchers in the time range
func (d *DebugAPI) GetValue(ctx context.Context, start, end time.Time, matchers []*labels.Matcher) (model.Value, api.Warnings, error) {
	fields := logrus.Fields{
		"api":      "GetValue",
		"start":    start,
		"end":      end,
		"matchers": matchers,
	}

	logrus.WithFields(fields).Debug(d.PrefixMessage)

	s := time.Now()
	v, w, err := d.API.GetValue(ctx, start, end, matchers)
	fields["took"] = time.Now().Sub(s)

	if logrus.GetLevel() > logrus.DebugLevel {
		fields["value"] = v
		fields["warnings"] = w
		fields["error"] = err
		logrus.WithFields(fields).Trace(d.PrefixMessage)
	} else {
		logrus.WithFields(fields).Debug(d.PrefixMessage)
	}

	return v, w, err
}
<|endoftext|>