<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cache

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"

	"golang.org/x/mod/modfile"
	"golang.org/x/tools/internal/event"
	"golang.org/x/tools/internal/lsp/debug/tag"
	"golang.org/x/tools/internal/lsp/protocol"
	"golang.org/x/tools/internal/lsp/source"
	"golang.org/x/tools/internal/memoize"
	"golang.org/x/tools/internal/span"
	errors "golang.org/x/xerrors"
)

const (
	SyntaxError = "syntax"
)

type parseModHandle struct {
	handle *memoize.Handle
}

type parseModData struct {
	parsed *source.ParsedModule

	// err is any error encountered while parsing the file.
	err error
}

func (mh *parseModHandle) parse(ctx context.Context, snapshot *snapshot) (*source.ParsedModule, error) {
	v, err := mh.handle.Get(ctx, snapshot.generation, snapshot)
	if err != nil {
		return nil, err
	}
	data := v.(*parseModData)
	return data.parsed, data.err
}

func (s *snapshot) ParseMod(ctx context.Context, modFH source.FileHandle) (*source.ParsedModule, error) {
	if handle := s.getParseModHandle(modFH.URI()); handle != nil {
		return handle.parse(ctx, s)
	}
	h := s.generation.Bind(modFH.FileIdentity(), func(ctx context.Context, _ memoize.Arg) interface{} {
		_, done := event.Start(ctx, "cache.ParseModHandle", tag.URI.Of(modFH.URI()))
		defer done()

		contents, err := modFH.Read()
		if err != nil {
			return &parseModData{err: err}
		}
		m := &protocol.ColumnMapper{
			URI:       modFH.URI(),
			Converter: span.NewContentConverter(modFH.URI().Filename(), contents),
			Content:   contents,
		}
		data := &parseModData{
			parsed: &source.ParsedModule{
				Mapper: m,
			},
		}
		data.parsed.File, data.err = modfile.Parse(modFH.URI().Filename(), contents, nil)
		if data.err != nil {
			// Attempt to convert the error to a standardized parse error.
			if parseErr, extractErr := extractModParseErrors(modFH.URI(), m, data.err, contents); extractErr == nil {
				data.parsed.ParseErrors = []source.Error{*parseErr}
			}
			// If the file was still parsed, we don't want to treat this as a
			// fatal error. Note: This currently cannot happen as modfile.Parse
			// always returns an error when the file is nil.
			if data.parsed.File != nil {
				data.err = nil
			}
		}
		return data
	})

	pmh := &parseModHandle{handle: h}
	s.mu.Lock()
	s.parseModHandles[modFH.URI()] = pmh
	s.mu.Unlock()

	return pmh.parse(ctx, s)
}

func (s *snapshot) sumFH(ctx context.Context, modFH source.FileHandle) (source.FileHandle, error) {
	// Get the go.sum file, either from the snapshot or directly from the
	// cache. Avoid (*snapshot).GetFile here, as we don't want to add
	// nonexistent file handles to the snapshot if the file does not exist.
	sumURI := span.URIFromPath(sumFilename(modFH.URI()))
	var sumFH source.FileHandle = s.FindFile(sumURI)
	if sumFH == nil {
		var err error
		sumFH, err = s.view.session.cache.getFile(ctx, sumURI)
		if err != nil {
			return nil, err
		}
	}
	_, err := sumFH.Read()
	if err != nil {
		return nil, err
	}
	return sumFH, nil
}

func sumFilename(modURI span.URI) string {
	return strings.TrimSuffix(modURI.Filename(), ".mod") + ".sum"
}

// extractModParseErrors processes the raw errors returned by modfile.Parse,
// extracting the filenames and line numbers that correspond to the errors.
func extractModParseErrors(uri span.URI, m *protocol.ColumnMapper, parseErr error, content []byte) (*source.Error, error) {
	re := regexp.MustCompile(`.*:([\d]+): (.+)`)
	matches := re.FindStringSubmatch(strings.TrimSpace(parseErr.Error()))
	if len(matches) < 3 {
		return nil, errors.Errorf("could not parse go.mod error message: %s", parseErr)
	}
	line, err := strconv.Atoi(matches[1])
	if err != nil {
		return nil, err
	}
	lines := strings.Split(string(content), "\n")
	if line > len(lines) {
		return nil, errors.Errorf("could not parse go.mod error message %q, line number %v out of range", content, line)
	}
	// The error returned from the modfile package only returns a line number,
	// so we assume that the diagnostic should be for the entire line.
	endOfLine := len(lines[line-1])
	sOffset, err := m.Converter.ToOffset(line, 0)
	if err != nil {
		return nil, err
	}
	eOffset, err := m.Converter.ToOffset(line, endOfLine)
	if err != nil {
		return nil, err
	}
	spn := span.New(uri, span.NewPoint(line, 0, sOffset), span.NewPoint(line, endOfLine, eOffset))
	rng, err := m.Range(spn)
	if err != nil {
		return nil, err
	}
	return &source.Error{
		Category: SyntaxError,
		Message:  matches[2],
		Range:    rng,
		URI:      uri,
	}, nil
}

// modKey is uniquely identifies cached data for `go mod why` or dependencies
// to upgrade.
type modKey struct {
	sessionID, cfg, view string
	mod                  source.FileIdentity
	verb                 modAction
}

type modAction int

const (
	why modAction = iota
	upgrade
)

type modWhyHandle struct {
	handle *memoize.Handle
}

type modWhyData struct {
	// why keeps track of the `go mod why` results for each require statement
	// in the go.mod file.
	why map[string]string

	err error
}

func (mwh *modWhyHandle) why(ctx context.Context, snapshot *snapshot) (map[string]string, error) {
	v, err := mwh.handle.Get(ctx, snapshot.generation, snapshot)
	if err != nil {
		return nil, err
	}
	data := v.(*modWhyData)
	return data.why, data.err
}

func (s *snapshot) ModWhy(ctx context.Context, fh source.FileHandle) (map[string]string, error) {
	if err := s.awaitLoaded(ctx); err != nil {
		return nil, err
	}
	if handle := s.getModWhyHandle(fh.URI()); handle != nil {
		return handle.why(ctx, s)
	}
	// Make sure to use the module root as the working directory.
	cfg := s.configWithDir(ctx, filepath.Dir(fh.URI().Filename()))
	key := modKey{
		sessionID: s.view.session.id,
		cfg:       hashConfig(cfg),
		mod:       fh.FileIdentity(),
		view:      s.view.root.Filename(),
		verb:      why,
	}
	h := s.generation.Bind(key, func(ctx context.Context, arg memoize.Arg) interface{} {
		ctx, done := event.Start(ctx, "cache.ModWhyHandle", tag.URI.Of(fh.URI()))
		defer done()

		snapshot := arg.(*snapshot)

		pm, err := snapshot.ParseMod(ctx, fh)
		if err != nil {
			return &modWhyData{err: err}
		}
		// No requires to explain.
		if len(pm.File.Require) == 0 {
			return &modWhyData{}
		}
		// Run `go mod why` on all the dependencies.
		args := []string{"why", "-m"}
		for _, req := range pm.File.Require {
			args = append(args, req.Mod.Path)
		}
		stdout, err := snapshot.RunGoCommand(ctx, "mod", args)
		if err != nil {
			return &modWhyData{err: err}
		}
		whyList := strings.Split(stdout.String(), "\n\n")
		if len(whyList) != len(pm.File.Require) {
			return &modWhyData{
				err: fmt.Errorf("mismatched number of results: got %v, want %v", len(whyList), len(pm.File.Require)),
			}
		}
		why := make(map[string]string, len(pm.File.Require))
		for i, req := range pm.File.Require {
			why[req.Mod.Path] = whyList[i]
		}
		return &modWhyData{why: why}
	})

	mwh := &modWhyHandle{handle: h}
	s.mu.Lock()
	s.modWhyHandles[fh.URI()] = mwh
	s.mu.Unlock()

	return mwh.why(ctx, s)
}

type modUpgradeHandle struct {
	handle *memoize.Handle
}

type modUpgradeData struct {
	// upgrades maps modules to their latest versions.
	upgrades map[string]string

	err error
}

func (muh *modUpgradeHandle) upgrades(ctx context.Context, snapshot *snapshot) (map[string]string, error) {
	v, err := muh.handle.Get(ctx, snapshot.generation, snapshot)
	if v == nil {
		return nil, err
	}
	data := v.(*modUpgradeData)
	return data.upgrades, data.err
}

// moduleUpgrade describes a module that can be upgraded to a particular
// version.
type moduleUpgrade struct {
	Path   string
	Update struct {
		Version string
	}
}

func (s *snapshot) ModUpgrade(ctx context.Context, fh source.FileHandle) (map[string]string, error) {
	if err := s.awaitLoaded(ctx); err != nil {
		return nil, err
	}
	if handle := s.getModUpgradeHandle(fh.URI()); handle != nil {
		return handle.upgrades(ctx, s)
	}
	// Use the module root as the working directory.
	cfg := s.configWithDir(ctx, filepath.Dir(fh.URI().Filename()))
	key := modKey{
		sessionID: s.view.session.id,
		cfg:       hashConfig(cfg),
		mod:       fh.FileIdentity(),
		view:      s.view.root.Filename(),
		verb:      upgrade,
	}
	h := s.generation.Bind(key, func(ctx context.Context, arg memoize.Arg) interface{} {
		ctx, done := event.Start(ctx, "cache.ModUpgradeHandle", tag.URI.Of(fh.URI()))
		defer done()

		snapshot := arg.(*snapshot)

		pm, err := s.ParseMod(ctx, fh)
		if err != nil {
			return &modUpgradeData{err: err}
		}

		// No requires to upgrade.
		if len(pm.File.Require) == 0 {
			return &modUpgradeData{}
		}
		// Run "go list -mod readonly -u -m all" to be able to see which deps can be
		// upgraded without modifying mod file.
		args := []string{"-u", "-m", "-json", "all"}
		if !snapshot.view.tmpMod || containsVendor(fh.URI()) {
			// Use -mod=readonly if the module contains a vendor directory
			// (see golang/go#38711).
			args = append([]string{"-mod", "readonly"}, args...)
		}
		stdout, err := snapshot.RunGoCommand(ctx, "list", args)
		if err != nil {
			return &modUpgradeData{err: err}
		}
		var upgradeList []moduleUpgrade
		dec := json.NewDecoder(stdout)
		for {
			var m moduleUpgrade
			if err := dec.Decode(&m); err == io.EOF {
				break
			} else if err != nil {
				return &modUpgradeData{err: err}
			}
			upgradeList = append(upgradeList, m)
		}
		if len(upgradeList) <= 1 {
			return &modUpgradeData{}
		}
		upgrades := make(map[string]string)
		for _, upgrade := range upgradeList[1:] {
			if upgrade.Update.Version == "" {
				continue
			}
			upgrades[upgrade.Path] = upgrade.Update.Version
		}
		return &modUpgradeData{
			upgrades: upgrades,
		}
	})
	muh := &modUpgradeHandle{handle: h}
	s.mu.Lock()
	s.modUpgradeHandles[fh.URI()] = muh
	s.mu.Unlock()

	return muh.upgrades(ctx, s)
}

// containsVendor reports whether the module has a vendor folder.
func containsVendor(modURI span.URI) bool {
	dir := filepath.Dir(modURI.Filename())
	f, err := os.Stat(filepath.Join(dir, "vendor"))
	if err != nil {
		return false
	}
	return f.IsDir()
}
<commit_msg>internal/lsp/cache: use the right snapshot in ModUpgrade<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cache

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"

	"golang.org/x/mod/modfile"
	"golang.org/x/tools/internal/event"
	"golang.org/x/tools/internal/lsp/debug/tag"
	"golang.org/x/tools/internal/lsp/protocol"
	"golang.org/x/tools/internal/lsp/source"
	"golang.org/x/tools/internal/memoize"
	"golang.org/x/tools/internal/span"
	errors "golang.org/x/xerrors"
)

const (
	SyntaxError = "syntax"
)

type parseModHandle struct {
	handle *memoize.Handle
}

type parseModData struct {
	parsed *source.ParsedModule

	// err is any error encountered while parsing the file.
	err error
}

func (mh *parseModHandle) parse(ctx context.Context, snapshot *snapshot) (*source.ParsedModule, error) {
	v, err := mh.handle.Get(ctx, snapshot.generation, snapshot)
	if err != nil {
		return nil, err
	}
	data := v.(*parseModData)
	return data.parsed, data.err
}

func (s *snapshot) ParseMod(ctx context.Context, modFH source.FileHandle) (*source.ParsedModule, error) {
	if handle := s.getParseModHandle(modFH.URI()); handle != nil {
		return handle.parse(ctx, s)
	}
	h := s.generation.Bind(modFH.FileIdentity(), func(ctx context.Context, _ memoize.Arg) interface{} {
		_, done := event.Start(ctx, "cache.ParseModHandle", tag.URI.Of(modFH.URI()))
		defer done()

		contents, err := modFH.Read()
		if err != nil {
			return &parseModData{err: err}
		}
		m := &protocol.ColumnMapper{
			URI:       modFH.URI(),
			Converter: span.NewContentConverter(modFH.URI().Filename(), contents),
			Content:   contents,
		}
		data := &parseModData{
			parsed: &source.ParsedModule{
				Mapper: m,
			},
		}
		data.parsed.File, data.err = modfile.Parse(modFH.URI().Filename(), contents, nil)
		if data.err != nil {
			// Attempt to convert the error to a standardized parse error.
			if parseErr, extractErr := extractModParseErrors(modFH.URI(), m, data.err, contents); extractErr == nil {
				data.parsed.ParseErrors = []source.Error{*parseErr}
			}
			// If the file was still parsed, we don't want to treat this as a
			// fatal error. Note: This currently cannot happen as modfile.Parse
			// always returns an error when the file is nil.
			if data.parsed.File != nil {
				data.err = nil
			}
		}
		return data
	})

	pmh := &parseModHandle{handle: h}
	s.mu.Lock()
	s.parseModHandles[modFH.URI()] = pmh
	s.mu.Unlock()

	return pmh.parse(ctx, s)
}

func (s *snapshot) sumFH(ctx context.Context, modFH source.FileHandle) (source.FileHandle, error) {
	// Get the go.sum file, either from the snapshot or directly from the
	// cache. Avoid (*snapshot).GetFile here, as we don't want to add
	// nonexistent file handles to the snapshot if the file does not exist.
	sumURI := span.URIFromPath(sumFilename(modFH.URI()))
	var sumFH source.FileHandle = s.FindFile(sumURI)
	if sumFH == nil {
		var err error
		sumFH, err = s.view.session.cache.getFile(ctx, sumURI)
		if err != nil {
			return nil, err
		}
	}
	_, err := sumFH.Read()
	if err != nil {
		return nil, err
	}
	return sumFH, nil
}

func sumFilename(modURI span.URI) string {
	return strings.TrimSuffix(modURI.Filename(), ".mod") + ".sum"
}

// extractModParseErrors processes the raw errors returned by modfile.Parse,
// extracting the filenames and line numbers that correspond to the errors.
func extractModParseErrors(uri span.URI, m *protocol.ColumnMapper, parseErr error, content []byte) (*source.Error, error) {
	re := regexp.MustCompile(`.*:([\d]+): (.+)`)
	matches := re.FindStringSubmatch(strings.TrimSpace(parseErr.Error()))
	if len(matches) < 3 {
		return nil, errors.Errorf("could not parse go.mod error message: %s", parseErr)
	}
	line, err := strconv.Atoi(matches[1])
	if err != nil {
		return nil, err
	}
	lines := strings.Split(string(content), "\n")
	if line > len(lines) {
		return nil, errors.Errorf("could not parse go.mod error message %q, line number %v out of range", content, line)
	}
	// The error returned from the modfile package only returns a line number,
	// so we assume that the diagnostic should be for the entire line.
	endOfLine := len(lines[line-1])
	sOffset, err := m.Converter.ToOffset(line, 0)
	if err != nil {
		return nil, err
	}
	eOffset, err := m.Converter.ToOffset(line, endOfLine)
	if err != nil {
		return nil, err
	}
	spn := span.New(uri, span.NewPoint(line, 0, sOffset), span.NewPoint(line, endOfLine, eOffset))
	rng, err := m.Range(spn)
	if err != nil {
		return nil, err
	}
	return &source.Error{
		Category: SyntaxError,
		Message:  matches[2],
		Range:    rng,
		URI:      uri,
	}, nil
}

// modKey is uniquely identifies cached data for `go mod why` or dependencies
// to upgrade.
type modKey struct {
	sessionID, cfg, view string
	mod                  source.FileIdentity
	verb                 modAction
}

type modAction int

const (
	why modAction = iota
	upgrade
)

type modWhyHandle struct {
	handle *memoize.Handle
}

type modWhyData struct {
	// why keeps track of the `go mod why` results for each require statement
	// in the go.mod file.
	why map[string]string

	err error
}

func (mwh *modWhyHandle) why(ctx context.Context, snapshot *snapshot) (map[string]string, error) {
	v, err := mwh.handle.Get(ctx, snapshot.generation, snapshot)
	if err != nil {
		return nil, err
	}
	data := v.(*modWhyData)
	return data.why, data.err
}

func (s *snapshot) ModWhy(ctx context.Context, fh source.FileHandle) (map[string]string, error) {
	if err := s.awaitLoaded(ctx); err != nil {
		return nil, err
	}
	if handle := s.getModWhyHandle(fh.URI()); handle != nil {
		return handle.why(ctx, s)
	}
	// Make sure to use the module root as the working directory.
	cfg := s.configWithDir(ctx, filepath.Dir(fh.URI().Filename()))
	key := modKey{
		sessionID: s.view.session.id,
		cfg:       hashConfig(cfg),
		mod:       fh.FileIdentity(),
		view:      s.view.root.Filename(),
		verb:      why,
	}
	h := s.generation.Bind(key, func(ctx context.Context, arg memoize.Arg) interface{} {
		ctx, done := event.Start(ctx, "cache.ModWhyHandle", tag.URI.Of(fh.URI()))
		defer done()

		snapshot := arg.(*snapshot)

		pm, err := snapshot.ParseMod(ctx, fh)
		if err != nil {
			return &modWhyData{err: err}
		}
		// No requires to explain.
		if len(pm.File.Require) == 0 {
			return &modWhyData{}
		}
		// Run `go mod why` on all the dependencies.
		args := []string{"why", "-m"}
		for _, req := range pm.File.Require {
			args = append(args, req.Mod.Path)
		}
		stdout, err := snapshot.RunGoCommand(ctx, "mod", args)
		if err != nil {
			return &modWhyData{err: err}
		}
		whyList := strings.Split(stdout.String(), "\n\n")
		if len(whyList) != len(pm.File.Require) {
			return &modWhyData{
				err: fmt.Errorf("mismatched number of results: got %v, want %v", len(whyList), len(pm.File.Require)),
			}
		}
		why := make(map[string]string, len(pm.File.Require))
		for i, req := range pm.File.Require {
			why[req.Mod.Path] = whyList[i]
		}
		return &modWhyData{why: why}
	})

	mwh := &modWhyHandle{handle: h}
	s.mu.Lock()
	s.modWhyHandles[fh.URI()] = mwh
	s.mu.Unlock()

	return mwh.why(ctx, s)
}

type modUpgradeHandle struct {
	handle *memoize.Handle
}

type modUpgradeData struct {
	// upgrades maps modules to their latest versions.
	upgrades map[string]string

	err error
}

func (muh *modUpgradeHandle) upgrades(ctx context.Context, snapshot *snapshot) (map[string]string, error) {
	v, err := muh.handle.Get(ctx, snapshot.generation, snapshot)
	if v == nil {
		return nil, err
	}
	data := v.(*modUpgradeData)
	return data.upgrades, data.err
}

// moduleUpgrade describes a module that can be upgraded to a particular
// version.
type moduleUpgrade struct {
	Path   string
	Update struct {
		Version string
	}
}

func (s *snapshot) ModUpgrade(ctx context.Context, fh source.FileHandle) (map[string]string, error) {
	if err := s.awaitLoaded(ctx); err != nil {
		return nil, err
	}
	if handle := s.getModUpgradeHandle(fh.URI()); handle != nil {
		return handle.upgrades(ctx, s)
	}
	// Use the module root as the working directory.
	cfg := s.configWithDir(ctx, filepath.Dir(fh.URI().Filename()))
	key := modKey{
		sessionID: s.view.session.id,
		cfg:       hashConfig(cfg),
		mod:       fh.FileIdentity(),
		view:      s.view.root.Filename(),
		verb:      upgrade,
	}
	h := s.generation.Bind(key, func(ctx context.Context, arg memoize.Arg) interface{} {
		ctx, done := event.Start(ctx, "cache.ModUpgradeHandle", tag.URI.Of(fh.URI()))
		defer done()

		snapshot := arg.(*snapshot)

		pm, err := snapshot.ParseMod(ctx, fh)
		if err != nil {
			return &modUpgradeData{err: err}
		}

		// No requires to upgrade.
		if len(pm.File.Require) == 0 {
			return &modUpgradeData{}
		}
		// Run "go list -mod readonly -u -m all" to be able to see which deps can be
		// upgraded without modifying mod file.
		args := []string{"-u", "-m", "-json", "all"}
		if !snapshot.view.tmpMod || containsVendor(fh.URI()) {
			// Use -mod=readonly if the module contains a vendor directory
			// (see golang/go#38711).
			args = append([]string{"-mod", "readonly"}, args...)
		}
		stdout, err := snapshot.RunGoCommand(ctx, "list", args)
		if err != nil {
			return &modUpgradeData{err: err}
		}
		var upgradeList []moduleUpgrade
		dec := json.NewDecoder(stdout)
		for {
			var m moduleUpgrade
			if err := dec.Decode(&m); err == io.EOF {
				break
			} else if err != nil {
				return &modUpgradeData{err: err}
			}
			upgradeList = append(upgradeList, m)
		}
		if len(upgradeList) <= 1 {
			return &modUpgradeData{}
		}
		upgrades := make(map[string]string)
		for _, upgrade := range upgradeList[1:] {
			if upgrade.Update.Version == "" {
				continue
			}
			upgrades[upgrade.Path] = upgrade.Update.Version
		}
		return &modUpgradeData{
			upgrades: upgrades,
		}
	})
	muh := &modUpgradeHandle{handle: h}
	s.mu.Lock()
	s.modUpgradeHandles[fh.URI()] = muh
	s.mu.Unlock()

	return muh.upgrades(ctx, s)
}

// containsVendor reports whether the module has a vendor folder.
func containsVendor(modURI span.URI) bool {
	dir := filepath.Dir(modURI.Filename())
	f, err := os.Stat(filepath.Join(dir, "vendor"))
	if err != nil {
		return false
	}
	return f.IsDir()
}
<|endoftext|>


<commit_before>// Copyright © 2017 Circonus, Inc. <support@circonus.com>
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//

package server

import (
	"context"
	"errors"
	"fmt"
	"net"
	"net/http"
	"os"
	"regexp"
	"runtime"
	"sync"
	"time"

	"github.com/circonus-labs/circonus-agent/internal/builtins"
	"github.com/circonus-labs/circonus-agent/internal/check"
	"github.com/circonus-labs/circonus-agent/internal/config"
	"github.com/circonus-labs/circonus-agent/internal/config/defaults"
	"github.com/circonus-labs/circonus-agent/internal/plugins"
	"github.com/circonus-labs/circonus-agent/internal/statsd"
	cgm "github.com/circonus-labs/circonus-gometrics/v3"
	"github.com/rs/zerolog"
	"github.com/rs/zerolog/log"
	"github.com/spf13/viper"
	"golang.org/x/sync/errgroup"
)

type httpServer struct {
	address *net.TCPAddr
	server  *http.Server
}

type socketServer struct {
	address  *net.UnixAddr
	listener *net.UnixListener
	server   *http.Server
}

type sslServer struct {
	server   *http.Server
	address  *net.TCPAddr
	certFile string
	keyFile  string
}

// Server defines the listening servers.
type Server struct {
	check      *check.Check
	group      *errgroup.Group
	builtins   *builtins.Builtins
	plugins    *plugins.Plugins
	statsdSvr  *statsd.Server
	svrHTTPS   *sslServer
	svrHTTP    []*httpServer
	svrSockets []*socketServer
	groupCtx   context.Context
	logger     zerolog.Logger
}

type previousMetrics struct {
	ts      time.Time
	metrics *cgm.Metrics
}

var (
	pluginPathRx    = regexp.MustCompile("^/(run(/[a-zA-Z0-9_-]*)?)?$")
	inventoryPathRx = regexp.MustCompile("^/inventory/?$")
	writePathRx     = regexp.MustCompile("^/write/[a-zA-Z0-9_-]+$")
	statsPathRx     = regexp.MustCompile("^/stats/?$")
	promPathRx      = regexp.MustCompile("^/prom/?$")
	lastMetrics     = &previousMetrics{}
	lastMetricsmu   sync.Mutex
)

// New creates a new instance of the listening servers.
func New(ctx context.Context, c *check.Check, b *builtins.Builtins, p *plugins.Plugins, ss *statsd.Server) (*Server, error) {
	g, gctx := errgroup.WithContext(ctx)
	s := Server{
		group:     g,
		groupCtx:  gctx,
		logger:    log.With().Str("pkg", "server").Logger(),
		builtins:  b,
		plugins:   p,
		statsdSvr: ss,
		check:     c,
	}

	// HTTP listener (1-n)
	{
		serverList := viper.GetStringSlice(config.KeyListen)
		if len(serverList) == 0 {
			serverList = []string{defaults.Listen}
		}
		for idx, addr := range serverList {
			ta, err := config.ParseListen(addr)
			if err != nil {
				s.logger.Error().Err(err).Int("id", idx).Str("addr", addr).Msg("resolving address")
				return nil, fmt.Errorf("HTTP Server: %w", err)
			}

			svr := httpServer{
				address: ta,
				server: &http.Server{
					Addr:    ta.String(),
					Handler: http.HandlerFunc(s.router),
				},
			}
			svr.server.SetKeepAlivesEnabled(false)

			s.svrHTTP = append(s.svrHTTP, &svr)
		}
	}

	// HTTPS listener (singular)
	if addr := viper.GetString(config.KeySSLListen); addr != "" {
		ta, err := net.ResolveTCPAddr("tcp", addr)
		if err != nil {
			s.logger.Error().Err(err).Str("addr", addr).Msg("resolving address")
			return nil, fmt.Errorf("SSL Server: %w", err)
		}

		certFile := viper.GetString(config.KeySSLCertFile)
		if _, err := os.Stat(certFile); os.IsNotExist(err) {
			s.logger.Error().Err(err).Str("cert_file", certFile).Msg("SSL server")
			return nil, fmt.Errorf("SSL server cert file: %w", err)
		}

		keyFile := viper.GetString(config.KeySSLKeyFile)
		if _, err := os.Stat(keyFile); os.IsNotExist(err) {
			s.logger.Error().Err(err).Str("key_file", keyFile).Msg("SSL server")
			return nil, fmt.Errorf("SSL server key file: %w", err)
		}

		svr := sslServer{
			address:  ta,
			certFile: certFile,
			keyFile:  keyFile,
			server: &http.Server{
				Addr:    ta.String(),
				Handler: http.HandlerFunc(s.router),
				// Handler: httpgzip.NewHandler(http.HandlerFunc(s.router), []string{"application/json"}),
			},
		}

		svr.server.SetKeepAlivesEnabled(false)
		s.svrHTTPS = &svr
	}

	// Socket listener (1-n)
	if runtime.GOOS != "windows" {
		socketList := viper.GetStringSlice(config.KeyListenSocket)
		for idx, addr := range socketList {
			ua, err := net.ResolveUnixAddr("unix", addr)
			if err != nil {
				s.logger.Error().Err(err).Int("id", idx).Str("addr", addr).Msg("resolving address")
				return nil, fmt.Errorf("socket server: %w", err)
			}

			if _, serr := os.Stat(ua.String()); serr == nil || !os.IsNotExist(serr) {
				s.logger.Error().Int("id", idx).Str("socket_file", ua.String()).Msg("already exists")
				return nil, fmt.Errorf("socket server file (%s) exists", ua.String()) //nolint:goerr113
			}

			ul, err := net.ListenUnix(ua.Network(), ua)
			if err != nil {
				s.logger.Error().Err(err).Int("id", idx).Str("addr", ua.String()).Msg("creating socket")
				return nil, fmt.Errorf("creating socket: %w", err)
			}

			s.svrSockets = append(s.svrSockets, &socketServer{
				address:  ua,
				listener: ul,
				server:   &http.Server{Handler: http.HandlerFunc(s.socketHandler)},
			})
		}
	}

	return &s, nil
}

// GetReverseAgentAddress returns the address reverse should use to talk to the agent.
// Initially, this is the first server address.
func (s *Server) GetReverseAgentAddress() (string, error) {
	if len(s.svrHTTP) == 0 {
		return "", fmt.Errorf("no listen servers defined") //nolint:goerr113
	}
	return s.svrHTTP[0].address.String(), nil
}

// Start main listening server(s).
func (s *Server) Start() error {
	if len(s.svrHTTP) == 0 && s.svrHTTPS == nil && len(s.svrSockets) > 0 {
		return fmt.Errorf("no servers defined") //nolint:goerr113
	}

	s.group.Go(s.startHTTPS)

	for _, svrHTTP := range s.svrHTTP {
		svr := svrHTTP
		s.group.Go(func() error {
			return s.startHTTP(svr)
		})
	}

	for _, svrSocket := range s.svrSockets {
		svr := svrSocket
		s.group.Go(func() error {
			return s.startSocket(svr)
		})
	}

	go func() {
		<-s.groupCtx.Done()
		s.Stop()
	}()

	return s.group.Wait() //nolint:wrapcheck
}

// Stop the servers in an orderly, graceful fashion.
func (s *Server) Stop() {
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()

	for _, svrHTTP := range s.svrHTTP {
		s.logger.Info().Msg("stopping HTTP server")
		err := svrHTTP.server.Shutdown(ctx)
		if err != nil {
			s.logger.Warn().Err(err).Msg("closing HTTP server")
		}
	}

	if s.svrHTTPS != nil {
		s.logger.Info().Msg("stopping HTTPS server")
		err := s.svrHTTPS.server.Shutdown(ctx)
		if err != nil {
			s.logger.Warn().Err(err).Msg("closing HTTPS server")
		}
	}

	for _, svrSocket := range s.svrSockets {
		s.logger.Info().Str("server", svrSocket.address.Name).Msg("stopping Socket server")
		err := svrSocket.server.Shutdown(ctx)
		if err != nil {
			s.logger.Warn().Err(err).Str("server", svrSocket.address.Name).Msg("closing Socket server")
		}
	}
}

func (s *Server) startHTTP(svr *httpServer) error {
	if svr == nil {
		s.logger.Debug().Msg("no listen configured, skipping server")
		return nil
	}
	if svr.address == nil || svr.server == nil {
		s.logger.Debug().Msg("listen not configured, skipping server")
		return nil
	}

	s.logger.Info().Str("listen", svr.address.String()).Msg("Starting")
	if err := svr.server.ListenAndServe(); err != nil {
		if !errors.Is(err, http.ErrServerClosed) {
			s.logger.Fatal().Err(err).Msg("HTTP Server, stopping agent")
			return fmt.Errorf("HTTP server: %w", err)
		}

	}
	return nil
}

func (s *Server) startHTTPS() error {
	if s.svrHTTPS == nil {
		s.logger.Debug().Msg("no SSL listen configured, skipping server")
		return nil
	}
	s.logger.Info().Str("listen", s.svrHTTPS.server.Addr).Msg("SSL starting")
	if err := s.svrHTTPS.server.ListenAndServeTLS(s.svrHTTPS.certFile, s.svrHTTPS.keyFile); err != nil {
		if !errors.Is(err, http.ErrServerClosed) {
			s.logger.Fatal().Err(err).Msg("SSL Server, stopping agent")
			return fmt.Errorf("SSL server: %w", err)
		}
	}
	return nil
}

func (s *Server) startSocket(svr *socketServer) error {
	if svr == nil {
		s.logger.Debug().Msg("no socket configured, skipping")
		return nil
	}
	if svr.address == nil || svr.listener == nil || svr.server == nil {
		s.logger.Debug().Msg("socket not configured, skipping")
		return nil
	}
	if runtime.GOOS == "windows" {
		s.logger.Warn().Msg("platform does not support unix sockets")
		return nil
	}

	s.logger.Info().Str("listen", svr.address.String()).Msg("Socket starting")
	if err := svr.server.Serve(svr.listener); err != nil {
		if !errors.Is(err, http.ErrServerClosed) {
			s.logger.Fatal().Err(err).Str("socket", svr.address.String()).Msg("Socket Server, stopping agent")
			return fmt.Errorf("socket server: %w", err)
		}
	}
	return nil
}
<commit_msg>fix: G112 add ReadHeaderTimeout to http.Server<commit_after>// Copyright © 2017 Circonus, Inc. <support@circonus.com>
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//

package server

import (
	"context"
	"errors"
	"fmt"
	"net"
	"net/http"
	"os"
	"regexp"
	"runtime"
	"sync"
	"time"

	"github.com/circonus-labs/circonus-agent/internal/builtins"
	"github.com/circonus-labs/circonus-agent/internal/check"
	"github.com/circonus-labs/circonus-agent/internal/config"
	"github.com/circonus-labs/circonus-agent/internal/config/defaults"
	"github.com/circonus-labs/circonus-agent/internal/plugins"
	"github.com/circonus-labs/circonus-agent/internal/statsd"
	cgm "github.com/circonus-labs/circonus-gometrics/v3"
	"github.com/rs/zerolog"
	"github.com/rs/zerolog/log"
	"github.com/spf13/viper"
	"golang.org/x/sync/errgroup"
)

type httpServer struct {
	address *net.TCPAddr
	server  *http.Server
}

type socketServer struct {
	address  *net.UnixAddr
	listener *net.UnixListener
	server   *http.Server
}

type sslServer struct {
	server   *http.Server
	address  *net.TCPAddr
	certFile string
	keyFile  string
}

// Server defines the listening servers.
type Server struct {
	check      *check.Check
	group      *errgroup.Group
	builtins   *builtins.Builtins
	plugins    *plugins.Plugins
	statsdSvr  *statsd.Server
	svrHTTPS   *sslServer
	svrHTTP    []*httpServer
	svrSockets []*socketServer
	groupCtx   context.Context
	logger     zerolog.Logger
}

type previousMetrics struct {
	ts      time.Time
	metrics *cgm.Metrics
}

var (
	pluginPathRx    = regexp.MustCompile("^/(run(/[a-zA-Z0-9_-]*)?)?$")
	inventoryPathRx = regexp.MustCompile("^/inventory/?$")
	writePathRx     = regexp.MustCompile("^/write/[a-zA-Z0-9_-]+$")
	statsPathRx     = regexp.MustCompile("^/stats/?$")
	promPathRx      = regexp.MustCompile("^/prom/?$")
	lastMetrics     = &previousMetrics{}
	lastMetricsmu   sync.Mutex
)

// New creates a new instance of the listening servers.
func New(ctx context.Context, c *check.Check, b *builtins.Builtins, p *plugins.Plugins, ss *statsd.Server) (*Server, error) {
	g, gctx := errgroup.WithContext(ctx)
	s := Server{
		group:     g,
		groupCtx:  gctx,
		logger:    log.With().Str("pkg", "server").Logger(),
		builtins:  b,
		plugins:   p,
		statsdSvr: ss,
		check:     c,
	}

	// HTTP listener (1-n)
	{
		serverList := viper.GetStringSlice(config.KeyListen)
		if len(serverList) == 0 {
			serverList = []string{defaults.Listen}
		}
		for idx, addr := range serverList {
			ta, err := config.ParseListen(addr)
			if err != nil {
				s.logger.Error().Err(err).Int("id", idx).Str("addr", addr).Msg("resolving address")
				return nil, fmt.Errorf("HTTP Server: %w", err)
			}

			svr := httpServer{
				address: ta,
				server: &http.Server{
					Addr:              ta.String(),
					Handler:           http.HandlerFunc(s.router),
					ReadHeaderTimeout: 2 * time.Second,
				},
			}
			svr.server.SetKeepAlivesEnabled(false)

			s.svrHTTP = append(s.svrHTTP, &svr)
		}
	}

	// HTTPS listener (singular)
	if addr := viper.GetString(config.KeySSLListen); addr != "" {
		ta, err := net.ResolveTCPAddr("tcp", addr)
		if err != nil {
			s.logger.Error().Err(err).Str("addr", addr).Msg("resolving address")
			return nil, fmt.Errorf("SSL Server: %w", err)
		}

		certFile := viper.GetString(config.KeySSLCertFile)
		if _, err := os.Stat(certFile); os.IsNotExist(err) {
			s.logger.Error().Err(err).Str("cert_file", certFile).Msg("SSL server")
			return nil, fmt.Errorf("SSL server cert file: %w", err)
		}

		keyFile := viper.GetString(config.KeySSLKeyFile)
		if _, err := os.Stat(keyFile); os.IsNotExist(err) {
			s.logger.Error().Err(err).Str("key_file", keyFile).Msg("SSL server")
			return nil, fmt.Errorf("SSL server key file: %w", err)
		}

		svr := sslServer{
			address:  ta,
			certFile: certFile,
			keyFile:  keyFile,
			server: &http.Server{
				Addr:    ta.String(),
				Handler: http.HandlerFunc(s.router),
				// Handler: httpgzip.NewHandler(http.HandlerFunc(s.router), []string{"application/json"}),
				ReadHeaderTimeout: 2 * time.Second,
			},
		}

		svr.server.SetKeepAlivesEnabled(false)
		s.svrHTTPS = &svr
	}

	// Socket listener (1-n)
	if runtime.GOOS != "windows" {
		socketList := viper.GetStringSlice(config.KeyListenSocket)
		for idx, addr := range socketList {
			ua, err := net.ResolveUnixAddr("unix", addr)
			if err != nil {
				s.logger.Error().Err(err).Int("id", idx).Str("addr", addr).Msg("resolving address")
				return nil, fmt.Errorf("socket server: %w", err)
			}

			if _, serr := os.Stat(ua.String()); serr == nil || !os.IsNotExist(serr) {
				s.logger.Error().Int("id", idx).Str("socket_file", ua.String()).Msg("already exists")
				return nil, fmt.Errorf("socket server file (%s) exists", ua.String()) //nolint:goerr113
			}

			ul, err := net.ListenUnix(ua.Network(), ua)
			if err != nil {
				s.logger.Error().Err(err).Int("id", idx).Str("addr", ua.String()).Msg("creating socket")
				return nil, fmt.Errorf("creating socket: %w", err)
			}

			s.svrSockets = append(s.svrSockets, &socketServer{
				address:  ua,
				listener: ul,
				server: &http.Server{
					ReadHeaderTimeout: 2 * time.Second,
					Handler:           http.HandlerFunc(s.socketHandler),
				},
			})
		}
	}

	return &s, nil
}

// GetReverseAgentAddress returns the address reverse should use to talk to the agent.
// Initially, this is the first server address.
func (s *Server) GetReverseAgentAddress() (string, error) {
	if len(s.svrHTTP) == 0 {
		return "", fmt.Errorf("no listen servers defined") //nolint:goerr113
	}
	return s.svrHTTP[0].address.String(), nil
}

// Start main listening server(s).
func (s *Server) Start() error {
	if len(s.svrHTTP) == 0 && s.svrHTTPS == nil && len(s.svrSockets) > 0 {
		return fmt.Errorf("no servers defined") //nolint:goerr113
	}

	s.group.Go(s.startHTTPS)

	for _, svrHTTP := range s.svrHTTP {
		svr := svrHTTP
		s.group.Go(func() error {
			return s.startHTTP(svr)
		})
	}

	for _, svrSocket := range s.svrSockets {
		svr := svrSocket
		s.group.Go(func() error {
			return s.startSocket(svr)
		})
	}

	go func() {
		<-s.groupCtx.Done()
		s.Stop()
	}()

	return s.group.Wait() //nolint:wrapcheck
}

// Stop the servers in an orderly, graceful fashion.
func (s *Server) Stop() {
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()

	for _, svrHTTP := range s.svrHTTP {
		s.logger.Info().Msg("stopping HTTP server")
		err := svrHTTP.server.Shutdown(ctx)
		if err != nil {
			s.logger.Warn().Err(err).Msg("closing HTTP server")
		}
	}

	if s.svrHTTPS != nil {
		s.logger.Info().Msg("stopping HTTPS server")
		err := s.svrHTTPS.server.Shutdown(ctx)
		if err != nil {
			s.logger.Warn().Err(err).Msg("closing HTTPS server")
		}
	}

	for _, svrSocket := range s.svrSockets {
		s.logger.Info().Str("server", svrSocket.address.Name).Msg("stopping Socket server")
		err := svrSocket.server.Shutdown(ctx)
		if err != nil {
			s.logger.Warn().Err(err).Str("server", svrSocket.address.Name).Msg("closing Socket server")
		}
	}
}

func (s *Server) startHTTP(svr *httpServer) error {
	if svr == nil {
		s.logger.Debug().Msg("no listen configured, skipping server")
		return nil
	}
	if svr.address == nil || svr.server == nil {
		s.logger.Debug().Msg("listen not configured, skipping server")
		return nil
	}

	s.logger.Info().Str("listen", svr.address.String()).Msg("Starting")
	if err := svr.server.ListenAndServe(); err != nil {
		if !errors.Is(err, http.ErrServerClosed) {
			s.logger.Fatal().Err(err).Msg("HTTP Server, stopping agent")
			return fmt.Errorf("HTTP server: %w", err)
		}

	}
	return nil
}

func (s *Server) startHTTPS() error {
	if s.svrHTTPS == nil {
		s.logger.Debug().Msg("no SSL listen configured, skipping server")
		return nil
	}
	s.logger.Info().Str("listen", s.svrHTTPS.server.Addr).Msg("SSL starting")
	if err := s.svrHTTPS.server.ListenAndServeTLS(s.svrHTTPS.certFile, s.svrHTTPS.keyFile); err != nil {
		if !errors.Is(err, http.ErrServerClosed) {
			s.logger.Fatal().Err(err).Msg("SSL Server, stopping agent")
			return fmt.Errorf("SSL server: %w", err)
		}
	}
	return nil
}

func (s *Server) startSocket(svr *socketServer) error {
	if svr == nil {
		s.logger.Debug().Msg("no socket configured, skipping")
		return nil
	}
	if svr.address == nil || svr.listener == nil || svr.server == nil {
		s.logger.Debug().Msg("socket not configured, skipping")
		return nil
	}
	if runtime.GOOS == "windows" {
		s.logger.Warn().Msg("platform does not support unix sockets")
		return nil
	}

	s.logger.Info().Str("listen", svr.address.String()).Msg("Socket starting")
	if err := svr.server.Serve(svr.listener); err != nil {
		if !errors.Is(err, http.ErrServerClosed) {
			s.logger.Fatal().Err(err).Str("socket", svr.address.String()).Msg("Socket Server, stopping agent")
			return fmt.Errorf("socket server: %w", err)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package project

import (
	"fmt"
	"strconv"

	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/instance/instancetype"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/units"
	"github.com/pkg/errors"
)

// CheckLimitsUponInstanceCreation returns an error if any project-specific
// limit is violated when creating a new instance.
func CheckLimitsUponInstanceCreation(tx *db.ClusterTx, projectName string, req api.InstancesPost) error {
	project, profiles, instances, err := fetchProject(tx, projectName)
	if err != nil {
		return err
	}

	err = checkInstanceCountLimit(project, len(instances), req.Type)
	if err != nil {
		return err
	}

	// Add the instance being created.
	instances = append(instances, db.Instance{
		Profiles: req.Profiles,
		Config:   req.Config,
	})

	err = checkAggregateInstanceLimits(tx, project, instances, profiles)
	if err != nil {
		return err
	}

	return nil
}

// Check that we have not reached the maximum number of instances for
// this type.
func checkInstanceCountLimit(project *api.Project, instanceCount int, instanceType api.InstanceType) error {
	var key string
	switch instanceType {
	case api.InstanceTypeContainer:
		key = "limits.containers"
	case api.InstanceTypeVM:
		key = "limits.virtual-machines"
	default:
		return fmt.Errorf("Unexpected instance type '%s'", instanceType)
	}
	value, ok := project.Config[key]
	if ok {
		limit, err := strconv.Atoi(value)
		if err != nil || limit < 0 {
			return fmt.Errorf("Unexpected '%s' value: '%s'", key, value)
		}
		if instanceCount >= limit {
			return fmt.Errorf(
				"Reached maximum number of instances of type %s in project %s",
				instanceType, project.Name)
		}
	}

	return nil
}

// Check that we would not violate the project limits if we were to commit the
// given instances and profiles.
func checkAggregateInstanceLimits(tx *db.ClusterTx, project *api.Project, instances []db.Instance, profiles []db.Profile) error {
	// List of config keys for which we need to check aggregate values
	// across all project instances.
	aggregateKeys := []string{}
	for key := range project.Config {
		if shared.StringInSlice(key, []string{"limits.memory"}) {
			aggregateKeys = append(aggregateKeys, key)
		}
	}
	if len(aggregateKeys) == 0 {
		return nil
	}

	instances = expandInstancesConfig(instances, profiles)

	totals, err := getTotalsAcrossInstances(instances, aggregateKeys)
	if err != nil {
		return err
	}

	for _, key := range aggregateKeys {
		parser := aggregateLimitConfigValueParsers[key]
		max, err := parser(project.Config[key])
		if err != nil {
			return err
		}
		if totals[key] > max {
			return fmt.Errorf(
				"Reached maximum aggregate value %s for %q in project %s",
				project.Config[key], key, project.Name)
		}
	}

	return nil
}

// CheckLimitsUponInstanceUpdate returns an error if any project-specific limit
// is violated when updating an existing instance.
func CheckLimitsUponInstanceUpdate(tx *db.ClusterTx, projectName, instanceName string, req api.InstancePut) error {
	project, profiles, instances, err := fetchProject(tx, projectName)
	if err != nil {
		return err
	}

	// Change the instance being updated.
	for i, instance := range instances {
		if instance.Name != instanceName {
			continue
		}
		instances[i].Profiles = req.Profiles
		instances[i].Config = req.Config
	}

	err = checkAggregateInstanceLimits(tx, project, instances, profiles)
	if err != nil {
		return err
	}

	return nil
}

// ValidateLimitsUponProjectUpdate checks the new limits to be set on a project
// are valid.
func ValidateLimitsUponProjectUpdate(tx *db.ClusterTx, projectName string, config map[string]string, changed []string) error {
	_, profiles, instances, err := fetchProject(tx, projectName)
	if err != nil {
		return err
	}

	instances = expandInstancesConfig(instances, profiles)

	for _, key := range changed {
		switch key {
		case "limits.containers":
			fallthrough
		case "limits.virtual-machines":
			err := validateInstanceCountLimit(instances, key, config[key], projectName)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

// Check that limits.containers or limits.virtual-machines is equal or above
// the current count.
func validateInstanceCountLimit(instances []db.Instance, key, value, project string) error {
	instanceType := countConfigInstanceType[key]
	limit, err := strconv.Atoi(value)
	if err != nil {
		return err
	}
	dbType, err := instancetype.New(string(instanceType))
	if err != nil {
		return err
	}
	count := 0
	for _, instance := range instances {
		if instance.Type == dbType {
			count++
		}
	}
	if limit < count {
		return fmt.Errorf(
			"'%s' is too low: there currently are %d instances of type %s in project %s",
			key, count, instanceType, project)
	}
	return nil
}

var countConfigInstanceType = map[string]api.InstanceType{
	"limits.containers":       api.InstanceTypeContainer,
	"limits.virtual-machines": api.InstanceTypeVM,
}

// Fetch the given project from the database along with its profiles and instances.
func fetchProject(tx *db.ClusterTx, projectName string) (*api.Project, []db.Profile, []db.Instance, error) {
	project, err := tx.ProjectGet(projectName)
	if err != nil {
		return nil, nil, nil, errors.Wrap(err, "Fetch project database object")
	}

	profilesFilter := db.ProfileFilter{}

	// If the project has the profiles feature enabled, we use its own
	// profiles to expand the instances configs, otherwise we use the
	// profiles from the default project.
	if projectName == "default" || project.Config["features.profiles"] == "true" {
		profilesFilter.Project = projectName
	} else {
		profilesFilter.Project = "default"
	}

	profiles, err := tx.ProfileList(profilesFilter)
	if err != nil {
		return nil, nil, nil, errors.Wrap(err, "Fetch profiles from database")
	}

	instances, err := tx.InstanceList(db.InstanceFilter{Project: projectName})
	if err != nil {
		return nil, nil, nil, errors.Wrap(err, "Fetch project instances from database")
	}

	return project, profiles, instances, nil
}

// Expand the configuration of the given instances, taking the give project
// profiles into account.
func expandInstancesConfig(instances []db.Instance, profiles []db.Profile) []db.Instance {
	expandedInstances := make([]db.Instance, len(instances))

	// Index of all profiles by name.
	profilesByName := map[string]db.Profile{}
	for _, profile := range profiles {
		profilesByName[profile.Name] = profile
	}

	for i, instance := range instances {
		profiles := make([]api.Profile, len(instance.Profiles))

		for j, name := range instance.Profiles {
			profile := profilesByName[name]
			profiles[j] = *db.ProfileToAPI(&profile)
		}

		expandedInstances[i] = instance
		expandedInstances[i].Config = db.ProfilesExpandConfig(instance.Config, profiles)
	}

	return expandedInstances
}

// Sum of the effective instance-level value for the given limits across all
// project instances. If excludeInstance is not the empty string, exclude the
// instance with that name.
func getTotalsAcrossInstances(instances []db.Instance, keys []string) (map[string]int64, error) {
	totals := map[string]int64{}

	for _, key := range keys {
		totals[key] = 0
	}

	for _, instance := range instances {
		limits, err := getInstanceLimits(instance, keys)
		if err != nil {
			return nil, err
		}
		for _, key := range keys {
			totals[key] += limits[key]
		}
	}

	return totals, nil
}

// Return the effective instance-level values for the limits with the given
// keys.
func getInstanceLimits(instance db.Instance, keys []string) (map[string]int64, error) {
	limits := map[string]int64{}

	for _, key := range keys {
		value, ok := instance.Config[key]
		if !ok || value == "" {
			return nil, fmt.Errorf(
				"Instance %s in project %s has no '%s' config, either directly or via a profile",
				instance.Name, instance.Project, key)
		}
		parser := aggregateLimitConfigValueParsers[key]
		limit, err := parser(value)
		if err != nil {
			return nil, errors.Wrapf(
				err, "Parse '%s' for instance %s in project %s",
				key, instance.Name, instance.Project)
		}
		limits[key] = limit
	}

	return limits, nil
}

var aggregateLimitConfigValueParsers = map[string]func(string) (int64, error){
	"limits.memory": units.ParseByteSizeString,
}
<commit_msg>lxd/project: Validate changes to the project's "limits.memory" value<commit_after>package project

import (
	"fmt"
	"strconv"

	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/instance/instancetype"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/units"
	"github.com/pkg/errors"
)

// CheckLimitsUponInstanceCreation returns an error if any project-specific
// limit is violated when creating a new instance.
func CheckLimitsUponInstanceCreation(tx *db.ClusterTx, projectName string, req api.InstancesPost) error {
	project, profiles, instances, err := fetchProject(tx, projectName)
	if err != nil {
		return err
	}

	err = checkInstanceCountLimit(project, len(instances), req.Type)
	if err != nil {
		return err
	}

	// Add the instance being created.
	instances = append(instances, db.Instance{
		Profiles: req.Profiles,
		Config:   req.Config,
	})

	err = checkAggregateInstanceLimits(tx, project, instances, profiles)
	if err != nil {
		return err
	}

	return nil
}

// Check that we have not reached the maximum number of instances for
// this type.
func checkInstanceCountLimit(project *api.Project, instanceCount int, instanceType api.InstanceType) error {
	var key string
	switch instanceType {
	case api.InstanceTypeContainer:
		key = "limits.containers"
	case api.InstanceTypeVM:
		key = "limits.virtual-machines"
	default:
		return fmt.Errorf("Unexpected instance type '%s'", instanceType)
	}
	value, ok := project.Config[key]
	if ok {
		limit, err := strconv.Atoi(value)
		if err != nil || limit < 0 {
			return fmt.Errorf("Unexpected '%s' value: '%s'", key, value)
		}
		if instanceCount >= limit {
			return fmt.Errorf(
				"Reached maximum number of instances of type %s in project %s",
				instanceType, project.Name)
		}
	}

	return nil
}

// Check that we would not violate the project limits if we were to commit the
// given instances and profiles.
func checkAggregateInstanceLimits(tx *db.ClusterTx, project *api.Project, instances []db.Instance, profiles []db.Profile) error {
	// List of config keys for which we need to check aggregate values
	// across all project instances.
	aggregateKeys := []string{}
	for key := range project.Config {
		if shared.StringInSlice(key, []string{"limits.memory"}) {
			aggregateKeys = append(aggregateKeys, key)
		}
	}
	if len(aggregateKeys) == 0 {
		return nil
	}

	instances = expandInstancesConfig(instances, profiles)

	totals, err := getTotalsAcrossInstances(instances, aggregateKeys)
	if err != nil {
		return err
	}

	for _, key := range aggregateKeys {
		parser := aggregateLimitConfigValueParsers[key]
		max, err := parser(project.Config[key])
		if err != nil {
			return err
		}
		if totals[key] > max {
			return fmt.Errorf(
				"Reached maximum aggregate value %s for %q in project %s",
				project.Config[key], key, project.Name)
		}
	}

	return nil
}

// CheckLimitsUponInstanceUpdate returns an error if any project-specific limit
// is violated when updating an existing instance.
func CheckLimitsUponInstanceUpdate(tx *db.ClusterTx, projectName, instanceName string, req api.InstancePut) error {
	project, profiles, instances, err := fetchProject(tx, projectName)
	if err != nil {
		return err
	}

	// Change the instance being updated.
	for i, instance := range instances {
		if instance.Name != instanceName {
			continue
		}
		instances[i].Profiles = req.Profiles
		instances[i].Config = req.Config
	}

	err = checkAggregateInstanceLimits(tx, project, instances, profiles)
	if err != nil {
		return err
	}

	return nil
}

// ValidateLimitsUponProjectUpdate checks the new limits to be set on a project
// are valid.
func ValidateLimitsUponProjectUpdate(tx *db.ClusterTx, projectName string, config map[string]string, changed []string) error {
	_, profiles, instances, err := fetchProject(tx, projectName)
	if err != nil {
		return err
	}

	instances = expandInstancesConfig(instances, profiles)

	// List of keys that need to check aggregate values across all project
	// instances.
	aggregateKeys := []string{}

	for _, key := range changed {
		switch key {
		case "limits.containers":
			fallthrough
		case "limits.virtual-machines":
			err := validateInstanceCountLimit(instances, key, config[key], projectName)
			if err != nil {
				return err
			}
		case "limits.memory":
			aggregateKeys = append(aggregateKeys, key)

		}
	}

	if len(aggregateKeys) > 0 {
		totals, err := getTotalsAcrossInstances(instances, aggregateKeys)
		if err != nil {
			return err
		}
		for _, key := range aggregateKeys {
			err := validateAggregateLimit(totals, key, config[key])
			if err != nil {
				return err
			}
		}

	}

	return nil
}

// Check that limits.containers or limits.virtual-machines is equal or above
// the current count.
func validateInstanceCountLimit(instances []db.Instance, key, value, project string) error {
	instanceType := countConfigInstanceType[key]
	limit, err := strconv.Atoi(value)
	if err != nil {
		return err
	}
	dbType, err := instancetype.New(string(instanceType))
	if err != nil {
		return err
	}
	count := 0
	for _, instance := range instances {
		if instance.Type == dbType {
			count++
		}
	}
	if limit < count {
		return fmt.Errorf(
			"'%s' is too low: there currently are %d instances of type %s in project %s",
			key, count, instanceType, project)
	}
	return nil
}

var countConfigInstanceType = map[string]api.InstanceType{
	"limits.containers":       api.InstanceTypeContainer,
	"limits.virtual-machines": api.InstanceTypeVM,
}

// Validates an aggregate limit, checking that the new value is not below the
// current total amount.
func validateAggregateLimit(totals map[string]int64, key, value string) error {
	parser := aggregateLimitConfigValueParsers[key]
	limit, err := parser(value)
	if err != nil {
		errors.Wrapf(err, "Invalid value '%s' for limit %s", value, key)
	}

	total := totals[key]
	if limit < total {
		return fmt.Errorf("'%s' is too low: current total is %s", key, units.GetByteSizeString(total, 1))
	}

	return nil
}

// Fetch the given project from the database along with its profiles and instances.
func fetchProject(tx *db.ClusterTx, projectName string) (*api.Project, []db.Profile, []db.Instance, error) {
	project, err := tx.ProjectGet(projectName)
	if err != nil {
		return nil, nil, nil, errors.Wrap(err, "Fetch project database object")
	}

	profilesFilter := db.ProfileFilter{}

	// If the project has the profiles feature enabled, we use its own
	// profiles to expand the instances configs, otherwise we use the
	// profiles from the default project.
	if projectName == "default" || project.Config["features.profiles"] == "true" {
		profilesFilter.Project = projectName
	} else {
		profilesFilter.Project = "default"
	}

	profiles, err := tx.ProfileList(profilesFilter)
	if err != nil {
		return nil, nil, nil, errors.Wrap(err, "Fetch profiles from database")
	}

	instances, err := tx.InstanceList(db.InstanceFilter{Project: projectName})
	if err != nil {
		return nil, nil, nil, errors.Wrap(err, "Fetch project instances from database")
	}

	return project, profiles, instances, nil
}

// Expand the configuration of the given instances, taking the give project
// profiles into account.
func expandInstancesConfig(instances []db.Instance, profiles []db.Profile) []db.Instance {
	expandedInstances := make([]db.Instance, len(instances))

	// Index of all profiles by name.
	profilesByName := map[string]db.Profile{}
	for _, profile := range profiles {
		profilesByName[profile.Name] = profile
	}

	for i, instance := range instances {
		profiles := make([]api.Profile, len(instance.Profiles))

		for j, name := range instance.Profiles {
			profile := profilesByName[name]
			profiles[j] = *db.ProfileToAPI(&profile)
		}

		expandedInstances[i] = instance
		expandedInstances[i].Config = db.ProfilesExpandConfig(instance.Config, profiles)
	}

	return expandedInstances
}

// Sum of the effective instance-level value for the given limits across all
// project instances. If excludeInstance is not the empty string, exclude the
// instance with that name.
func getTotalsAcrossInstances(instances []db.Instance, keys []string) (map[string]int64, error) {
	totals := map[string]int64{}

	for _, key := range keys {
		totals[key] = 0
	}

	for _, instance := range instances {
		limits, err := getInstanceLimits(instance, keys)
		if err != nil {
			return nil, err
		}
		for _, key := range keys {
			totals[key] += limits[key]
		}
	}

	return totals, nil
}

// Return the effective instance-level values for the limits with the given
// keys.
func getInstanceLimits(instance db.Instance, keys []string) (map[string]int64, error) {
	limits := map[string]int64{}

	for _, key := range keys {
		value, ok := instance.Config[key]
		if !ok || value == "" {
			return nil, fmt.Errorf(
				"Instance %s in project %s has no '%s' config, either directly or via a profile",
				instance.Name, instance.Project, key)
		}
		parser := aggregateLimitConfigValueParsers[key]
		limit, err := parser(value)
		if err != nil {
			return nil, errors.Wrapf(
				err, "Parse '%s' for instance %s in project %s",
				key, instance.Name, instance.Project)
		}
		limits[key] = limit
	}

	return limits, nil
}

var aggregateLimitConfigValueParsers = map[string]func(string) (int64, error){
	"limits.memory": units.ParseByteSizeString,
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package azure

import (
	"fmt"
	"strings"

	"github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage"
	"github.com/Azure/go-autorest/autorest/to"
	"github.com/golang/glog"
)

type accountWithLocation struct {
	Name, StorageType, Location string
}

// getStorageAccounts gets name, type, location of all storage accounts in a resource group which matches matchingAccountType, matchingLocation
func (az *Cloud) getStorageAccounts(matchingAccountType, matchingLocation string) ([]accountWithLocation, error) {
	ctx, cancel := getContextWithCancel()
	defer cancel()
	result, err := az.StorageAccountClient.ListByResourceGroup(ctx, az.ResourceGroup)
	if err != nil {
		return nil, err
	}
	if result.Value == nil {
		return nil, fmt.Errorf("unexpected error when listing storage accounts from resource group %s", az.ResourceGroup)
	}

	accounts := []accountWithLocation{}
	for _, acct := range *result.Value {
		if acct.Name != nil && acct.Location != nil && acct.Sku != nil {
			storageType := string((*acct.Sku).Name)
			if matchingAccountType != "" && !strings.EqualFold(matchingAccountType, storageType) {
				continue
			}

			location := *acct.Location
			if matchingLocation != "" && !strings.EqualFold(matchingLocation, location) {
				continue
			}
			accounts = append(accounts, accountWithLocation{Name: *acct.Name, StorageType: storageType, Location: location})
		}
	}

	return accounts, nil
}

// getStorageAccesskey gets the storage account access key
func (az *Cloud) getStorageAccesskey(account string) (string, error) {
	ctx, cancel := getContextWithCancel()
	defer cancel()

	result, err := az.StorageAccountClient.ListKeys(ctx, az.ResourceGroup, account)
	if err != nil {
		return "", err
	}
	if result.Keys == nil {
		return "", fmt.Errorf("empty keys")
	}

	for _, k := range *result.Keys {
		if k.Value != nil && *k.Value != "" {
			v := *k.Value
			if ind := strings.LastIndex(v, " "); ind >= 0 {
				v = v[(ind + 1):]
			}
			return v, nil
		}
	}
	return "", fmt.Errorf("no valid keys")
}

// ensureStorageAccount search storage account, create one storage account(with genAccountNamePrefix) if not found, return accountName, accountKey
func (az *Cloud) ensureStorageAccount(accountName, accountType, location, genAccountNamePrefix string) (string, string, error) {
	if len(accountName) == 0 {
		// find a storage account that matches accountType
		accounts, err := az.getStorageAccounts(accountType, location)
		if err != nil {
			return "", "", fmt.Errorf("could not list storage accounts for account type %s: %v", accountType, err)
		}

		if len(accounts) > 0 {
			accountName = accounts[0].Name
			glog.V(4).Infof("found a matching account %s type %s location %s", accounts[0].Name, accounts[0].StorageType, accounts[0].Location)
		}

		if len(accountName) == 0 {
			// not found a matching account, now create a new account in current resource group
			accountName = generateStorageAccountName(genAccountNamePrefix)
			if location == "" {
				location = az.Location
			}
			if accountType == "" {
				accountType = defaultStorageAccountType
			}

			glog.V(2).Infof("azure - no matching account found, begin to create a new account %s in resource group %s, location: %s, accountType: %s",
				accountName, az.ResourceGroup, location, accountType)
			cp := storage.AccountCreateParameters{
				Sku:      &storage.Sku{Name: storage.SkuName(accountType)},
				Tags:     map[string]*string{"created-by": to.StringPtr("azure")},
				Location: &location}

			ctx, cancel := getContextWithCancel()
			defer cancel()
			_, err := az.StorageAccountClient.Create(ctx, az.ResourceGroup, accountName, cp)
			if err != nil {
				return "", "", fmt.Errorf(fmt.Sprintf("Failed to create storage account %s, error: %s", accountName, err))
			}
		}
	}

	// find the access key with this account
	accountKey, err := az.getStorageAccesskey(accountName)
	if err != nil {
		return "", "", fmt.Errorf("could not get storage key for storage account %s: %v", accountName, err)
	}

	return accountName, accountKey, nil
}
<commit_msg>set EnableHTTPSTrafficOnly in storageAccount creation<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package azure

import (
	"fmt"
	"strings"

	"github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage"
	"github.com/Azure/go-autorest/autorest/to"
	"github.com/golang/glog"
)

type accountWithLocation struct {
	Name, StorageType, Location string
}

// getStorageAccounts gets name, type, location of all storage accounts in a resource group which matches matchingAccountType, matchingLocation
func (az *Cloud) getStorageAccounts(matchingAccountType, matchingLocation string) ([]accountWithLocation, error) {
	ctx, cancel := getContextWithCancel()
	defer cancel()
	result, err := az.StorageAccountClient.ListByResourceGroup(ctx, az.ResourceGroup)
	if err != nil {
		return nil, err
	}
	if result.Value == nil {
		return nil, fmt.Errorf("unexpected error when listing storage accounts from resource group %s", az.ResourceGroup)
	}

	accounts := []accountWithLocation{}
	for _, acct := range *result.Value {
		if acct.Name != nil && acct.Location != nil && acct.Sku != nil {
			storageType := string((*acct.Sku).Name)
			if matchingAccountType != "" && !strings.EqualFold(matchingAccountType, storageType) {
				continue
			}

			location := *acct.Location
			if matchingLocation != "" && !strings.EqualFold(matchingLocation, location) {
				continue
			}
			accounts = append(accounts, accountWithLocation{Name: *acct.Name, StorageType: storageType, Location: location})
		}
	}

	return accounts, nil
}

// getStorageAccesskey gets the storage account access key
func (az *Cloud) getStorageAccesskey(account string) (string, error) {
	ctx, cancel := getContextWithCancel()
	defer cancel()

	result, err := az.StorageAccountClient.ListKeys(ctx, az.ResourceGroup, account)
	if err != nil {
		return "", err
	}
	if result.Keys == nil {
		return "", fmt.Errorf("empty keys")
	}

	for _, k := range *result.Keys {
		if k.Value != nil && *k.Value != "" {
			v := *k.Value
			if ind := strings.LastIndex(v, " "); ind >= 0 {
				v = v[(ind + 1):]
			}
			return v, nil
		}
	}
	return "", fmt.Errorf("no valid keys")
}

// ensureStorageAccount search storage account, create one storage account(with genAccountNamePrefix) if not found, return accountName, accountKey
func (az *Cloud) ensureStorageAccount(accountName, accountType, location, genAccountNamePrefix string) (string, string, error) {
	if len(accountName) == 0 {
		// find a storage account that matches accountType
		accounts, err := az.getStorageAccounts(accountType, location)
		if err != nil {
			return "", "", fmt.Errorf("could not list storage accounts for account type %s: %v", accountType, err)
		}

		if len(accounts) > 0 {
			accountName = accounts[0].Name
			glog.V(4).Infof("found a matching account %s type %s location %s", accounts[0].Name, accounts[0].StorageType, accounts[0].Location)
		}

		if len(accountName) == 0 {
			// not found a matching account, now create a new account in current resource group
			accountName = generateStorageAccountName(genAccountNamePrefix)
			if location == "" {
				location = az.Location
			}
			if accountType == "" {
				accountType = defaultStorageAccountType
			}

			glog.V(2).Infof("azure - no matching account found, begin to create a new account %s in resource group %s, location: %s, accountType: %s",
				accountName, az.ResourceGroup, location, accountType)
			cp := storage.AccountCreateParameters{
				Sku: &storage.Sku{Name: storage.SkuName(accountType)},
				AccountPropertiesCreateParameters: &storage.AccountPropertiesCreateParameters{EnableHTTPSTrafficOnly: to.BoolPtr(true)},
				Tags:     map[string]*string{"created-by": to.StringPtr("azure")},
				Location: &location}

			ctx, cancel := getContextWithCancel()
			defer cancel()
			_, err := az.StorageAccountClient.Create(ctx, az.ResourceGroup, accountName, cp)
			if err != nil {
				return "", "", fmt.Errorf(fmt.Sprintf("Failed to create storage account %s, error: %s", accountName, err))
			}
		}
	}

	// find the access key with this account
	accountKey, err := az.getStorageAccesskey(accountName)
	if err != nil {
		return "", "", fmt.Errorf("could not get storage key for storage account %s: %v", accountName, err)
	}

	return accountName, accountKey, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package topologymanager

import (
	"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask"
	"k8s.io/kubernetes/pkg/kubelet/lifecycle"
)

type singleNumaNodePolicy struct {
	//List of NUMA Nodes available on the underlying machine
	numaNodes []int
}

var _ Policy = &singleNumaNodePolicy{}

// PolicySingleNumaNode policy name.
const PolicySingleNumaNode string = "single-numa-node"

// NewSingleNumaNodePolicy returns single-numa-node policy.
func NewSingleNumaNodePolicy(numaNodes []int) Policy {
	return &singleNumaNodePolicy{numaNodes: numaNodes}
}

func (p *singleNumaNodePolicy) Name() string {
	return PolicySingleNumaNode
}

func (p *singleNumaNodePolicy) canAdmitPodResult(hint *TopologyHint) lifecycle.PodAdmitResult {
	if !hint.Preferred {
		return lifecycle.PodAdmitResult{
			Admit:   false,
			Reason:  "Topology Affinity Error",
			Message: "Resources cannot be allocated with Topology Locality",
		}
	}
	return lifecycle.PodAdmitResult{
		Admit: true,
	}
}

// Return hints that have valid bitmasks with exactly one bit set.
func filterSingleNumaHints(allResourcesHints [][]TopologyHint) [][]TopologyHint {
	var filteredResourcesHints [][]TopologyHint
	for _, oneResourceHints := range allResourcesHints {
		var filtered []TopologyHint
		for _, hint := range oneResourceHints {
			if hint.NUMANodeAffinity == nil && hint.Preferred == true {
				filtered = append(filtered, hint)
			}
			if hint.NUMANodeAffinity != nil && hint.NUMANodeAffinity.Count() == 1 && hint.Preferred == true {
				filtered = append(filtered, hint)
			}
		}
		filteredResourcesHints = append(filteredResourcesHints, filtered)
	}
	return filteredResourcesHints
}

func (p *singleNumaNodePolicy) mergeProvidersHints(filteredHints [][]TopologyHint) TopologyHint {
	// Set the default affinity as an any-numa affinity containing the list
	// of NUMA Nodes available on this machine.
	defaultAffinity, _ := bitmask.NewBitMask(p.numaNodes...)

	// Filter to only include don't cares and hints with a single NUMA node.
	filteredHints = filterSingleNumaHints(filteredHints)

	// Set the bestHint to return from this function as {nil false}.
	// This will only be returned if no better hint can be found when
	// merging hints from each hint provider.
	bestHint := TopologyHint{defaultAffinity, false}
	iterateAllProviderTopologyHints(filteredHints, func(permutation []TopologyHint) {
		// Get the NUMANodeAffinity from each hint in the permutation and see if any
		// of them encode unpreferred allocations.
		mergedHint := mergePermutation(p.numaNodes, permutation)

		// Only consider mergedHints that result in a NUMANodeAffinity > 0 to
		// replace the current bestHint.
		if mergedHint.NUMANodeAffinity.Count() == 0 {
			return
		}

		// If the current bestHint is non-preferred and the new mergedHint is
		// preferred, always choose the preferred hint over the non-preferred one.
		if mergedHint.Preferred && !bestHint.Preferred {
			bestHint = mergedHint
			return
		}

		// If the current bestHint is preferred and the new mergedHint is
		// non-preferred, never update bestHint, regardless of mergedHint's
		// narowness.
		if !mergedHint.Preferred && bestHint.Preferred {
			return
		}

		// If mergedHint and bestHint has the same preference, only consider
		// mergedHints that have a narrower NUMANodeAffinity than the
		// NUMANodeAffinity in the current bestHint.
		if !mergedHint.NUMANodeAffinity.IsNarrowerThan(bestHint.NUMANodeAffinity) {
			return
		}

		// In all other cases, update bestHint to the current mergedHint
		bestHint = mergedHint
	})

	if bestHint.NUMANodeAffinity.IsEqual(defaultAffinity) {
		bestHint = TopologyHint{nil, bestHint.Preferred}
	}

	return bestHint
}

func (p *singleNumaNodePolicy) Merge(providersHints []map[string][]TopologyHint) (TopologyHint, lifecycle.PodAdmitResult) {
	filteredHints := filterProvidersHints(providersHints)
	hint := p.mergeProvidersHints(filteredHints)
	admit := p.canAdmitPodResult(&hint)
	return hint, admit
}
<commit_msg>Move filterSingleNumaHints call to top level Merge<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package topologymanager

import (
	"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask"
	"k8s.io/kubernetes/pkg/kubelet/lifecycle"
)

type singleNumaNodePolicy struct {
	//List of NUMA Nodes available on the underlying machine
	numaNodes []int
}

var _ Policy = &singleNumaNodePolicy{}

// PolicySingleNumaNode policy name.
const PolicySingleNumaNode string = "single-numa-node"

// NewSingleNumaNodePolicy returns single-numa-node policy.
func NewSingleNumaNodePolicy(numaNodes []int) Policy {
	return &singleNumaNodePolicy{numaNodes: numaNodes}
}

func (p *singleNumaNodePolicy) Name() string {
	return PolicySingleNumaNode
}

func (p *singleNumaNodePolicy) canAdmitPodResult(hint *TopologyHint) lifecycle.PodAdmitResult {
	if !hint.Preferred {
		return lifecycle.PodAdmitResult{
			Admit:   false,
			Reason:  "Topology Affinity Error",
			Message: "Resources cannot be allocated with Topology Locality",
		}
	}
	return lifecycle.PodAdmitResult{
		Admit: true,
	}
}

// Return hints that have valid bitmasks with exactly one bit set.
func filterSingleNumaHints(allResourcesHints [][]TopologyHint) [][]TopologyHint {
	var filteredResourcesHints [][]TopologyHint
	for _, oneResourceHints := range allResourcesHints {
		var filtered []TopologyHint
		for _, hint := range oneResourceHints {
			if hint.NUMANodeAffinity == nil && hint.Preferred == true {
				filtered = append(filtered, hint)
			}
			if hint.NUMANodeAffinity != nil && hint.NUMANodeAffinity.Count() == 1 && hint.Preferred == true {
				filtered = append(filtered, hint)
			}
		}
		filteredResourcesHints = append(filteredResourcesHints, filtered)
	}
	return filteredResourcesHints
}

func (p *singleNumaNodePolicy) mergeProvidersHints(filteredHints [][]TopologyHint) TopologyHint {
	// Set the default affinity as an any-numa affinity containing the list
	// of NUMA Nodes available on this machine.
	defaultAffinity, _ := bitmask.NewBitMask(p.numaNodes...)

	// Set the bestHint to return from this function as {nil false}.
	// This will only be returned if no better hint can be found when
	// merging hints from each hint provider.
	bestHint := TopologyHint{defaultAffinity, false}
	iterateAllProviderTopologyHints(filteredHints, func(permutation []TopologyHint) {
		// Get the NUMANodeAffinity from each hint in the permutation and see if any
		// of them encode unpreferred allocations.
		mergedHint := mergePermutation(p.numaNodes, permutation)

		// Only consider mergedHints that result in a NUMANodeAffinity > 0 to
		// replace the current bestHint.
		if mergedHint.NUMANodeAffinity.Count() == 0 {
			return
		}

		// If the current bestHint is non-preferred and the new mergedHint is
		// preferred, always choose the preferred hint over the non-preferred one.
		if mergedHint.Preferred && !bestHint.Preferred {
			bestHint = mergedHint
			return
		}

		// If the current bestHint is preferred and the new mergedHint is
		// non-preferred, never update bestHint, regardless of mergedHint's
		// narowness.
		if !mergedHint.Preferred && bestHint.Preferred {
			return
		}

		// If mergedHint and bestHint has the same preference, only consider
		// mergedHints that have a narrower NUMANodeAffinity than the
		// NUMANodeAffinity in the current bestHint.
		if !mergedHint.NUMANodeAffinity.IsNarrowerThan(bestHint.NUMANodeAffinity) {
			return
		}

		// In all other cases, update bestHint to the current mergedHint
		bestHint = mergedHint
	})

	if bestHint.NUMANodeAffinity.IsEqual(defaultAffinity) {
		bestHint = TopologyHint{nil, bestHint.Preferred}
	}

	return bestHint
}

func (p *singleNumaNodePolicy) Merge(providersHints []map[string][]TopologyHint) (TopologyHint, lifecycle.PodAdmitResult) {
	filteredHints := filterProvidersHints(providersHints)
	// Filter to only include don't cares and hints with a single NUMA node.
	singleNumaHints := filterSingleNumaHints(filteredHints)
	hint := p.mergeProvidersHints(singleNumaHints)
	admit := p.canAdmitPodResult(&hint)
	return hint, admit
}
<|endoftext|>


<commit_before>// Copyright 2012 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package app

import (
	"bytes"
	"crypto/rand"
	"fmt"
	"github.com/globocom/config"
	"io"
	"launchpad.net/goamz/aws"
	"launchpad.net/goamz/iam"
	"launchpad.net/goamz/s3"
	"strings"
	"text/template"
)

type s3Env struct {
	aws.Auth
	bucket             string
	endpoint           string
	locationConstraint bool
}

func (s *s3Env) empty() bool {
	return s.bucket == "" || s.AccessKey == "" || s.SecretKey == ""
}

const (
	randBytes      = 32
	s3InstanceName = "tsurus3"
)

var (
	rReader = rand.Reader
	policy  = template.Must(template.New("policy").Parse(`{
  "Statement": [
    {
      "Action": [
        "s3:CreateBucket",
        "s3:DeleteBucket",
        "s3:DeleteBucketPolicy",
        "s3:DeleteBucketWebsite",
        "s3:DeleteObject",
        "s3:DeleteObjectVersion",
        "s3:PutBucketLogging",
        "s3:PutBucketPolicy",
        "s3:PutBucketRequestPayment",
        "s3:PutBucketVersioning",
        "s3:PutBucketWebsite"
      ],
      "Effect": "Deny",
      "Resource": [
        "arn:aws:s3:::{{.}}/*",
        "arn:aws:s3:::{{.}}"
      ]
    },
    {
      "Action": [
        "s3:*"
      ],
      "Effect": "Allow",
      "Resource": [
        "arn:aws:s3:::{{.}}/*",
        "arn:aws:s3:::{{.}}"
      ]
    }
  ]
}`))
)

func getAWSAuth() aws.Auth {
	access, err := config.GetString("aws:access-key-id")
	if err != nil {
		panic("FATAL: aws:access-key-id must be defined in configuration file.")
	}
	secret, err := config.GetString("aws:secret-access-key")
	if err != nil {
		panic("FATAL: aws:secret-access-key must be defined in configuration file.")
	}
	return aws.Auth{
		AccessKey: access,
		SecretKey: secret,
	}
}

func getS3Endpoint() *s3.S3 {
	regionName, _ := config.GetString("aws:s3:region-name")
	endpoint, err := config.GetString("aws:s3:endpoint")
	if err != nil {
		panic("FATAL: aws:s3:endpoint must be defined in configuration file.")
	}
	bucketEndpoint, _ := config.GetString("aws:s3:bucketEndpoint")
	locationConstraint, err := config.GetBool("aws:s3:location-constraint")
	if err != nil {
		panic("FATAL: aws:s3:location-constraint must be defined in configuration file.")
	}
	lowercaseBucket, err := config.GetBool("aws:s3:lowercase-bucket")
	if err != nil {
		panic("FATAL: aws:s3:lowercase-bucket must be defined in configuration file.")
	}
	region := aws.Region{
		Name:                 regionName,
		S3Endpoint:           endpoint,
		S3BucketEndpoint:     bucketEndpoint,
		S3LocationConstraint: locationConstraint,
		S3LowercaseBucket:    lowercaseBucket,
	}
	return s3.New(getAWSAuth(), region)
}

func getIAMEndpoint() *iam.IAM {
	endpoint, err := config.GetString("aws:iam:endpoint")
	if err != nil {
		panic("FATAL: aws:iam:endpoint must be defined in configuration file.")
	}
	region := aws.Region{IAMEndpoint: endpoint}
	return iam.New(getAWSAuth(), region)
}

func createBucket(app *App) (*s3Env, error) {
	var env s3Env
	appName := strings.ToLower(app.Name)
	errChan := make(chan error)
	bChan := make(chan s3.Bucket)
	kChan := make(chan iam.AccessKey)
	go func(c chan s3.Bucket) {
		randPart := make([]byte, randBytes)
		n, err := rReader.Read(randPart)
		if err != nil {
			errChan <- err
			return
		}
		if n != randBytes {
			errChan <- io.ErrShortBuffer
			return
		}
		name := fmt.Sprintf("%s%x", appName, randPart)
		s := getS3Endpoint()
		env.endpoint = s.S3Endpoint
		env.locationConstraint = s.S3LocationConstraint
		bucket := s.Bucket(name)
		if err := bucket.PutBucket(s3.BucketOwnerFull); err != nil {
			errChan <- err
			return
		}
		c <- *bucket
	}(bChan)
	iamEndpoint := getIAMEndpoint()
	go func(c chan iam.AccessKey) {
		uResp, err := iamEndpoint.CreateUser(appName, fmt.Sprintf("/%s/", appName))
		if err != nil {
			errChan <- err
			return
		}
		kResp, err := iamEndpoint.CreateAccessKey(uResp.User.Name)
		if err != nil {
			errChan <- err
			return
		}
		c <- kResp.AccessKey
	}(kChan)
	var p iam.UserPolicy
	for env.empty() {
		select {
		case k := <-kChan:
			env.AccessKey = k.Id
			env.SecretKey = k.Secret
			p.User = k.UserName
		case bucket := <-bChan:
			env.bucket = bucket.Name
		case err := <-errChan:
			return nil, err
		}
	}
	p.Name = fmt.Sprintf("app-%s-bucket", appName)
	var buf bytes.Buffer
	policy.Execute(&buf, env.bucket)
	p.Document = buf.String()
	if _, err := iamEndpoint.PutUserPolicy(p); err != nil {
		return nil, err
	}
	return &env, nil
}

func destroyBucket(app *App) error {
	appName := strings.ToLower(app.Name)
	env := app.InstanceEnv(s3InstanceName)
	accessKeyId := env["TSURU_S3_ACCESS_KEY_ID"].Value
	bucketName := env["TSURU_S3_BUCKET"].Value
	policyName := fmt.Sprintf("app-%s-bucket", appName)
	s3Endpoint := getS3Endpoint()
	iamEndpoint := getIAMEndpoint()
	if _, err := iamEndpoint.DeleteUserPolicy(policyName, appName); err != nil {
		return err
	}
	bucket := s3Endpoint.Bucket(bucketName)
	if err := bucket.DelBucket(); err != nil {
		return err
	}
	if _, err := iamEndpoint.DeleteAccessKey(accessKeyId); err != nil {
		return err
	}
	_, err := iamEndpoint.DeleteUser(appName)
	return err
}
<commit_msg>api/app: allow user to delete objects<commit_after>// Copyright 2012 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package app

import (
	"bytes"
	"crypto/rand"
	"fmt"
	"github.com/globocom/config"
	"io"
	"launchpad.net/goamz/aws"
	"launchpad.net/goamz/iam"
	"launchpad.net/goamz/s3"
	"strings"
	"text/template"
)

type s3Env struct {
	aws.Auth
	bucket             string
	endpoint           string
	locationConstraint bool
}

func (s *s3Env) empty() bool {
	return s.bucket == "" || s.AccessKey == "" || s.SecretKey == ""
}

const (
	randBytes      = 32
	s3InstanceName = "tsurus3"
)

var (
	rReader = rand.Reader
	policy  = template.Must(template.New("policy").Parse(`{
  "Statement": [
    {
      "Action": [
        "s3:CreateBucket",
        "s3:DeleteBucket",
        "s3:DeleteBucketPolicy",
        "s3:DeleteBucketWebsite",
        "s3:PutBucketLogging",
        "s3:PutBucketPolicy",
        "s3:PutBucketRequestPayment",
        "s3:PutBucketVersioning",
        "s3:PutBucketWebsite"
      ],
      "Effect": "Deny",
      "Resource": [
        "arn:aws:s3:::{{.}}/*",
        "arn:aws:s3:::{{.}}"
      ]
    },
    {
      "Action": [
        "s3:*"
      ],
      "Effect": "Allow",
      "Resource": [
        "arn:aws:s3:::{{.}}/*",
        "arn:aws:s3:::{{.}}"
      ]
    }
  ]
}`))
)

func getAWSAuth() aws.Auth {
	access, err := config.GetString("aws:access-key-id")
	if err != nil {
		panic("FATAL: aws:access-key-id must be defined in configuration file.")
	}
	secret, err := config.GetString("aws:secret-access-key")
	if err != nil {
		panic("FATAL: aws:secret-access-key must be defined in configuration file.")
	}
	return aws.Auth{
		AccessKey: access,
		SecretKey: secret,
	}
}

func getS3Endpoint() *s3.S3 {
	regionName, _ := config.GetString("aws:s3:region-name")
	endpoint, err := config.GetString("aws:s3:endpoint")
	if err != nil {
		panic("FATAL: aws:s3:endpoint must be defined in configuration file.")
	}
	bucketEndpoint, _ := config.GetString("aws:s3:bucketEndpoint")
	locationConstraint, err := config.GetBool("aws:s3:location-constraint")
	if err != nil {
		panic("FATAL: aws:s3:location-constraint must be defined in configuration file.")
	}
	lowercaseBucket, err := config.GetBool("aws:s3:lowercase-bucket")
	if err != nil {
		panic("FATAL: aws:s3:lowercase-bucket must be defined in configuration file.")
	}
	region := aws.Region{
		Name:                 regionName,
		S3Endpoint:           endpoint,
		S3BucketEndpoint:     bucketEndpoint,
		S3LocationConstraint: locationConstraint,
		S3LowercaseBucket:    lowercaseBucket,
	}
	return s3.New(getAWSAuth(), region)
}

func getIAMEndpoint() *iam.IAM {
	endpoint, err := config.GetString("aws:iam:endpoint")
	if err != nil {
		panic("FATAL: aws:iam:endpoint must be defined in configuration file.")
	}
	region := aws.Region{IAMEndpoint: endpoint}
	return iam.New(getAWSAuth(), region)
}

func createBucket(app *App) (*s3Env, error) {
	var env s3Env
	appName := strings.ToLower(app.Name)
	errChan := make(chan error)
	bChan := make(chan s3.Bucket)
	kChan := make(chan iam.AccessKey)
	go func(c chan s3.Bucket) {
		randPart := make([]byte, randBytes)
		n, err := rReader.Read(randPart)
		if err != nil {
			errChan <- err
			return
		}
		if n != randBytes {
			errChan <- io.ErrShortBuffer
			return
		}
		name := fmt.Sprintf("%s%x", appName, randPart)
		s := getS3Endpoint()
		env.endpoint = s.S3Endpoint
		env.locationConstraint = s.S3LocationConstraint
		bucket := s.Bucket(name)
		if err := bucket.PutBucket(s3.BucketOwnerFull); err != nil {
			errChan <- err
			return
		}
		c <- *bucket
	}(bChan)
	iamEndpoint := getIAMEndpoint()
	go func(c chan iam.AccessKey) {
		uResp, err := iamEndpoint.CreateUser(appName, fmt.Sprintf("/%s/", appName))
		if err != nil {
			errChan <- err
			return
		}
		kResp, err := iamEndpoint.CreateAccessKey(uResp.User.Name)
		if err != nil {
			errChan <- err
			return
		}
		c <- kResp.AccessKey
	}(kChan)
	var p iam.UserPolicy
	for env.empty() {
		select {
		case k := <-kChan:
			env.AccessKey = k.Id
			env.SecretKey = k.Secret
			p.User = k.UserName
		case bucket := <-bChan:
			env.bucket = bucket.Name
		case err := <-errChan:
			return nil, err
		}
	}
	p.Name = fmt.Sprintf("app-%s-bucket", appName)
	var buf bytes.Buffer
	policy.Execute(&buf, env.bucket)
	p.Document = buf.String()
	if _, err := iamEndpoint.PutUserPolicy(p); err != nil {
		return nil, err
	}
	return &env, nil
}

func destroyBucket(app *App) error {
	appName := strings.ToLower(app.Name)
	env := app.InstanceEnv(s3InstanceName)
	accessKeyId := env["TSURU_S3_ACCESS_KEY_ID"].Value
	bucketName := env["TSURU_S3_BUCKET"].Value
	policyName := fmt.Sprintf("app-%s-bucket", appName)
	s3Endpoint := getS3Endpoint()
	iamEndpoint := getIAMEndpoint()
	if _, err := iamEndpoint.DeleteUserPolicy(policyName, appName); err != nil {
		return err
	}
	bucket := s3Endpoint.Bucket(bucketName)
	if err := bucket.DelBucket(); err != nil {
		return err
	}
	if _, err := iamEndpoint.DeleteAccessKey(accessKeyId); err != nil {
		return err
	}
	_, err := iamEndpoint.DeleteUser(appName)
	return err
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"net/http"
	"testing"
	"time"

	"github.com/GoogleCloudPlatform/golang-samples/internal/cloudrunci"
	"github.com/GoogleCloudPlatform/golang-samples/internal/testutil"
)

func TestPubSubEventsService(t *testing.T) {
	tc := testutil.EndToEndTest(t)

	service := cloudrunci.NewService("run-events-storage", tc.ProjectID)
	if err := service.Deploy(); err != nil {
		t.Fatalf("service.Deploy %q: %v", service.Name, err)
	}
	defer service.Clean()

	requestPath := "/"
	req, err := service.NewRequest("POST", requestPath)
	if err != nil {
		t.Fatalf("service.NewRequest: %v", err)
	}

	client := http.Client{Timeout: 10 * time.Second}
	resp, err := client.Do(req)
	if err != nil {
		t.Fatalf("client.Do: %v", err)
	}
	defer resp.Body.Close()
	fmt.Printf("client.Do: %s %s\n", req.Method, req.URL)

	if got := resp.StatusCode; got != http.StatusBadRequest {
		t.Errorf("response status: got %d, want %d", got, http.StatusBadRequest)
	}
}
<commit_msg>fix(run/events-storage): e2e test checking for invalid criteria (#1676)<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"net/http"
	"testing"
	"time"

	"github.com/GoogleCloudPlatform/golang-samples/internal/cloudrunci"
	"github.com/GoogleCloudPlatform/golang-samples/internal/testutil"
)

func TestPubSubEventsService(t *testing.T) {
	tc := testutil.EndToEndTest(t)

	service := cloudrunci.NewService("events-storage", tc.ProjectID)
	if err := service.Deploy(); err != nil {
		t.Fatalf("service.Deploy %q: %v", service.Name, err)
	}
	defer service.Clean()

	requestPath := "/"
	req, err := service.NewRequest("POST", requestPath)
	if err != nil {
		t.Fatalf("service.NewRequest: %v", err)
	}

	client := http.Client{Timeout: 10 * time.Second}
	resp, err := client.Do(req)
	if err != nil {
		t.Fatalf("client.Do: %v", err)
	}
	defer resp.Body.Close()
	fmt.Printf("client.Do: %s %s\n", req.Method, req.URL)

	if got := resp.StatusCode; got != http.StatusOK {
		t.Errorf("response status: got %d, want %d", got, http.StatusOK)
	}
}
<|endoftext|>


<commit_before>package healthsyncer

import (
	"context"
	"fmt"
	"reflect"
	"sort"
	"time"

	"github.com/pkg/errors"
	"github.com/rancher/norman/condition"
	v32 "github.com/rancher/rancher/pkg/apis/management.cattle.io/v3"
	corev1 "github.com/rancher/rancher/pkg/generated/norman/core/v1"
	v3 "github.com/rancher/rancher/pkg/generated/norman/management.cattle.io/v3"
	"github.com/rancher/rancher/pkg/namespace"
	"github.com/rancher/rancher/pkg/types/config"
	"github.com/rancher/wrangler/pkg/ticker"
	"github.com/sirupsen/logrus"
	appsv1 "k8s.io/api/apps/v1"
	v1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/client-go/kubernetes"
)

const (
	syncInterval = 15 * time.Second
)

type ClusterControllerLifecycle interface {
	Stop(cluster *v3.Cluster)
}

type HealthSyncer struct {
	ctx               context.Context
	clusterName       string
	clusterLister     v3.ClusterLister
	clusters          v3.ClusterInterface
	componentStatuses corev1.ComponentStatusInterface
	namespaces        corev1.NamespaceInterface
	k8s               kubernetes.Interface
}

func Register(ctx context.Context, workload *config.UserContext) {
	h := &HealthSyncer{
		ctx:               ctx,
		clusterName:       workload.ClusterName,
		clusterLister:     workload.Management.Management.Clusters("").Controller().Lister(),
		clusters:          workload.Management.Management.Clusters(""),
		componentStatuses: workload.Core.ComponentStatuses(""),
		namespaces:        workload.Core.Namespaces(""),
		k8s:               workload.K8sClient,
	}

	go h.syncHealth(ctx, syncInterval)
}

func (h *HealthSyncer) syncHealth(ctx context.Context, syncHealth time.Duration) {
	for range ticker.Context(ctx, syncHealth) {
		err := h.updateClusterHealth()
		if err != nil && !apierrors.IsConflict(err) {
			logrus.Error(err)
		}
	}
}

func (h *HealthSyncer) getComponentStatus(cluster *v3.Cluster) error {
	ctx, cancel := context.WithTimeout(h.ctx, 5*time.Second)
	defer cancel()

	// Prior to k8s v1.14, we only needed to list the ComponentStatuses from the user cluster.
	// As of k8s v1.14, kubeapi returns a successful ComponentStatuses response even if etcd is not available.
	// To work around this, now we try to get a namespace from the API, even if not found, it means the API is up.
	if _, err := h.k8s.CoreV1().Namespaces().Get(ctx, "kube-system", metav1.GetOptions{}); err != nil && !apierrors.IsNotFound(err) {
		return condition.Error("ComponentStatsFetchingFailure", errors.Wrap(err, "Failed to communicate with API server"))
	}

	cses, err := h.componentStatuses.List(metav1.ListOptions{})
	if err != nil {
		return condition.Error("ComponentStatsFetchingFailure", errors.Wrap(err, "Failed to communicate with API server"))
	}
	cluster.Status.ComponentStatuses = []v32.ClusterComponentStatus{}
	for _, cs := range cses.Items {
		clusterCS := convertToClusterComponentStatus(&cs)
		cluster.Status.ComponentStatuses = append(cluster.Status.ComponentStatuses, *clusterCS)
	}
	sort.Slice(cluster.Status.ComponentStatuses, func(i, j int) bool {
		return cluster.Status.ComponentStatuses[i].Name < cluster.Status.ComponentStatuses[j].Name
	})
	return nil
}

func (h *HealthSyncer) updateClusterHealth() error {
	oldCluster, err := h.getCluster()
	if err != nil {
		return err
	}
	cluster := oldCluster.DeepCopy()
	if !v32.ClusterConditionProvisioned.IsTrue(cluster) {
		logrus.Debugf("Skip updating cluster health - cluster [%s] not provisioned yet", h.clusterName)
		return nil
	}

	wasReady := v32.ClusterConditionReady.IsTrue(cluster)
	newObj, err := v32.ClusterConditionReady.Do(cluster, func() (runtime.Object, error) {
		for i := 0; ; i++ {
			err := h.getComponentStatus(cluster)
			if err == nil && !wasReady {
				err = h.checkClusterAgent()
			}
			if err == nil || i > 1 {
				return cluster, errors.Wrap(err, "cluster health check failed")
			}
			select {
			case <-h.ctx.Done():
				return cluster, err
			case <-time.After(5 * time.Second):
			}
		}
	})

	if err == nil {
		v32.ClusterConditionWaiting.True(newObj)
		v32.ClusterConditionWaiting.Message(newObj, "")
	}

	if !reflect.DeepEqual(oldCluster, newObj) {
		if _, err := h.clusters.Update(newObj.(*v3.Cluster)); err != nil {
			return errors.Wrapf(err, "[updateClusterHealth] Failed to update cluster [%s]", cluster.Name)
		}
	}

	// Purposefully not return error.  This is so when the cluster goes unavailable we don't just keep failing
	// which will essentially keep the controller alive forever, instead of shutting down.
	return nil
}

func (h *HealthSyncer) checkClusterAgent() error {
	if c, err := h.clusterLister.Get("", h.clusterName); err == nil && c.Spec.Internal {
		return nil
	}

	deployment, err := h.k8s.AppsV1().Deployments(namespace.System).Get(h.ctx, "cattle-cluster-agent", metav1.GetOptions{})
	if err != nil {
		return err
	}
	if condition.Cond(appsv1.DeploymentAvailable).IsTrue(deployment) {
		return nil
	}
	return fmt.Errorf("cluster agent is not ready")
}

func (h *HealthSyncer) getCluster() (*v3.Cluster, error) {
	return h.clusterLister.Get("", h.clusterName)
}

func convertToClusterComponentStatus(cs *v1.ComponentStatus) *v32.ClusterComponentStatus {
	return &v32.ClusterComponentStatus{
		Name:       cs.Name,
		Conditions: cs.Conditions,
	}
}
<commit_msg>removing cluster agent check before cluster is ready<commit_after>package healthsyncer

import (
	"context"
	"reflect"
	"sort"
	"time"

	"github.com/pkg/errors"
	"github.com/rancher/norman/condition"
	v32 "github.com/rancher/rancher/pkg/apis/management.cattle.io/v3"
	corev1 "github.com/rancher/rancher/pkg/generated/norman/core/v1"
	v3 "github.com/rancher/rancher/pkg/generated/norman/management.cattle.io/v3"
	"github.com/rancher/rancher/pkg/types/config"
	"github.com/rancher/wrangler/pkg/ticker"
	"github.com/sirupsen/logrus"
	v1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/client-go/kubernetes"
)

const (
	syncInterval = 15 * time.Second
)

type ClusterControllerLifecycle interface {
	Stop(cluster *v3.Cluster)
}

type HealthSyncer struct {
	ctx               context.Context
	clusterName       string
	clusterLister     v3.ClusterLister
	clusters          v3.ClusterInterface
	componentStatuses corev1.ComponentStatusInterface
	namespaces        corev1.NamespaceInterface
	k8s               kubernetes.Interface
}

func Register(ctx context.Context, workload *config.UserContext) {
	h := &HealthSyncer{
		ctx:               ctx,
		clusterName:       workload.ClusterName,
		clusterLister:     workload.Management.Management.Clusters("").Controller().Lister(),
		clusters:          workload.Management.Management.Clusters(""),
		componentStatuses: workload.Core.ComponentStatuses(""),
		namespaces:        workload.Core.Namespaces(""),
		k8s:               workload.K8sClient,
	}

	go h.syncHealth(ctx, syncInterval)
}

func (h *HealthSyncer) syncHealth(ctx context.Context, syncHealth time.Duration) {
	for range ticker.Context(ctx, syncHealth) {
		err := h.updateClusterHealth()
		if err != nil && !apierrors.IsConflict(err) {
			logrus.Error(err)
		}
	}
}

func (h *HealthSyncer) getComponentStatus(cluster *v3.Cluster) error {
	ctx, cancel := context.WithTimeout(h.ctx, 5*time.Second)
	defer cancel()

	// Prior to k8s v1.14, we only needed to list the ComponentStatuses from the user cluster.
	// As of k8s v1.14, kubeapi returns a successful ComponentStatuses response even if etcd is not available.
	// To work around this, now we try to get a namespace from the API, even if not found, it means the API is up.
	if _, err := h.k8s.CoreV1().Namespaces().Get(ctx, "kube-system", metav1.GetOptions{}); err != nil && !apierrors.IsNotFound(err) {
		return condition.Error("ComponentStatsFetchingFailure", errors.Wrap(err, "Failed to communicate with API server"))
	}

	cses, err := h.componentStatuses.List(metav1.ListOptions{})
	if err != nil {
		return condition.Error("ComponentStatsFetchingFailure", errors.Wrap(err, "Failed to communicate with API server"))
	}
	cluster.Status.ComponentStatuses = []v32.ClusterComponentStatus{}
	for _, cs := range cses.Items {
		clusterCS := convertToClusterComponentStatus(&cs)
		cluster.Status.ComponentStatuses = append(cluster.Status.ComponentStatuses, *clusterCS)
	}
	sort.Slice(cluster.Status.ComponentStatuses, func(i, j int) bool {
		return cluster.Status.ComponentStatuses[i].Name < cluster.Status.ComponentStatuses[j].Name
	})
	return nil
}

func (h *HealthSyncer) updateClusterHealth() error {
	oldCluster, err := h.getCluster()
	if err != nil {
		return err
	}
	cluster := oldCluster.DeepCopy()
	if !v32.ClusterConditionProvisioned.IsTrue(cluster) {
		logrus.Debugf("Skip updating cluster health - cluster [%s] not provisioned yet", h.clusterName)
		return nil
	}

	newObj, err := v32.ClusterConditionReady.Do(cluster, func() (runtime.Object, error) {
		for i := 0; ; i++ {
			err := h.getComponentStatus(cluster)
			if err == nil || i > 1 {
				return cluster, errors.Wrap(err, "cluster health check failed")
			}
			select {
			case <-h.ctx.Done():
				return cluster, err
			case <-time.After(5 * time.Second):
			}
		}
	})

	if err == nil {
		v32.ClusterConditionWaiting.True(newObj)
		v32.ClusterConditionWaiting.Message(newObj, "")
	}

	if !reflect.DeepEqual(oldCluster, newObj) {
		if _, err := h.clusters.Update(newObj.(*v3.Cluster)); err != nil {
			return errors.Wrapf(err, "[updateClusterHealth] Failed to update cluster [%s]", cluster.Name)
		}
	}

	// Purposefully not return error.  This is so when the cluster goes unavailable we don't just keep failing
	// which will essentially keep the controller alive forever, instead of shutting down.
	return nil
}

func (h *HealthSyncer) getCluster() (*v3.Cluster, error) {
	return h.clusterLister.Get("", h.clusterName)
}

func convertToClusterComponentStatus(cs *v1.ComponentStatus) *v32.ClusterComponentStatus {
	return &v32.ClusterComponentStatus{
		Name:       cs.Name,
		Conditions: cs.Conditions,
	}
}
<|endoftext|>