<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package publicsuffix provides a public suffix list based on data from
// http://publicsuffix.org/. A public suffix is one under which Internet users
// can directly register names.
package publicsuffix

// TODO: specify case sensitivity and leading/trailing dot behavior for
// func PublicSuffix and func EffectiveTLDPlusOne.

import (
	"exp/cookiejar"
	"fmt"
	"strings"
)

// List implements the cookiejar.PublicSuffixList interface by calling the
// PublicSuffix function.
var List cookiejar.PublicSuffixList = list{}

type list struct{}

func (list) PublicSuffix(domain string) string {
	ps, _ := PublicSuffix(domain)
	return ps
}

func (list) String() string {
	return version
}

// PublicSuffix returns the public suffix of the domain using a copy of the
// publicsuffix.org database compiled into the library.
//
// icann is whether the public suffix is managed by the Internet Corporation
// for Assigned Names and Numbers. If not, the public suffix is privately
// managed. For example, foo.org and foo.co.uk are ICANN domains,
// foo.dyndns.org and foo.blogspot.co.uk are private domains.
//
// Use cases for distinguishing ICANN domains like foo.com from private
// domains like foo.appspot.com can be found at
// https://wiki.mozilla.org/Public_Suffix_List/Use_Cases
func PublicSuffix(domain string) (publicSuffix string, icann bool) {
	lo, hi := uint32(0), uint32(numTLD)
	s, suffix, wildcard := domain, len(domain), false
loop:
	for {
		dot := strings.LastIndex(s, ".")
		if wildcard {
			suffix = 1 + dot
		}
		if lo == hi {
			break
		}
		f := find(s[1+dot:], lo, hi)
		if f == notFound {
			break
		}

		u := nodes[f] >> (nodesBitsTextOffset + nodesBitsTextLength)
		icann = u&(1<<nodesBitsICANN-1) != 0
		u >>= nodesBitsICANN
		u = children[u&(1<<nodesBitsChildren-1)]
		lo = u & (1<<childrenBitsLo - 1)
		u >>= childrenBitsLo
		hi = u & (1<<childrenBitsHi - 1)
		u >>= childrenBitsHi
		switch u & (1<<childrenBitsNodeType - 1) {
		case nodeTypeNormal:
			suffix = 1 + dot
		case nodeTypeException:
			suffix = 1 + len(s)
			break loop
		}
		u >>= childrenBitsNodeType
		wildcard = u&(1<<childrenBitsWildcard-1) != 0

		if dot == -1 {
			break
		}
		s = s[:dot]
	}
	if suffix == len(domain) {
		// If no rules match, the prevailing rule is "*".
		return domain[1+strings.LastIndex(domain, "."):], icann
	}
	return domain[suffix:], icann
}

const notFound uint32 = 1<<32 - 1

// find returns the index of the node in the range [lo, hi) whose label equals
// label, or notFound if there is no such node. The range is assumed to be in
// strictly increasing node label order.
func find(label string, lo, hi uint32) uint32 {
	for lo < hi {
		mid := lo + (hi-lo)/2
		s := nodeLabel(mid)
		if s < label {
			lo = mid + 1
		} else if s == label {
			return mid
		} else {
			hi = mid
		}
	}
	return notFound
}

// nodeLabel returns the label for the i'th node.
func nodeLabel(i uint32) string {
	x := nodes[i]
	length := x & (1<<nodesBitsTextLength - 1)
	x >>= nodesBitsTextLength
	offset := x & (1<<nodesBitsTextOffset - 1)
	return text[offset : offset+length]
}

// EffectiveTLDPlusOne returns the effective top level domain plus one more
// label. For example, the eTLD+1 for "foo.bar.golang.org" is "golang.org".
func EffectiveTLDPlusOne(domain string) (string, error) {
	suffix, _ := PublicSuffix(domain)
	if len(domain) <= len(suffix) {
		return "", fmt.Errorf("publicsuffix: cannot derive eTLD+1 for domain %q", domain)
	}
	i := len(domain) - len(suffix) - 1
	if domain[i] != '.' {
		return "", fmt.Errorf("publicsuffix: invalid public suffix %q for domain %q", suffix, domain)
	}
	return domain[1+strings.LastIndex(domain[:i], "."):], nil
}
<commit_msg>go.net/publicsuffix: rename exp/cookiejar as net/http/cookiejar.<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package publicsuffix provides a public suffix list based on data from
// http://publicsuffix.org/. A public suffix is one under which Internet users
// can directly register names.
package publicsuffix

// TODO: specify case sensitivity and leading/trailing dot behavior for
// func PublicSuffix and func EffectiveTLDPlusOne.

import (
	"fmt"
	"net/http/cookiejar"
	"strings"
)

// List implements the cookiejar.PublicSuffixList interface by calling the
// PublicSuffix function.
var List cookiejar.PublicSuffixList = list{}

type list struct{}

func (list) PublicSuffix(domain string) string {
	ps, _ := PublicSuffix(domain)
	return ps
}

func (list) String() string {
	return version
}

// PublicSuffix returns the public suffix of the domain using a copy of the
// publicsuffix.org database compiled into the library.
//
// icann is whether the public suffix is managed by the Internet Corporation
// for Assigned Names and Numbers. If not, the public suffix is privately
// managed. For example, foo.org and foo.co.uk are ICANN domains,
// foo.dyndns.org and foo.blogspot.co.uk are private domains.
//
// Use cases for distinguishing ICANN domains like foo.com from private
// domains like foo.appspot.com can be found at
// https://wiki.mozilla.org/Public_Suffix_List/Use_Cases
func PublicSuffix(domain string) (publicSuffix string, icann bool) {
	lo, hi := uint32(0), uint32(numTLD)
	s, suffix, wildcard := domain, len(domain), false
loop:
	for {
		dot := strings.LastIndex(s, ".")
		if wildcard {
			suffix = 1 + dot
		}
		if lo == hi {
			break
		}
		f := find(s[1+dot:], lo, hi)
		if f == notFound {
			break
		}

		u := nodes[f] >> (nodesBitsTextOffset + nodesBitsTextLength)
		icann = u&(1<<nodesBitsICANN-1) != 0
		u >>= nodesBitsICANN
		u = children[u&(1<<nodesBitsChildren-1)]
		lo = u & (1<<childrenBitsLo - 1)
		u >>= childrenBitsLo
		hi = u & (1<<childrenBitsHi - 1)
		u >>= childrenBitsHi
		switch u & (1<<childrenBitsNodeType - 1) {
		case nodeTypeNormal:
			suffix = 1 + dot
		case nodeTypeException:
			suffix = 1 + len(s)
			break loop
		}
		u >>= childrenBitsNodeType
		wildcard = u&(1<<childrenBitsWildcard-1) != 0

		if dot == -1 {
			break
		}
		s = s[:dot]
	}
	if suffix == len(domain) {
		// If no rules match, the prevailing rule is "*".
		return domain[1+strings.LastIndex(domain, "."):], icann
	}
	return domain[suffix:], icann
}

const notFound uint32 = 1<<32 - 1

// find returns the index of the node in the range [lo, hi) whose label equals
// label, or notFound if there is no such node. The range is assumed to be in
// strictly increasing node label order.
func find(label string, lo, hi uint32) uint32 {
	for lo < hi {
		mid := lo + (hi-lo)/2
		s := nodeLabel(mid)
		if s < label {
			lo = mid + 1
		} else if s == label {
			return mid
		} else {
			hi = mid
		}
	}
	return notFound
}

// nodeLabel returns the label for the i'th node.
func nodeLabel(i uint32) string {
	x := nodes[i]
	length := x & (1<<nodesBitsTextLength - 1)
	x >>= nodesBitsTextLength
	offset := x & (1<<nodesBitsTextOffset - 1)
	return text[offset : offset+length]
}

// EffectiveTLDPlusOne returns the effective top level domain plus one more
// label. For example, the eTLD+1 for "foo.bar.golang.org" is "golang.org".
func EffectiveTLDPlusOne(domain string) (string, error) {
	suffix, _ := PublicSuffix(domain)
	if len(domain) <= len(suffix) {
		return "", fmt.Errorf("publicsuffix: cannot derive eTLD+1 for domain %q", domain)
	}
	i := len(domain) - len(suffix) - 1
	if domain[i] != '.' {
		return "", fmt.Errorf("publicsuffix: invalid public suffix %q for domain %q", suffix, domain)
	}
	return domain[1+strings.LastIndex(domain[:i], "."):], nil
}
<|endoftext|>


<commit_before>package vsphere

import (
	"fmt"

	yaml "gopkg.in/yaml.v2"

	"github.com/cloudfoundry/bosh-bootloader/storage"
	"github.com/cloudfoundry/bosh-bootloader/terraform"
)

type OpsGenerator struct {
	terraformManager terraformManager
}

type terraformManager interface {
	GetOutputs() (terraform.Outputs, error)
}

func NewOpsGenerator(terraformManager terraformManager) OpsGenerator {
	return OpsGenerator{
		terraformManager: terraformManager,
	}
}

func (o OpsGenerator) Generate(state storage.State) (string, error) {
	return `---
- type: replace
  path: /azs
  value:
  - name: z1
    cloud_properties:
      datacenters:
      - clusters: [((vcenter_cluster)): {}]
  - name: z2
    cloud_properties:
      datacenters:
      - clusters: [((vcenter_cluster)): {}]
  - name: z3
    cloud_properties:
      datacenters:
      - clusters: [((vcenter_cluster)): {}]

- type: replace
  path: /compilation
  value:
    workers: 5
    reuse_compilation_vms: true
    az: z1
    vm_type: default
    network: default

- type: replace
  path: /disk_types
  value:
  - name: default
    disk_size: 3000
  - name: large
    disk_size: 50_000

- type: replace
  path: /networks
  value:
  - name: default
    type: manual
    subnets:
    - range: ((internal_cidr))
      gateway: ((internal_gw))
      azs: [z1, z2, z3]
      dns: [8.8.8.8]
      reserved: []
      cloud_properties:
        name: ((network_name))

- type: replace
  path: /vm_types
  value:
  - name: default
    cloud_properties:
      cpu: 2
      ram: 1024
      disk: 3240
  - name: large
    cloud_properties:
      cpu: 2
      ram: 4096
      disk: 30_240

- type: remove
  path: /vm_extensions
`, nil
}

type VarsYAML struct {
	InternalCIDR   string `yaml:"internal_cidr,omitempty"`
	InternalGW     string `yaml:"internal_gw,omitempty"`
	NetworkName    string `yaml:"network_name,omitempty"`
	VCenterCluster string `yaml:"vcenter_cluster,omitempty"`
}

func (o OpsGenerator) GenerateVars(state storage.State) (string, error) {
	terraformOutputs, err := o.terraformManager.GetOutputs()
	if err != nil {
		return "", fmt.Errorf("Get terraform outputs: %s", err)
	}
	varsYAML := VarsYAML{
		InternalCIDR:   terraformOutputs.GetString("internal_cidr"),
		InternalGW:     terraformOutputs.GetString("internal_gw"),
		NetworkName:    terraformOutputs.GetString("network_name"),
		VCenterCluster: terraformOutputs.GetString("vcenter_cluster"),
	}
	varsBytes, err := yaml.Marshal(varsYAML)
	if err != nil {
		panic(err) // not tested; cannot occur
	}
	return string(varsBytes), nil
}
<commit_msg>Increase vm types ram/disk for vsphere.<commit_after>package vsphere

import (
	"fmt"

	yaml "gopkg.in/yaml.v2"

	"github.com/cloudfoundry/bosh-bootloader/storage"
	"github.com/cloudfoundry/bosh-bootloader/terraform"
)

type OpsGenerator struct {
	terraformManager terraformManager
}

type terraformManager interface {
	GetOutputs() (terraform.Outputs, error)
}

func NewOpsGenerator(terraformManager terraformManager) OpsGenerator {
	return OpsGenerator{
		terraformManager: terraformManager,
	}
}

func (o OpsGenerator) Generate(state storage.State) (string, error) {
	return `---
- type: replace
  path: /azs
  value:
  - name: z1
    cloud_properties:
      datacenters:
      - clusters: [((vcenter_cluster)): {}]
  - name: z2
    cloud_properties:
      datacenters:
      - clusters: [((vcenter_cluster)): {}]
  - name: z3
    cloud_properties:
      datacenters:
      - clusters: [((vcenter_cluster)): {}]

- type: replace
  path: /compilation
  value:
    workers: 5
    reuse_compilation_vms: true
    az: z1
    vm_type: default
    network: default

- type: replace
  path: /disk_types
  value:
  - name: default
    disk_size: 3000
  - name: large
    disk_size: 50_000

- type: replace
  path: /networks
  value:
  - name: default
    type: manual
    subnets:
    - range: ((internal_cidr))
      gateway: ((internal_gw))
      azs: [z1, z2, z3]
      dns: [8.8.8.8]
      reserved: []
      cloud_properties:
        name: ((network_name))

- type: replace
  path: /vm_types
  value:
  - name: default
    cloud_properties:
      cpu: 2
      ram: 8_192
      disk: 30_000
  - name: large
    cloud_properties:
      cpu: 2
      ram: 8_192
      disk: 640_000

- type: remove
  path: /vm_extensions
`, nil
}

type VarsYAML struct {
	InternalCIDR   string `yaml:"internal_cidr,omitempty"`
	InternalGW     string `yaml:"internal_gw,omitempty"`
	NetworkName    string `yaml:"network_name,omitempty"`
	VCenterCluster string `yaml:"vcenter_cluster,omitempty"`
}

func (o OpsGenerator) GenerateVars(state storage.State) (string, error) {
	terraformOutputs, err := o.terraformManager.GetOutputs()
	if err != nil {
		return "", fmt.Errorf("Get terraform outputs: %s", err)
	}
	varsYAML := VarsYAML{
		InternalCIDR:   terraformOutputs.GetString("internal_cidr"),
		InternalGW:     terraformOutputs.GetString("internal_gw"),
		NetworkName:    terraformOutputs.GetString("network_name"),
		VCenterCluster: terraformOutputs.GetString("vcenter_cluster"),
	}
	varsBytes, err := yaml.Marshal(varsYAML)
	if err != nil {
		panic(err) // not tested; cannot occur
	}
	return string(varsBytes), nil
}
<|endoftext|>


<commit_before>package calcium

import (
	"context"
	"fmt"
	"sync"

	enginetypes "github.com/docker/docker/api/types"
	"github.com/projecteru2/core/types"
	log "github.com/sirupsen/logrus"
)

// ReplaceContainer replace containers with same resource
func (c *Calcium) ReplaceContainer(ctx context.Context, opts *types.DeployOptions) (chan *types.ReplaceContainerMessage, error) {
	oldContainers, err := c.ListContainers(ctx, opts.Name, opts.Entrypoint.Name, opts.Nodename)
	if err != nil {
		return nil, err
	}
	ch := make(chan *types.ReplaceContainerMessage)

	go func() {
		defer close(ch)
		lock, err := c.Lock(ctx, opts.Podname, c.config.LockTimeout)
		if err != nil {
			log.Errorf("[ReplaceContainer] Lock pod failed %v", err)
			return
		}
		defer lock.Unlock(ctx)

		// 并发控制
		step := opts.Count
		wg := sync.WaitGroup{}

		for index, oldContainer := range oldContainers {
			log.Debug("[ReplaceContainer] Replace old container : %s", oldContainer.ID)
			wg.Add(1)
			go func(deployOpts types.DeployOptions, oldContainer *types.Container, index int) {
				defer wg.Done()
				// 使用复制之后的配置
				// 停老的，起新的
				deployOpts.Memory = oldContainer.Memory
				deployOpts.CPUQuota = oldContainer.Quota

				createMessage, err := c.replaceAndRemove(ctx, &deployOpts, oldContainer, index)
				ch <- &types.ReplaceContainerMessage{
					CreateContainerMessage: createMessage,
					OldContainerID:         oldContainer.ID,
					Error:                  err,
				}
				if err != nil {
					log.Errorf("[ReplaceContainer] Replace and remove failed %v, old container restarted", err)
				}
				// 传 opts 的值，产生一次复制
			}(*opts, oldContainer, index)
			if index != 0 && step%index == 0 {
				wg.Wait()
			}
		}
		wg.Wait()
	}()

	return ch, nil
}

func (c *Calcium) replaceAndRemove(
	ctx context.Context,
	opts *types.DeployOptions,
	oldContainer *types.Container,
	index int) (*types.CreateContainerMessage, error) {
	var err error

	// 锁住，防止删除
	lock, err := c.Lock(ctx, fmt.Sprintf("rmcontainer_%s", oldContainer.ID), int(c.config.GlobalTimeout.Seconds()))
	if err != nil {
		return nil, err
	}
	defer lock.Unlock(ctx)

	// 确保得到锁的时候容器没被干掉
	_, err = oldContainer.Inspect(ctx)
	if err != nil {
		return nil, err
	}

	// 停掉老的
	if err = c.stopOneContainer(ctx, oldContainer); err != nil {
		return nil, err
	}

	// 创建新容器，复用资源，如果失败会被自动回收，但是这里要重启老容器
	// 实际上会从 node 的抽象中减掉这部分的资源，因此资源计数器可能不准确，如果成功了，remove 老容器即可恢复
	createMessage := c.createAndStartContainer(ctx, index, oldContainer.Node, opts, oldContainer.CPU)
	if createMessage.Error != nil {
		// 重启容器, 并不关心是否启动成功
		if err = oldContainer.Engine.ContainerStart(ctx, oldContainer.ID, enginetypes.ContainerStartOptions{}); err != nil {
			log.Errorf("[replaceAndRemove] Old container %s restart failed %v", oldContainer.ID, err)
		}
		return nil, createMessage.Error
	}

	// 这里横竖会保证资源回收, 因此即便 remove 失败我们只需要考虑新容器占据了准确的资源配额即可
	if err = c.removeAndCleanOneContainer(ctx, oldContainer); err != nil {
		return nil, err
	}

	return createMessage, nil
}
<commit_msg>missing resource update<commit_after>package calcium

import (
	"context"
	"fmt"
	"sync"

	enginetypes "github.com/docker/docker/api/types"
	"github.com/projecteru2/core/types"
	log "github.com/sirupsen/logrus"
)

// ReplaceContainer replace containers with same resource
func (c *Calcium) ReplaceContainer(ctx context.Context, opts *types.DeployOptions) (chan *types.ReplaceContainerMessage, error) {
	oldContainers, err := c.ListContainers(ctx, opts.Name, opts.Entrypoint.Name, opts.Nodename)
	if err != nil {
		return nil, err
	}
	ch := make(chan *types.ReplaceContainerMessage)

	go func() {
		defer close(ch)
		lock, err := c.Lock(ctx, opts.Podname, c.config.LockTimeout)
		if err != nil {
			log.Errorf("[ReplaceContainer] Lock pod failed %v", err)
			return
		}
		defer lock.Unlock(ctx)

		// 并发控制
		step := opts.Count
		wg := sync.WaitGroup{}

		for index, oldContainer := range oldContainers {
			log.Debug("[ReplaceContainer] Replace old container : %s", oldContainer.ID)
			wg.Add(1)
			go func(deployOpts types.DeployOptions, oldContainer *types.Container, index int) {
				defer wg.Done()
				// 使用复制之后的配置
				// 停老的，起新的
				deployOpts.Memory = oldContainer.Memory
				deployOpts.CPUQuota = oldContainer.Quota

				createMessage, err := c.replaceAndRemove(ctx, &deployOpts, oldContainer, index)
				ch <- &types.ReplaceContainerMessage{
					CreateContainerMessage: createMessage,
					OldContainerID:         oldContainer.ID,
					Error:                  err,
				}
				if err != nil {
					log.Errorf("[ReplaceContainer] Replace and remove failed %v, old container restarted", err)
				}
				// 传 opts 的值，产生一次复制
			}(*opts, oldContainer, index)
			if index != 0 && step%index == 0 {
				wg.Wait()
			}
		}
		wg.Wait()
	}()

	return ch, nil
}

func (c *Calcium) replaceAndRemove(
	ctx context.Context,
	opts *types.DeployOptions,
	oldContainer *types.Container,
	index int) (*types.CreateContainerMessage, error) {
	var err error

	// 锁住，防止删除
	lock, err := c.Lock(ctx, fmt.Sprintf("rmcontainer_%s", oldContainer.ID), int(c.config.GlobalTimeout.Seconds()))
	if err != nil {
		return nil, err
	}
	defer lock.Unlock(ctx)

	// 确保得到锁的时候容器没被干掉
	_, err = oldContainer.Inspect(ctx)
	if err != nil {
		return nil, err
	}

	// 预先扣除资源，若成功，老资源会回收，若失败，新资源也会被回收
	err = c.store.UpdateNodeResource(ctx, oldContainer.Podname, oldContainer.Nodename, oldContainer.CPU, oldContainer.Memory, "-")
	if err != nil {
		return nil, err
	}

	// 停掉老的
	if err = c.stopOneContainer(ctx, oldContainer); err != nil {
		return nil, err
	}

	// 创建新容器，复用资源，如果失败会被自动回收，但是这里要重启老容器
	// 实际上会从 node 的抽象中减掉这部分的资源，因此资源计数器可能不准确，如果成功了，remove 老容器即可恢复
	createMessage := c.createAndStartContainer(ctx, index, oldContainer.Node, opts, oldContainer.CPU)
	if createMessage.Error != nil {
		// 重启容器, 并不关心是否启动成功
		if err = oldContainer.Engine.ContainerStart(ctx, oldContainer.ID, enginetypes.ContainerStartOptions{}); err != nil {
			log.Errorf("[replaceAndRemove] Old container %s restart failed %v", oldContainer.ID, err)
		}
		return nil, createMessage.Error
	}

	// 这里横竖会保证资源回收, 因此即便 remove 失败我们只需要考虑新容器占据了准确的资源配额即可
	if err = c.removeAndCleanOneContainer(ctx, oldContainer); err != nil {
		return nil, err
	}

	return createMessage, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"compress/gzip"
	"crypto/md5"
	"encoding/hex"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/ugorji/go/codec"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"regexp"
	"time"
)

type DummyServerParams struct {
	WriteTimeout time.Duration
	ReadTimeout  time.Duration
	ListenOn     string
}

var progName = os.Args[0]

func MustParseDuration(s string) time.Duration {
	d, err := time.ParseDuration(s)
	if err != nil {
		panic(err)
	}
	return d
}

func Error(fmtStr string, args ...interface{}) {
	fmt.Fprint(os.Stderr, progName, ": ")
	fmt.Fprintf(os.Stderr, fmtStr, args...)
	fmt.Fprint(os.Stderr, "\n")
}

func ParseArgs() *DummyServerParams {
	readTimeout := (time.Duration)(0)
	writeTimeout := (time.Duration)(0)
	listenOn := ""

	flagSet := flag.NewFlagSet(progName, flag.ExitOnError)

	flagSet.DurationVar(&readTimeout, "read-timeout", MustParseDuration("10s"), "read timeout on wire")
	flagSet.DurationVar(&writeTimeout, "write-timeout", MustParseDuration("10s"), "write timeout on wire")
	flagSet.StringVar(&listenOn, "listen-on", "127.0.0.1:80", "interface address and port on which the dummy server listens")
	flagSet.Parse(os.Args[1:])

	return &DummyServerParams{
		ReadTimeout:  readTimeout,
		WriteTimeout: writeTimeout,
		ListenOn:     listenOn,
	}
}

func internalServerError(resp http.ResponseWriter) {
	resp.WriteHeader(500)
	resp.Write([]byte(`{"errorMessage":"Internal Server Error"}`))
}

func handle(resp http.ResponseWriter, req *http.Request, matchparams map[string]string) {
	resp.Header().Set("Content-Type", "application/json; charset=UTF-8")
	h := md5.New()
	format := matchparams["format"]
	body, err := ioutil.ReadAll(req.Body)
	if err != nil {
		internalServerError(resp)
		return
	}
	rdr := (io.Reader)(bytes.NewReader(body))
	if format == "msgpack.gz" {
		rdr, err = gzip.NewReader(rdr)
		if err != nil {
			internalServerError(resp)
			return
		}
	}
	_codec := &codec.MsgpackHandle{}
	decoder := codec.NewDecoder(rdr, _codec)
	numRecords := 0
	for {
		v := map[string]interface{}{}
		err := decoder.Decode(&v)
		if err != nil {
			if err == io.EOF {
				break
			} else {
				Error("%s", err.Error())
				break
			}
		}
		numRecords += 1
	}
	fmt.Printf("%d records received\n", numRecords)
	io.Copy(h, bytes.NewReader(body))
	md5sum := make([]byte, 0, h.Size())
	md5sum = h.Sum(md5sum)
	uniqueId, _ := matchparams["uniqueId"]
	respData := map[string]interface{}{
		"unique_id":    uniqueId,
		"database":     matchparams["database"],
		"table":        matchparams["table"],
		"md5_hex":      hex.EncodeToString(md5sum),
		"elapsed_time": 0.,
	}
	payload, err := json.Marshal(respData)
	if err != nil {
		internalServerError(resp)
		return
	}
	resp.WriteHeader(200)
	resp.Write(payload)
}

type RegexpServeMuxHandler func(http.ResponseWriter, *http.Request, map[string]string)

type regexpServeMuxEntry struct {
	pattern *regexp.Regexp
	handler RegexpServeMuxHandler
}

type RegexpServeMux struct {
	entries []*regexpServeMuxEntry
}

func (mux *RegexpServeMux) Handle(pattern string, handler RegexpServeMuxHandler) error {
	rex, err := regexp.Compile(pattern)
	if err != nil {
		return err
	}
	mux.entries = append(mux.entries, &regexpServeMuxEntry{
		pattern: rex,
		handler: handler,
	})
	return nil
}

func (mux *RegexpServeMux) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
	submatches := [][]byte{}
	candidate := (*regexpServeMuxEntry)(nil)
	for _, entry := range mux.entries {
		submatches = entry.pattern.FindSubmatch([]byte(req.URL.Path))
		if submatches != nil {
			candidate = entry
			break
		}
	}
	if candidate == nil {
		resp.WriteHeader(400)
		return
	}
	matchparams := map[string]string{}
	for i, name := range candidate.pattern.SubexpNames() {
		if submatches[i] != nil {
			// XXX: assuming URL is encoded in UTF-8
			matchparams[name] = string(submatches[i])
		}
	}
	candidate.handler(resp, req, matchparams)
}

func newRegexpServeMux() *RegexpServeMux {
	return &RegexpServeMux{
		entries: make([]*regexpServeMuxEntry, 0, 16),
	}
}

func buildMux() *RegexpServeMux {
	mux := newRegexpServeMux()
	err := mux.Handle("^/v3/table/import_with_id/(?P<database>[^/]+)/(?P<table>[^/]+)/(?P<uniqueId>[^/]+)/(?P<format>[^/]+)$", handle)
	if err != nil {
		panic(err.Error())
	}
	err = mux.Handle("^/v3/table/import/(?P<database>[^/]+)/(?P<table>[^/]+)/(?P<format>[^/]+)$", handle)
	if err != nil {
		panic(err.Error())
	}
	return mux
}

var mux = buildMux()

func main() {
	params := ParseArgs()
	server := http.Server{
		Addr:         params.ListenOn,
		ReadTimeout:  params.ReadTimeout,
		WriteTimeout: params.WriteTimeout,
		Handler:      mux,
	}
	listener, err := net.Listen("tcp", params.ListenOn)
	if err != nil {
		Error("%s", err.Error())
		os.Exit(1)
	}
	fmt.Printf("Dummy server listening on %s ...\n", params.ListenOn)
	fmt.Print("Hit CTRL-C to stop\n")
	server.Serve(listener)
}
<commit_msg>Add -read-throttle option to throttle inbound bandwidth.<commit_after>package main

import (
	"bytes"
	"compress/gzip"
	"crypto/md5"
	"encoding/hex"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/ugorji/go/codec"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"regexp"
	"time"
)

type DummyServerParams struct {
	WriteTimeout time.Duration
	ReadTimeout  time.Duration
	ListenOn     string
	ReadThrottle int
}

var progName = os.Args[0]

func MustParseDuration(s string) time.Duration {
	d, err := time.ParseDuration(s)
	if err != nil {
		panic(err)
	}
	return d
}

func Error(fmtStr string, args ...interface{}) {
	fmt.Fprint(os.Stderr, progName, ": ")
	fmt.Fprintf(os.Stderr, fmtStr, args...)
	fmt.Fprint(os.Stderr, "\n")
}

func ParseArgs() *DummyServerParams {
	readTimeout := (time.Duration)(0)
	writeTimeout := (time.Duration)(0)
	readThrottle := 0
	listenOn := ""

	flagSet := flag.NewFlagSet(progName, flag.ExitOnError)

	flagSet.DurationVar(&readTimeout, "read-timeout", MustParseDuration("10s"), "read timeout on wire")
	flagSet.DurationVar(&writeTimeout, "write-timeout", MustParseDuration("10s"), "write timeout on wire")
	flagSet.IntVar(&readThrottle, "read-throttle", 0, "read slottling")
	flagSet.StringVar(&listenOn, "listen-on", "127.0.0.1:80", "interface address and port on which the dummy server listens")
	flagSet.Parse(os.Args[1:])

	return &DummyServerParams{
		ReadTimeout:  readTimeout,
		WriteTimeout: writeTimeout,
		ListenOn:     listenOn,
		ReadThrottle: readThrottle,
	}
}

func internalServerError(resp http.ResponseWriter) {
	resp.WriteHeader(500)
	resp.Write([]byte(`{"errorMessage":"Internal Server Error"}`))
}

func ReadThrottled(rdr io.Reader, l int, bps int) ([]byte, error) {
	_bps := int64(bps)
	b := make([]byte, 4096)
	t := time.Now()
	o := 0
	for o < l {
		if o + 4096 >= len(b) {
			_b := make([]byte, len(b) + len(b) / 2)
			copy(_b, b)
			b = _b
		}
		_t := time.Now()
		elapsed := _t.Sub(t)
		if elapsed > 0 {
			_o := int64(o) * int64(1000000000)
			cbps := _o / int64(elapsed)
			if cbps > _bps {
				time.Sleep(time.Duration(_o / _bps - int64(elapsed)))
			}
		}
		x := o + 4096
		if x >= len(b) {
			x = len(b)
		}
		n, err := rdr.Read(b[o:x])
		o += n
		if err != nil {
			if err != io.EOF {
				return nil, err
			} else {
				break
			}
		}
	}
	b = b[0:o]
	return b, nil
}

func handleReq(params *DummyServerParams, resp http.ResponseWriter, req *http.Request, matchparams map[string]string) {
	resp.Header().Set("Content-Type", "application/json; charset=UTF-8")
	h := md5.New()
	format := matchparams["format"]
	var body []byte
	var err error
	if params.ReadThrottle > 0 {
		body, err = ReadThrottled(req.Body, int(req.ContentLength), params.ReadThrottle)
	} else {
		body, err = ioutil.ReadAll(req.Body)
	}
	if err != nil {
		internalServerError(resp)
		return
	}
	rdr := (io.Reader)(bytes.NewReader(body))
	if format == "msgpack.gz" {
		rdr, err = gzip.NewReader(rdr)
		if err != nil {
			internalServerError(resp)
			return
		}
	}
	_codec := &codec.MsgpackHandle{}
	decoder := codec.NewDecoder(rdr, _codec)
	numRecords := 0
	for {
		v := map[string]interface{}{}
		err := decoder.Decode(&v)
		if err != nil {
			if err == io.EOF {
				break
			} else {
				Error("%s", err.Error())
				break
			}
		}
		numRecords += 1
	}
	fmt.Printf("%d records received\n", numRecords)
	io.Copy(h, bytes.NewReader(body))
	md5sum := make([]byte, 0, h.Size())
	md5sum = h.Sum(md5sum)
	uniqueId, _ := matchparams["uniqueId"]
	respData := map[string]interface{}{
		"unique_id":    uniqueId,
		"database":     matchparams["database"],
		"table":        matchparams["table"],
		"md5_hex":      hex.EncodeToString(md5sum),
		"elapsed_time": 0.,
	}
	payload, err := json.Marshal(respData)
	if err != nil {
		internalServerError(resp)
		return
	}
	resp.WriteHeader(200)
	resp.Write(payload)
}

type RegexpServeMuxHandler func(http.ResponseWriter, *http.Request, map[string]string)

type regexpServeMuxEntry struct {
	pattern *regexp.Regexp
	handler RegexpServeMuxHandler
}

type RegexpServeMux struct {
	entries []*regexpServeMuxEntry
}

func (mux *RegexpServeMux) Handle(pattern string, handler RegexpServeMuxHandler) error {
	rex, err := regexp.Compile(pattern)
	if err != nil {
		return err
	}
	mux.entries = append(mux.entries, &regexpServeMuxEntry{
		pattern: rex,
		handler: handler,
	})
	return nil
}

func (mux *RegexpServeMux) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
	submatches := [][]byte{}
	candidate := (*regexpServeMuxEntry)(nil)
	for _, entry := range mux.entries {
		submatches = entry.pattern.FindSubmatch([]byte(req.URL.Path))
		if submatches != nil {
			candidate = entry
			break
		}
	}
	if candidate == nil {
		resp.WriteHeader(400)
		return
	}
	matchparams := map[string]string{}
	for i, name := range candidate.pattern.SubexpNames() {
		if submatches[i] != nil {
			// XXX: assuming URL is encoded in UTF-8
			matchparams[name] = string(submatches[i])
		}
	}
	candidate.handler(resp, req, matchparams)
}

func newRegexpServeMux() *RegexpServeMux {
	return &RegexpServeMux{
		entries: make([]*regexpServeMuxEntry, 0, 16),
	}
}

func buildMux(handle RegexpServeMuxHandler) *RegexpServeMux {
	mux := newRegexpServeMux()
	err := mux.Handle("^/v3/table/import_with_id/(?P<database>[^/]+)/(?P<table>[^/]+)/(?P<uniqueId>[^/]+)/(?P<format>[^/]+)$", handle)
	if err != nil {
		panic(err.Error())
	}
	err = mux.Handle("^/v3/table/import/(?P<database>[^/]+)/(?P<table>[^/]+)/(?P<format>[^/]+)$", handle)
	if err != nil {
		panic(err.Error())
	}
	return mux
}

func main() {
	params := ParseArgs()
	var mux = buildMux(func (resp http.ResponseWriter, req *http.Request, matchparams map[string]string) {
		handleReq(params, resp, req, matchparams)
	})
	server := http.Server{
		Addr:         params.ListenOn,
		ReadTimeout:  params.ReadTimeout,
		WriteTimeout: params.WriteTimeout,
		Handler:      mux,
	}
	listener, err := net.Listen("tcp", params.ListenOn)
	if err != nil {
		Error("%s", err.Error())
		os.Exit(1)
	}
	fmt.Printf("Dummy server listening on %s ...\n", params.ListenOn)
	fmt.Print("Hit CTRL-C to stop\n")
	server.Serve(listener)
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The cert-manager Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package secret

import (
	"context"
	"crypto/x509"
	"errors"
	"fmt"
	"net/url"
	"strings"
	"time"

	"github.com/spf13/cobra"

	cmmeta "github.com/jetstack/cert-manager/pkg/apis/meta/v1"
	"github.com/jetstack/cert-manager/pkg/util/pki"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/cli-runtime/pkg/genericclioptions"
	"k8s.io/client-go/kubernetes"
	restclient "k8s.io/client-go/rest"
	cmdutil "k8s.io/kubectl/pkg/cmd/util"
	"k8s.io/kubectl/pkg/util/i18n"
	"k8s.io/kubectl/pkg/util/templates"
	k8sclock "k8s.io/utils/clock"
)

var clock k8sclock.Clock = k8sclock.RealClock{}

const validForTemplate = `Valid for:
	DNS Names: %s
	URIs: %s
	IP Addresses: %s
	Email Addresses: %s
	Usages: %s`

const validityPeriodTemplate = `Validity period:
	Not Before: %s
	Not After: %s`

const issuedByTemplate = `Issued By:
	Common Name		%s
	Organization		%s
	OrganizationalUnit	%s
	Country: 		%s`

const issuedForTemplate = `Issued For:
	Common Name		%s
	Organization		%s
	OrganizationalUnit	%s
	Country: 		%s`

const certificateTemplate = `Certificate:
	Signing Algorithm:	%s
	Public Key Algorithm: 	%s
	Serial Number:	%s
	Fingerprints: 	%s
	Is a CA certificate: %v
	CRL:	%s
	OCSP:	%s`

const debuggingTemplate = `Debugging:
	Trusted by this computer:	%s
	CRL Status:	%s
	OCSP Status:	%s`

var (
	long = templates.LongDesc(i18n.T(`
Get details about a kubernetes.io/tls typed secret`))

	example = templates.Examples(i18n.T(`
# Query information about a secret with name 'my-crt' in namespace 'my-namespace'
kubectl cert-manager inspect secret my-crt --namespace my-namespace
`))
)

// Options is a struct to support status certificate command
type Options struct {
	RESTConfig *restclient.Config
	// The Namespace that the Certificate to be queried about resides in.
	// This flag registration is handled by cmdutil.Factory
	Namespace string

	clientSet *kubernetes.Clientset

	genericclioptions.IOStreams
}

// NewOptions returns initialized Options
func NewOptions(ioStreams genericclioptions.IOStreams) *Options {
	return &Options{
		IOStreams: ioStreams,
	}
}

// NewCmdInspectSecret returns a cobra command for status certificate
func NewCmdInspectSecret(ioStreams genericclioptions.IOStreams, factory cmdutil.Factory) *cobra.Command {
	o := NewOptions(ioStreams)
	cmd := &cobra.Command{
		Use:     "secret",
		Short:   "Get details about a kubernetes.io/tls typed secret",
		Long:    long,
		Example: example,
		Run: func(cmd *cobra.Command, args []string) {
			cmdutil.CheckErr(o.Validate(args))
			cmdutil.CheckErr(o.Complete(factory))
			cmdutil.CheckErr(o.Run(args))
		},
	}
	return cmd
}

// Validate validates the provided options
func (o *Options) Validate(args []string) error {
	if len(args) < 1 {
		return errors.New("the name of the Secret has to be provided as argument")
	}
	if len(args) > 1 {
		return errors.New("only one argument can be passed in: the name of the Secret")
	}
	return nil
}

// Complete takes the factory and infers any remaining options.
func (o *Options) Complete(f cmdutil.Factory) error {
	var err error

	o.Namespace, _, err = f.ToRawKubeConfigLoader().Namespace()
	if err != nil {
		return err
	}

	o.RESTConfig, err = f.ToRESTConfig()
	if err != nil {
		return err
	}

	o.clientSet, err = kubernetes.NewForConfig(o.RESTConfig)
	if err != nil {
		return err
	}

	return nil
}

// Run executes status certificate command
func (o *Options) Run(args []string) error {
	ctx := context.TODO()

	secret, err := o.clientSet.CoreV1().Secrets(o.Namespace).Get(ctx, args[0], metav1.GetOptions{})
	if err != nil {
		return fmt.Errorf("error when finding Secret %q: %w\n", args[0], err)
	}

	certData := secret.Data[corev1.TLSCertKey]
	certs, err := splitPEMs(certData)
	if err != nil {
		return err
	}
	if len(certs) < 1 {
		return errors.New("no PEM data found in secret")
	}

	intermediates := [][]byte(nil)
	if len(certs) > 1 {
		intermediates = certs[1:]
	}

	// we only want to inspect the leaf certificate
	x509Cert, err := pki.DecodeX509CertificateBytes(certs[0])
	if err != nil {
		return fmt.Errorf("error when parsing 'tls.crt': %w", err)
	}

	out := []string{
		describeValidFor(x509Cert),
		describeValidityPeriod(x509Cert),
		describeIssuedBy(x509Cert),
		describeIssuedFor(x509Cert),
		describeCertificate(x509Cert),
		describeDebugging(x509Cert, intermediates, secret.Data[cmmeta.TLSCAKey]),
	}

	fmt.Println(strings.Join(out, "\n\n"))

	return nil
}

func describeValidFor(cert *x509.Certificate) string {
	return fmt.Sprintf(validForTemplate,
		printSlice(cert.DNSNames),
		printSlice(pki.URLsToString(cert.URIs)),
		printSlice(pki.IPAddressesToString(cert.IPAddresses)),
		printSlice(cert.EmailAddresses),
		printKeyUsage(pki.BuildCertManagerKeyUsages(cert.KeyUsage, cert.ExtKeyUsage)),
	)
}

func describeValidityPeriod(cert *x509.Certificate) string {
	return fmt.Sprintf(validityPeriodTemplate,
		cert.NotBefore.Format(time.RFC1123),
		cert.NotAfter.Format(time.RFC1123),
	)
}

func describeIssuedBy(cert *x509.Certificate) string {
	return fmt.Sprintf(issuedByTemplate,
		printOrNone(cert.Issuer.CommonName),
		printSliceOrOne(cert.Issuer.Organization),
		printSliceOrOne(cert.Issuer.OrganizationalUnit),
		printSliceOrOne(cert.Issuer.Country),
	)
}

func describeIssuedFor(cert *x509.Certificate) string {
	return fmt.Sprintf(issuedForTemplate,
		printOrNone(cert.Subject.CommonName),
		printSliceOrOne(cert.Subject.Organization),
		printSliceOrOne(cert.Subject.OrganizationalUnit),
		printSliceOrOne(cert.Subject.Country),
	)
}

func describeCertificate(cert *x509.Certificate) string {
	return fmt.Sprintf(certificateTemplate,
		cert.SignatureAlgorithm.String(),
		cert.PublicKeyAlgorithm.String(),
		cert.SerialNumber.String(),
		fingerprintCert(cert),
		cert.IsCA,
		printSliceOrOne(cert.CRLDistributionPoints),
		printSliceOrOne(cert.OCSPServer),
	)
}

func describeDebugging(cert *x509.Certificate, intermediates [][]byte, ca []byte) string {
	return fmt.Sprintf(debuggingTemplate,
		describeTrusted(cert, intermediates),
		describeCRL(cert),
		describeOCSP(cert, intermediates, ca),
	)
}

func describeCRL(cert *x509.Certificate) string {
	if len(cert.CRLDistributionPoints) < 1 {
		return "No CRL endpoints set"
	}

	hasChecked := false
	for _, crlURL := range cert.CRLDistributionPoints {
		u, err := url.Parse(crlURL)
		if err != nil {
			continue // not a valid URL
		}
		if u.Scheme != "ldap" && u.Scheme != "https" {
			continue
		}

		hasChecked = true
		valid, err := checkCRLValidCert(cert, crlURL)
		if err != nil {
			return fmt.Sprintf("Cannot check CRL: %s", err.Error())
		}
		if !valid {
			return fmt.Sprintf("Revoked by %s", crlURL)
		}
	}

	if !hasChecked {
		return "No CRL endpoints we support found"
	}

	return "Valid"
}

func describeOCSP(cert *x509.Certificate, intermediates [][]byte, ca []byte) string {
	if len(ca) > 1 {
		intermediates = append([][]byte{ca}, intermediates...)
	}
	if len(intermediates) < 1 {
		return "Cannot check OCSP, does not have a CA or intermediate certificate provided"
	}
	issuerCert, err := pki.DecodeX509CertificateBytes(intermediates[len(intermediates)-1])
	if err != nil {
		return fmt.Sprintf("Cannot parse intermediate certificate: %s", err.Error())
	}

	valid, err := checkOCSPValidCert(cert, issuerCert)
	if err != nil {
		return fmt.Sprintf("Cannot check OCSP: %s", err.Error())
	}

	if !valid {
		return "Marked as revoked"
	}

	return "valid"
}

func describeTrusted(cert *x509.Certificate, intermediates [][]byte) string {
	systemPool, err := x509.SystemCertPool()
	if err != nil {
		return fmt.Sprintf("Error getting system CA store: %s", err.Error())
	}
	for _, intermediate := range intermediates {
		systemPool.AppendCertsFromPEM(intermediate)
	}
	_, err = cert.Verify(x509.VerifyOptions{
		Roots:       systemPool,
		CurrentTime: clock.Now(),
	})
	if err == nil {
		return "yes"
	}
	return fmt.Sprintf("no: %s", err.Error())
}
<commit_msg>Sort imports<commit_after>/*
Copyright 2020 The cert-manager Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package secret

import (
	"context"
	"crypto/x509"
	"errors"
	"fmt"
	"net/url"
	"strings"
	"time"

	"github.com/spf13/cobra"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/cli-runtime/pkg/genericclioptions"
	"k8s.io/client-go/kubernetes"
	restclient "k8s.io/client-go/rest"
	cmdutil "k8s.io/kubectl/pkg/cmd/util"
	"k8s.io/kubectl/pkg/util/i18n"
	"k8s.io/kubectl/pkg/util/templates"
	k8sclock "k8s.io/utils/clock"

	cmmeta "github.com/jetstack/cert-manager/pkg/apis/meta/v1"
	"github.com/jetstack/cert-manager/pkg/util/pki"
)

var clock k8sclock.Clock = k8sclock.RealClock{}

const validForTemplate = `Valid for:
	DNS Names: %s
	URIs: %s
	IP Addresses: %s
	Email Addresses: %s
	Usages: %s`

const validityPeriodTemplate = `Validity period:
	Not Before: %s
	Not After: %s`

const issuedByTemplate = `Issued By:
	Common Name		%s
	Organization		%s
	OrganizationalUnit	%s
	Country: 		%s`

const issuedForTemplate = `Issued For:
	Common Name		%s
	Organization		%s
	OrganizationalUnit	%s
	Country: 		%s`

const certificateTemplate = `Certificate:
	Signing Algorithm:	%s
	Public Key Algorithm: 	%s
	Serial Number:	%s
	Fingerprints: 	%s
	Is a CA certificate: %v
	CRL:	%s
	OCSP:	%s`

const debuggingTemplate = `Debugging:
	Trusted by this computer:	%s
	CRL Status:	%s
	OCSP Status:	%s`

var (
	long = templates.LongDesc(i18n.T(`
Get details about a kubernetes.io/tls typed secret`))

	example = templates.Examples(i18n.T(`
# Query information about a secret with name 'my-crt' in namespace 'my-namespace'
kubectl cert-manager inspect secret my-crt --namespace my-namespace
`))
)

// Options is a struct to support status certificate command
type Options struct {
	RESTConfig *restclient.Config
	// The Namespace that the Certificate to be queried about resides in.
	// This flag registration is handled by cmdutil.Factory
	Namespace string

	clientSet *kubernetes.Clientset

	genericclioptions.IOStreams
}

// NewOptions returns initialized Options
func NewOptions(ioStreams genericclioptions.IOStreams) *Options {
	return &Options{
		IOStreams: ioStreams,
	}
}

// NewCmdInspectSecret returns a cobra command for status certificate
func NewCmdInspectSecret(ioStreams genericclioptions.IOStreams, factory cmdutil.Factory) *cobra.Command {
	o := NewOptions(ioStreams)
	cmd := &cobra.Command{
		Use:     "secret",
		Short:   "Get details about a kubernetes.io/tls typed secret",
		Long:    long,
		Example: example,
		Run: func(cmd *cobra.Command, args []string) {
			cmdutil.CheckErr(o.Validate(args))
			cmdutil.CheckErr(o.Complete(factory))
			cmdutil.CheckErr(o.Run(args))
		},
	}
	return cmd
}

// Validate validates the provided options
func (o *Options) Validate(args []string) error {
	if len(args) < 1 {
		return errors.New("the name of the Secret has to be provided as argument")
	}
	if len(args) > 1 {
		return errors.New("only one argument can be passed in: the name of the Secret")
	}
	return nil
}

// Complete takes the factory and infers any remaining options.
func (o *Options) Complete(f cmdutil.Factory) error {
	var err error

	o.Namespace, _, err = f.ToRawKubeConfigLoader().Namespace()
	if err != nil {
		return err
	}

	o.RESTConfig, err = f.ToRESTConfig()
	if err != nil {
		return err
	}

	o.clientSet, err = kubernetes.NewForConfig(o.RESTConfig)
	if err != nil {
		return err
	}

	return nil
}

// Run executes status certificate command
func (o *Options) Run(args []string) error {
	ctx := context.TODO()

	secret, err := o.clientSet.CoreV1().Secrets(o.Namespace).Get(ctx, args[0], metav1.GetOptions{})
	if err != nil {
		return fmt.Errorf("error when finding Secret %q: %w\n", args[0], err)
	}

	certData := secret.Data[corev1.TLSCertKey]
	certs, err := splitPEMs(certData)
	if err != nil {
		return err
	}
	if len(certs) < 1 {
		return errors.New("no PEM data found in secret")
	}

	intermediates := [][]byte(nil)
	if len(certs) > 1 {
		intermediates = certs[1:]
	}

	// we only want to inspect the leaf certificate
	x509Cert, err := pki.DecodeX509CertificateBytes(certs[0])
	if err != nil {
		return fmt.Errorf("error when parsing 'tls.crt': %w", err)
	}

	out := []string{
		describeValidFor(x509Cert),
		describeValidityPeriod(x509Cert),
		describeIssuedBy(x509Cert),
		describeIssuedFor(x509Cert),
		describeCertificate(x509Cert),
		describeDebugging(x509Cert, intermediates, secret.Data[cmmeta.TLSCAKey]),
	}

	fmt.Println(strings.Join(out, "\n\n"))

	return nil
}

func describeValidFor(cert *x509.Certificate) string {
	return fmt.Sprintf(validForTemplate,
		printSlice(cert.DNSNames),
		printSlice(pki.URLsToString(cert.URIs)),
		printSlice(pki.IPAddressesToString(cert.IPAddresses)),
		printSlice(cert.EmailAddresses),
		printKeyUsage(pki.BuildCertManagerKeyUsages(cert.KeyUsage, cert.ExtKeyUsage)),
	)
}

func describeValidityPeriod(cert *x509.Certificate) string {
	return fmt.Sprintf(validityPeriodTemplate,
		cert.NotBefore.Format(time.RFC1123),
		cert.NotAfter.Format(time.RFC1123),
	)
}

func describeIssuedBy(cert *x509.Certificate) string {
	return fmt.Sprintf(issuedByTemplate,
		printOrNone(cert.Issuer.CommonName),
		printSliceOrOne(cert.Issuer.Organization),
		printSliceOrOne(cert.Issuer.OrganizationalUnit),
		printSliceOrOne(cert.Issuer.Country),
	)
}

func describeIssuedFor(cert *x509.Certificate) string {
	return fmt.Sprintf(issuedForTemplate,
		printOrNone(cert.Subject.CommonName),
		printSliceOrOne(cert.Subject.Organization),
		printSliceOrOne(cert.Subject.OrganizationalUnit),
		printSliceOrOne(cert.Subject.Country),
	)
}

func describeCertificate(cert *x509.Certificate) string {
	return fmt.Sprintf(certificateTemplate,
		cert.SignatureAlgorithm.String(),
		cert.PublicKeyAlgorithm.String(),
		cert.SerialNumber.String(),
		fingerprintCert(cert),
		cert.IsCA,
		printSliceOrOne(cert.CRLDistributionPoints),
		printSliceOrOne(cert.OCSPServer),
	)
}

func describeDebugging(cert *x509.Certificate, intermediates [][]byte, ca []byte) string {
	return fmt.Sprintf(debuggingTemplate,
		describeTrusted(cert, intermediates),
		describeCRL(cert),
		describeOCSP(cert, intermediates, ca),
	)
}

func describeCRL(cert *x509.Certificate) string {
	if len(cert.CRLDistributionPoints) < 1 {
		return "No CRL endpoints set"
	}

	hasChecked := false
	for _, crlURL := range cert.CRLDistributionPoints {
		u, err := url.Parse(crlURL)
		if err != nil {
			continue // not a valid URL
		}
		if u.Scheme != "ldap" && u.Scheme != "https" {
			continue
		}

		hasChecked = true
		valid, err := checkCRLValidCert(cert, crlURL)
		if err != nil {
			return fmt.Sprintf("Cannot check CRL: %s", err.Error())
		}
		if !valid {
			return fmt.Sprintf("Revoked by %s", crlURL)
		}
	}

	if !hasChecked {
		return "No CRL endpoints we support found"
	}

	return "Valid"
}

func describeOCSP(cert *x509.Certificate, intermediates [][]byte, ca []byte) string {
	if len(ca) > 1 {
		intermediates = append([][]byte{ca}, intermediates...)
	}
	if len(intermediates) < 1 {
		return "Cannot check OCSP, does not have a CA or intermediate certificate provided"
	}
	issuerCert, err := pki.DecodeX509CertificateBytes(intermediates[len(intermediates)-1])
	if err != nil {
		return fmt.Sprintf("Cannot parse intermediate certificate: %s", err.Error())
	}

	valid, err := checkOCSPValidCert(cert, issuerCert)
	if err != nil {
		return fmt.Sprintf("Cannot check OCSP: %s", err.Error())
	}

	if !valid {
		return "Marked as revoked"
	}

	return "valid"
}

func describeTrusted(cert *x509.Certificate, intermediates [][]byte) string {
	systemPool, err := x509.SystemCertPool()
	if err != nil {
		return fmt.Sprintf("Error getting system CA store: %s", err.Error())
	}
	for _, intermediate := range intermediates {
		systemPool.AppendCertsFromPEM(intermediate)
	}
	_, err = cert.Verify(x509.VerifyOptions{
		Roots:       systemPool,
		CurrentTime: clock.Now(),
	})
	if err == nil {
		return "yes"
	}
	return fmt.Sprintf("no: %s", err.Error())
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package etcd

import (
	"fmt"
	"net"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/pkg/errors"

	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/klog/v2"
	utilsnet "k8s.io/utils/net"

	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
	"k8s.io/kubernetes/cmd/kubeadm/app/features"
	"k8s.io/kubernetes/cmd/kubeadm/app/images"
	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
	etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd"
	staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod"
	"k8s.io/kubernetes/cmd/kubeadm/app/util/users"
)

const (
	etcdVolumeName           = "etcd-data"
	certsVolumeName          = "etcd-certs"
	etcdHealthyCheckInterval = 5 * time.Second
	etcdHealthyCheckRetries  = 8
)

// CreateLocalEtcdStaticPodManifestFile will write local etcd static pod manifest file.
// This function is used by init - when the etcd cluster is empty - or by kubeadm
// upgrade - when the etcd cluster is already up and running (and the --initial-cluster flag have no impact)
func CreateLocalEtcdStaticPodManifestFile(manifestDir, patchesDir string, nodeName string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, isDryRun bool) error {
	if cfg.Etcd.External != nil {
		return errors.New("etcd static pod manifest cannot be generated for cluster using external etcd")
	}

	if err := prepareAndWriteEtcdStaticPod(manifestDir, patchesDir, cfg, endpoint, nodeName, []etcdutil.Member{}, isDryRun); err != nil {
		return err
	}

	klog.V(1).Infof("[etcd] wrote Static Pod manifest for a local etcd member to %q\n", kubeadmconstants.GetStaticPodFilepath(kubeadmconstants.Etcd, manifestDir))
	return nil
}

// CheckLocalEtcdClusterStatus verifies health state of local/stacked etcd cluster before installing a new etcd member
func CheckLocalEtcdClusterStatus(client clientset.Interface, certificatesDir string) error {
	klog.V(1).Info("[etcd] Checking etcd cluster health")

	// creates an etcd client that connects to all the local/stacked etcd members
	klog.V(1).Info("creating etcd client that connects to etcd pods")
	etcdClient, err := etcdutil.NewFromCluster(client, certificatesDir)
	if err != nil {
		return err
	}

	// Checking health state
	err = etcdClient.CheckClusterHealth()
	if err != nil {
		return errors.Wrap(err, "etcd cluster is not healthy")
	}

	return nil
}

// RemoveStackedEtcdMemberFromCluster will remove a local etcd member from etcd cluster,
// when reset the control plane node.
func RemoveStackedEtcdMemberFromCluster(client clientset.Interface, cfg *kubeadmapi.InitConfiguration) error {
	// creates an etcd client that connects to all the local/stacked etcd members
	klog.V(1).Info("[etcd] creating etcd client that connects to etcd pods")
	etcdClient, err := etcdutil.NewFromCluster(client, cfg.CertificatesDir)
	if err != nil {
		return err
	}

	members, err := etcdClient.ListMembers()
	if err != nil {
		return err
	}
	// If this is the only remaining stacked etcd member in the cluster, calling RemoveMember()
	// is not needed.
	if len(members) == 1 {
		etcdClientAddress := etcdutil.GetClientURL(&cfg.LocalAPIEndpoint)
		for _, endpoint := range etcdClient.Endpoints {
			if endpoint == etcdClientAddress {
				klog.V(1).Info("[etcd] This is the only remaining etcd member in the etcd cluster, skip removing it")
				return nil
			}
		}
	}

	// notifies the other members of the etcd cluster about the removing member
	etcdPeerAddress := etcdutil.GetPeerURL(&cfg.LocalAPIEndpoint)

	klog.V(2).Infof("[etcd] get the member id from peer: %s", etcdPeerAddress)
	id, err := etcdClient.GetMemberID(etcdPeerAddress)
	if err != nil {
		return err
	}

	klog.V(1).Infof("[etcd] removing etcd member: %s, id: %d", etcdPeerAddress, id)
	members, err = etcdClient.RemoveMember(id)
	if err != nil {
		return err
	}
	klog.V(1).Infof("[etcd] Updated etcd member list: %v", members)

	return nil
}

// CreateStackedEtcdStaticPodManifestFile will write local etcd static pod manifest file
// for an additional etcd member that is joining an existing local/stacked etcd cluster.
// Other members of the etcd cluster will be notified of the joining node in beforehand as well.
func CreateStackedEtcdStaticPodManifestFile(client clientset.Interface, manifestDir, patchesDir string, nodeName string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, isDryRun bool, certificatesDir string) error {
	// creates an etcd client that connects to all the local/stacked etcd members
	klog.V(1).Info("creating etcd client that connects to etcd pods")
	etcdClient, err := etcdutil.NewFromCluster(client, certificatesDir)
	if err != nil {
		return err
	}

	etcdPeerAddress := etcdutil.GetPeerURL(endpoint)

	var cluster []etcdutil.Member
	if isDryRun {
		fmt.Printf("[etcd] Would add etcd member: %s\n", etcdPeerAddress)
	} else {
		klog.V(1).Infof("[etcd] Adding etcd member: %s", etcdPeerAddress)
		cluster, err = etcdClient.AddMember(nodeName, etcdPeerAddress)
		if err != nil {
			return err
		}
		fmt.Println("[etcd] Announced new etcd member joining to the existing etcd cluster")
		klog.V(1).Infof("Updated etcd member list: %v", cluster)
	}

	fmt.Printf("[etcd] Creating static Pod manifest for %q\n", kubeadmconstants.Etcd)

	if err := prepareAndWriteEtcdStaticPod(manifestDir, patchesDir, cfg, endpoint, nodeName, cluster, isDryRun); err != nil {
		return err
	}

	if isDryRun {
		fmt.Println("[etcd] Would wait for the new etcd member to join the cluster")
		return nil
	}

	fmt.Printf("[etcd] Waiting for the new etcd member to join the cluster. This can take up to %v\n", etcdHealthyCheckInterval*etcdHealthyCheckRetries)
	if _, err := etcdClient.WaitForClusterAvailable(etcdHealthyCheckRetries, etcdHealthyCheckInterval); err != nil {
		return err
	}

	return nil
}

// GetEtcdPodSpec returns the etcd static Pod actualized to the context of the current configuration
// NB. GetEtcdPodSpec methods holds the information about how kubeadm creates etcd static pod manifests.
func GetEtcdPodSpec(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, nodeName string, initialCluster []etcdutil.Member) v1.Pod {
	const etcdHealthEndpoint = "/health?serializable=true"
	pathType := v1.HostPathDirectoryOrCreate
	etcdMounts := map[string]v1.Volume{
		etcdVolumeName:  staticpodutil.NewVolume(etcdVolumeName, cfg.Etcd.Local.DataDir, &pathType),
		certsVolumeName: staticpodutil.NewVolume(certsVolumeName, cfg.CertificatesDir+"/etcd", &pathType),
	}
	// probeHostname returns the correct localhost IP address family based on the endpoint AdvertiseAddress
	probeHostname, probePort, probeScheme := staticpodutil.GetEtcdProbeEndpoint(&cfg.Etcd, utilsnet.IsIPv6String(endpoint.AdvertiseAddress))
	return staticpodutil.ComponentPod(
		v1.Container{
			Name:            kubeadmconstants.Etcd,
			Command:         getEtcdCommand(cfg, endpoint, nodeName, initialCluster),
			Image:           images.GetEtcdImage(cfg),
			ImagePullPolicy: v1.PullIfNotPresent,
			// Mount the etcd datadir path read-write so etcd can store data in a more persistent manner
			VolumeMounts: []v1.VolumeMount{
				staticpodutil.NewVolumeMount(etcdVolumeName, cfg.Etcd.Local.DataDir, false),
				staticpodutil.NewVolumeMount(certsVolumeName, cfg.CertificatesDir+"/etcd", false),
			},
			Resources: v1.ResourceRequirements{
				Requests: v1.ResourceList{
					v1.ResourceCPU:    resource.MustParse("100m"),
					v1.ResourceMemory: resource.MustParse("100Mi"),
				},
			},
			LivenessProbe: staticpodutil.LivenessProbe(probeHostname, etcdHealthEndpoint, probePort, probeScheme),
			StartupProbe:  staticpodutil.StartupProbe(probeHostname, etcdHealthEndpoint, probePort, probeScheme, cfg.APIServer.TimeoutForControlPlane),
		},
		etcdMounts,
		// etcd will listen on the advertise address of the API server, in a different port (2379)
		map[string]string{kubeadmconstants.EtcdAdvertiseClientUrlsAnnotationKey: etcdutil.GetClientURL(endpoint)},
	)
}

// getEtcdCommand builds the right etcd command from the given config object
func getEtcdCommand(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, nodeName string, initialCluster []etcdutil.Member) []string {
	// localhost IP family should be the same that the AdvertiseAddress
	etcdLocalhostAddress := "127.0.0.1"
	if utilsnet.IsIPv6String(endpoint.AdvertiseAddress) {
		etcdLocalhostAddress = "::1"
	}
	defaultArguments := map[string]string{
		"name": nodeName,
		// TODO: start using --initial-corrupt-check once the graduated flag is available:
		// https://github.com/kubernetes/kubeadm/issues/2676
		"experimental-initial-corrupt-check": "true",
		"listen-client-urls":                 fmt.Sprintf("%s,%s", etcdutil.GetClientURLByIP(etcdLocalhostAddress), etcdutil.GetClientURL(endpoint)),
		"advertise-client-urls":              etcdutil.GetClientURL(endpoint),
		"listen-peer-urls":                   etcdutil.GetPeerURL(endpoint),
		"initial-advertise-peer-urls":        etcdutil.GetPeerURL(endpoint),
		"data-dir":                           cfg.Etcd.Local.DataDir,
		"cert-file":                          filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerCertName),
		"key-file":                           filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerKeyName),
		"trusted-ca-file":                    filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdCACertName),
		"client-cert-auth":                   "true",
		"peer-cert-file":                     filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdPeerCertName),
		"peer-key-file":                      filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdPeerKeyName),
		"peer-trusted-ca-file":               filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdCACertName),
		"peer-client-cert-auth":              "true",
		"snapshot-count":                     "10000",
		"listen-metrics-urls":                fmt.Sprintf("http://%s", net.JoinHostPort(etcdLocalhostAddress, strconv.Itoa(kubeadmconstants.EtcdMetricsPort))),
	}

	if len(initialCluster) == 0 {
		defaultArguments["initial-cluster"] = fmt.Sprintf("%s=%s", nodeName, etcdutil.GetPeerURL(endpoint))
	} else {
		// NB. the joining etcd member should be part of the initialCluster list
		endpoints := []string{}
		for _, member := range initialCluster {
			endpoints = append(endpoints, fmt.Sprintf("%s=%s", member.Name, member.PeerURL))
		}

		defaultArguments["initial-cluster"] = strings.Join(endpoints, ",")
		defaultArguments["initial-cluster-state"] = "existing"
	}

	command := []string{"etcd"}
	command = append(command, kubeadmutil.BuildArgumentListFromMap(defaultArguments, cfg.Etcd.Local.ExtraArgs)...)
	return command
}

func prepareAndWriteEtcdStaticPod(manifestDir string, patchesDir string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, nodeName string, initialCluster []etcdutil.Member, isDryRun bool) error {
	// gets etcd StaticPodSpec, actualized for the current ClusterConfiguration and the new list of etcd members
	spec := GetEtcdPodSpec(cfg, endpoint, nodeName, initialCluster)

	var usersAndGroups *users.UsersAndGroups
	var err error
	if features.Enabled(cfg.FeatureGates, features.RootlessControlPlane) {
		if isDryRun {
			fmt.Printf("[etcd] Would create users and groups for %q to run as non-root\n", kubeadmconstants.Etcd)
			fmt.Printf("[etcd] Would update static pod manifest for %q to run run as non-root\n", kubeadmconstants.Etcd)
		} else {
			usersAndGroups, err = staticpodutil.GetUsersAndGroups()
			if err != nil {
				return errors.Wrap(err, "failed to create users and groups")
			}
			// usersAndGroups is nil on non-linux.
			if usersAndGroups != nil {
				if err := staticpodutil.RunComponentAsNonRoot(kubeadmconstants.Etcd, &spec, usersAndGroups, cfg); err != nil {
					return errors.Wrapf(err, "failed to run component %q as non-root", kubeadmconstants.Etcd)
				}
			}
		}
	}

	// if patchesDir is defined, patch the static Pod manifest
	if patchesDir != "" {
		patchedSpec, err := staticpodutil.PatchStaticPod(&spec, patchesDir, os.Stdout)
		if err != nil {
			return errors.Wrapf(err, "failed to patch static Pod manifest file for %q", kubeadmconstants.Etcd)
		}
		spec = *patchedSpec
	}

	// writes etcd StaticPod to disk
	if err := staticpodutil.WriteStaticPodToDisk(kubeadmconstants.Etcd, manifestDir, spec); err != nil {
		return err
	}

	return nil
}
<commit_msg>kubeadm: use non-serializable startup probe for etcd pods<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package etcd

import (
	"fmt"
	"net"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/pkg/errors"

	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/klog/v2"
	utilsnet "k8s.io/utils/net"

	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
	"k8s.io/kubernetes/cmd/kubeadm/app/features"
	"k8s.io/kubernetes/cmd/kubeadm/app/images"
	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
	etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd"
	staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod"
	"k8s.io/kubernetes/cmd/kubeadm/app/util/users"
)

const (
	etcdVolumeName           = "etcd-data"
	certsVolumeName          = "etcd-certs"
	etcdHealthyCheckInterval = 5 * time.Second
	etcdHealthyCheckRetries  = 8
)

// CreateLocalEtcdStaticPodManifestFile will write local etcd static pod manifest file.
// This function is used by init - when the etcd cluster is empty - or by kubeadm
// upgrade - when the etcd cluster is already up and running (and the --initial-cluster flag have no impact)
func CreateLocalEtcdStaticPodManifestFile(manifestDir, patchesDir string, nodeName string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, isDryRun bool) error {
	if cfg.Etcd.External != nil {
		return errors.New("etcd static pod manifest cannot be generated for cluster using external etcd")
	}

	if err := prepareAndWriteEtcdStaticPod(manifestDir, patchesDir, cfg, endpoint, nodeName, []etcdutil.Member{}, isDryRun); err != nil {
		return err
	}

	klog.V(1).Infof("[etcd] wrote Static Pod manifest for a local etcd member to %q\n", kubeadmconstants.GetStaticPodFilepath(kubeadmconstants.Etcd, manifestDir))
	return nil
}

// CheckLocalEtcdClusterStatus verifies health state of local/stacked etcd cluster before installing a new etcd member
func CheckLocalEtcdClusterStatus(client clientset.Interface, certificatesDir string) error {
	klog.V(1).Info("[etcd] Checking etcd cluster health")

	// creates an etcd client that connects to all the local/stacked etcd members
	klog.V(1).Info("creating etcd client that connects to etcd pods")
	etcdClient, err := etcdutil.NewFromCluster(client, certificatesDir)
	if err != nil {
		return err
	}

	// Checking health state
	err = etcdClient.CheckClusterHealth()
	if err != nil {
		return errors.Wrap(err, "etcd cluster is not healthy")
	}

	return nil
}

// RemoveStackedEtcdMemberFromCluster will remove a local etcd member from etcd cluster,
// when reset the control plane node.
func RemoveStackedEtcdMemberFromCluster(client clientset.Interface, cfg *kubeadmapi.InitConfiguration) error {
	// creates an etcd client that connects to all the local/stacked etcd members
	klog.V(1).Info("[etcd] creating etcd client that connects to etcd pods")
	etcdClient, err := etcdutil.NewFromCluster(client, cfg.CertificatesDir)
	if err != nil {
		return err
	}

	members, err := etcdClient.ListMembers()
	if err != nil {
		return err
	}
	// If this is the only remaining stacked etcd member in the cluster, calling RemoveMember()
	// is not needed.
	if len(members) == 1 {
		etcdClientAddress := etcdutil.GetClientURL(&cfg.LocalAPIEndpoint)
		for _, endpoint := range etcdClient.Endpoints {
			if endpoint == etcdClientAddress {
				klog.V(1).Info("[etcd] This is the only remaining etcd member in the etcd cluster, skip removing it")
				return nil
			}
		}
	}

	// notifies the other members of the etcd cluster about the removing member
	etcdPeerAddress := etcdutil.GetPeerURL(&cfg.LocalAPIEndpoint)

	klog.V(2).Infof("[etcd] get the member id from peer: %s", etcdPeerAddress)
	id, err := etcdClient.GetMemberID(etcdPeerAddress)
	if err != nil {
		return err
	}

	klog.V(1).Infof("[etcd] removing etcd member: %s, id: %d", etcdPeerAddress, id)
	members, err = etcdClient.RemoveMember(id)
	if err != nil {
		return err
	}
	klog.V(1).Infof("[etcd] Updated etcd member list: %v", members)

	return nil
}

// CreateStackedEtcdStaticPodManifestFile will write local etcd static pod manifest file
// for an additional etcd member that is joining an existing local/stacked etcd cluster.
// Other members of the etcd cluster will be notified of the joining node in beforehand as well.
func CreateStackedEtcdStaticPodManifestFile(client clientset.Interface, manifestDir, patchesDir string, nodeName string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, isDryRun bool, certificatesDir string) error {
	// creates an etcd client that connects to all the local/stacked etcd members
	klog.V(1).Info("creating etcd client that connects to etcd pods")
	etcdClient, err := etcdutil.NewFromCluster(client, certificatesDir)
	if err != nil {
		return err
	}

	etcdPeerAddress := etcdutil.GetPeerURL(endpoint)

	var cluster []etcdutil.Member
	if isDryRun {
		fmt.Printf("[etcd] Would add etcd member: %s\n", etcdPeerAddress)
	} else {
		klog.V(1).Infof("[etcd] Adding etcd member: %s", etcdPeerAddress)
		cluster, err = etcdClient.AddMember(nodeName, etcdPeerAddress)
		if err != nil {
			return err
		}
		fmt.Println("[etcd] Announced new etcd member joining to the existing etcd cluster")
		klog.V(1).Infof("Updated etcd member list: %v", cluster)
	}

	fmt.Printf("[etcd] Creating static Pod manifest for %q\n", kubeadmconstants.Etcd)

	if err := prepareAndWriteEtcdStaticPod(manifestDir, patchesDir, cfg, endpoint, nodeName, cluster, isDryRun); err != nil {
		return err
	}

	if isDryRun {
		fmt.Println("[etcd] Would wait for the new etcd member to join the cluster")
		return nil
	}

	fmt.Printf("[etcd] Waiting for the new etcd member to join the cluster. This can take up to %v\n", etcdHealthyCheckInterval*etcdHealthyCheckRetries)
	if _, err := etcdClient.WaitForClusterAvailable(etcdHealthyCheckRetries, etcdHealthyCheckInterval); err != nil {
		return err
	}

	return nil
}

// GetEtcdPodSpec returns the etcd static Pod actualized to the context of the current configuration
// NB. GetEtcdPodSpec methods holds the information about how kubeadm creates etcd static pod manifests.
func GetEtcdPodSpec(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, nodeName string, initialCluster []etcdutil.Member) v1.Pod {
	pathType := v1.HostPathDirectoryOrCreate
	etcdMounts := map[string]v1.Volume{
		etcdVolumeName:  staticpodutil.NewVolume(etcdVolumeName, cfg.Etcd.Local.DataDir, &pathType),
		certsVolumeName: staticpodutil.NewVolume(certsVolumeName, cfg.CertificatesDir+"/etcd", &pathType),
	}
	// probeHostname returns the correct localhost IP address family based on the endpoint AdvertiseAddress
	probeHostname, probePort, probeScheme := staticpodutil.GetEtcdProbeEndpoint(&cfg.Etcd, utilsnet.IsIPv6String(endpoint.AdvertiseAddress))
	return staticpodutil.ComponentPod(
		v1.Container{
			Name:            kubeadmconstants.Etcd,
			Command:         getEtcdCommand(cfg, endpoint, nodeName, initialCluster),
			Image:           images.GetEtcdImage(cfg),
			ImagePullPolicy: v1.PullIfNotPresent,
			// Mount the etcd datadir path read-write so etcd can store data in a more persistent manner
			VolumeMounts: []v1.VolumeMount{
				staticpodutil.NewVolumeMount(etcdVolumeName, cfg.Etcd.Local.DataDir, false),
				staticpodutil.NewVolumeMount(certsVolumeName, cfg.CertificatesDir+"/etcd", false),
			},
			Resources: v1.ResourceRequirements{
				Requests: v1.ResourceList{
					v1.ResourceCPU:    resource.MustParse("100m"),
					v1.ResourceMemory: resource.MustParse("100Mi"),
				},
			},
			LivenessProbe: staticpodutil.LivenessProbe(probeHostname, "/health?exclude=NOSPACE&serializable=true", probePort, probeScheme),
			StartupProbe:  staticpodutil.StartupProbe(probeHostname, "/health?serializable=false", probePort, probeScheme, cfg.APIServer.TimeoutForControlPlane),
		},
		etcdMounts,
		// etcd will listen on the advertise address of the API server, in a different port (2379)
		map[string]string{kubeadmconstants.EtcdAdvertiseClientUrlsAnnotationKey: etcdutil.GetClientURL(endpoint)},
	)
}

// getEtcdCommand builds the right etcd command from the given config object
func getEtcdCommand(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, nodeName string, initialCluster []etcdutil.Member) []string {
	// localhost IP family should be the same that the AdvertiseAddress
	etcdLocalhostAddress := "127.0.0.1"
	if utilsnet.IsIPv6String(endpoint.AdvertiseAddress) {
		etcdLocalhostAddress = "::1"
	}
	defaultArguments := map[string]string{
		"name": nodeName,
		// TODO: start using --initial-corrupt-check once the graduated flag is available:
		// https://github.com/kubernetes/kubeadm/issues/2676
		"experimental-initial-corrupt-check": "true",
		"listen-client-urls":                 fmt.Sprintf("%s,%s", etcdutil.GetClientURLByIP(etcdLocalhostAddress), etcdutil.GetClientURL(endpoint)),
		"advertise-client-urls":              etcdutil.GetClientURL(endpoint),
		"listen-peer-urls":                   etcdutil.GetPeerURL(endpoint),
		"initial-advertise-peer-urls":        etcdutil.GetPeerURL(endpoint),
		"data-dir":                           cfg.Etcd.Local.DataDir,
		"cert-file":                          filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerCertName),
		"key-file":                           filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerKeyName),
		"trusted-ca-file":                    filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdCACertName),
		"client-cert-auth":                   "true",
		"peer-cert-file":                     filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdPeerCertName),
		"peer-key-file":                      filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdPeerKeyName),
		"peer-trusted-ca-file":               filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdCACertName),
		"peer-client-cert-auth":              "true",
		"snapshot-count":                     "10000",
		"listen-metrics-urls":                fmt.Sprintf("http://%s", net.JoinHostPort(etcdLocalhostAddress, strconv.Itoa(kubeadmconstants.EtcdMetricsPort))),
	}

	if len(initialCluster) == 0 {
		defaultArguments["initial-cluster"] = fmt.Sprintf("%s=%s", nodeName, etcdutil.GetPeerURL(endpoint))
	} else {
		// NB. the joining etcd member should be part of the initialCluster list
		endpoints := []string{}
		for _, member := range initialCluster {
			endpoints = append(endpoints, fmt.Sprintf("%s=%s", member.Name, member.PeerURL))
		}

		defaultArguments["initial-cluster"] = strings.Join(endpoints, ",")
		defaultArguments["initial-cluster-state"] = "existing"
	}

	command := []string{"etcd"}
	command = append(command, kubeadmutil.BuildArgumentListFromMap(defaultArguments, cfg.Etcd.Local.ExtraArgs)...)
	return command
}

func prepareAndWriteEtcdStaticPod(manifestDir string, patchesDir string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, nodeName string, initialCluster []etcdutil.Member, isDryRun bool) error {
	// gets etcd StaticPodSpec, actualized for the current ClusterConfiguration and the new list of etcd members
	spec := GetEtcdPodSpec(cfg, endpoint, nodeName, initialCluster)

	var usersAndGroups *users.UsersAndGroups
	var err error
	if features.Enabled(cfg.FeatureGates, features.RootlessControlPlane) {
		if isDryRun {
			fmt.Printf("[etcd] Would create users and groups for %q to run as non-root\n", kubeadmconstants.Etcd)
			fmt.Printf("[etcd] Would update static pod manifest for %q to run run as non-root\n", kubeadmconstants.Etcd)
		} else {
			usersAndGroups, err = staticpodutil.GetUsersAndGroups()
			if err != nil {
				return errors.Wrap(err, "failed to create users and groups")
			}
			// usersAndGroups is nil on non-linux.
			if usersAndGroups != nil {
				if err := staticpodutil.RunComponentAsNonRoot(kubeadmconstants.Etcd, &spec, usersAndGroups, cfg); err != nil {
					return errors.Wrapf(err, "failed to run component %q as non-root", kubeadmconstants.Etcd)
				}
			}
		}
	}

	// if patchesDir is defined, patch the static Pod manifest
	if patchesDir != "" {
		patchedSpec, err := staticpodutil.PatchStaticPod(&spec, patchesDir, os.Stdout)
		if err != nil {
			return errors.Wrapf(err, "failed to patch static Pod manifest file for %q", kubeadmconstants.Etcd)
		}
		spec = *patchedSpec
	}

	// writes etcd StaticPod to disk
	if err := staticpodutil.WriteStaticPodToDisk(kubeadmconstants.Etcd, manifestDir, spec); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package collectors

import (
	"strings"
	"time"

	"github.com/StackExchange/scollector/opentsdb"
)

func init() {
	collectors = append(collectors, &IntervalCollector{F: c_omreport_chassis, Interval: time.Minute * 5})
	collectors = append(collectors, &IntervalCollector{F: c_omreport_system, Interval: time.Minute * 5})
	collectors = append(collectors, &IntervalCollector{F: c_omreport_storage_enclosure, Interval: time.Minute * 5})
	collectors = append(collectors, &IntervalCollector{F: c_omreport_storage_vdisk, Interval: time.Minute * 5})
	collectors = append(collectors, &IntervalCollector{F: c_omreport_storage_controller, Interval: time.Minute * 5})
	collectors = append(collectors, &IntervalCollector{F: c_omreport_storage_battery, Interval: time.Minute * 5})
	collectors = append(collectors, &IntervalCollector{F: c_omreport_ps, Interval: time.Minute * 5})
	collectors = append(collectors, &IntervalCollector{F: c_omreport_ps_amps, Interval: time.Minute * 5})
	collectors = append(collectors, &IntervalCollector{F: c_omreport_ps_volts, Interval: time.Minute * 5})
}

func c_omreport_chassis() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) != 2 || fields[0] == "SEVERITY" {
			return
		}
		sev := 0
		if fields[0] != "Ok" {
			sev = 1
		}
		component := strings.Replace(fields[1], " ", "_", -1)
		Add(&md, "hw.chassis", sev, opentsdb.TagSet{"component": component})
	}, "omreport", "chassis", "-fmt", "ssv")
	return md
}

func c_omreport_system() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) != 2 || fields[0] == "SEVERITY" {
			return
		}
		sev := 0
		if fields[0] != "Ok" {
			sev = 1
		}
		component := strings.Replace(fields[1], " ", "_", -1)
		Add(&md, "hw.system", sev, opentsdb.TagSet{"component": component})
	}, "omreport", "system", "-fmt", "ssv")
	return md
}

func c_omreport_storage_enclosure() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) < 3 || fields[0] == "ID" {
			return
		}
		sev := 0
		if fields[1] != "Ok" && fields[1] != "Non-Critical" {
			sev = 1
		}
		id := strings.Replace(fields[0], ":", "_", -1)
		Add(&md, "hw.storage.enclosure", sev, opentsdb.TagSet{"id": id})
	}, "omreport", "storage", "enclosure", "-fmt", "ssv")
	return md
}

func c_omreport_storage_vdisk() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) < 3 || fields[0] == "ID" {
			return
		}
		sev := 0
		if fields[1] != "Ok" && fields[1] != "Non-Critical" {
			sev = 1
		}
		id := strings.Replace(fields[0], ":", "_", -1)
		Add(&md, "hw.storage.vdisk", sev, opentsdb.TagSet{"id": id})
	}, "omreport", "storage", "vdisk", "-fmt", "ssv")
	return md
}

func c_omreport_ps() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) < 3 || fields[0] == "Index" {
			return
		}
		sev := 0
		if fields[1] != "Ok" {
			sev = 1
		}
		id := strings.Replace(fields[0], ":", "_", -1)
		Add(&md, "hw.ps", sev, opentsdb.TagSet{"id": id})
	}, "omreport", "chassis", "pwrsupplies", "-fmt", "ssv")
	return md
}

func c_omreport_ps_amps() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) != 2 || !strings.Contains(fields[0], "Current") {
			return
		}
		i_fields := strings.Fields(fields[0])
		v_fields := strings.Fields(fields[1])
		if len(i_fields) < 2 && len(v_fields) < 2 {
			return
		}
		Add(&md, "hw.ps.current", v_fields[0], opentsdb.TagSet{"id": i_fields[0]})
	}, "omreport", "chassis", "pwrmonitoring", "-fmt", "ssv")
	return md
}

func c_omreport_ps_volts() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) != 8 || !strings.Contains(fields[2], "Voltage") || fields[3] == "[N/A]" {
			return
		}
		i_fields := strings.Fields(fields[2])
		v_fields := strings.Fields(fields[3])
		if len(i_fields) < 2 && len(v_fields) < 2 {
			return
		}
		Add(&md, "hw.ps.volts", v_fields[0], opentsdb.TagSet{"id": i_fields[0]})
	}, "omreport", "chassis", "volts", "-fmt", "ssv")
	return md
}

func c_omreport_storage_battery() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) < 3 || fields[0] == "ID" {
			return
		}
		sev := 0
		if fields[1] != "Ok" && fields[1] != "Non-Critical" {
			sev = 1
		}
		id := strings.Replace(fields[0], ":", "_", -1)
		Add(&md, "hw.storage.battery", sev, opentsdb.TagSet{"id": id})
	}, "omreport", "storage", "battery", "-fmt", "ssv")
	return md
}

func c_omreport_storage_controller() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) < 3 || fields[0] == "ID" {
			return
		}
		sev := 0
		if fields[1] != "Ok" && fields[1] != "Non-Critical" {
			sev = 1
		}
		c_omreport_storage_pdisk(fields[0], &md)
		id := strings.Replace(fields[0], ":", "_", -1)
		Add(&md, "hw.storage.controller", sev, opentsdb.TagSet{"id": id})
	}, "omreport", "storage", "controller", "-fmt", "ssv")
	return md
}

// c_omreport_storage_pdisk is called from the controller func, since it needs the encapsulating id.
func c_omreport_storage_pdisk(id string, md *opentsdb.MultiDataPoint) {
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) < 3 || fields[0] == "ID" {
			return
		}
		sev := 0
		if fields[1] != "Ok" && fields[1] != "Non-Critical" {
			sev = 1
		}
		//Need to find out what the various ID formats might be
		id := strings.Replace(fields[0], ":", "_", -1)
		Add(md, "hw.storage.pdisk", sev, opentsdb.TagSet{"id": id})
	}, "omreport", "storage", "pdisk", "controller="+id, "-fmt", "ssv")
}
<commit_msg>cmd/scollector: Split id field on Voltage/Current<commit_after>package collectors

import (
	"strings"
	"time"

	"github.com/StackExchange/scollector/opentsdb"
)

func init() {
	collectors = append(collectors, &IntervalCollector{F: c_omreport_chassis, Interval: time.Minute * 5})
	collectors = append(collectors, &IntervalCollector{F: c_omreport_system, Interval: time.Minute * 5})
	collectors = append(collectors, &IntervalCollector{F: c_omreport_storage_enclosure, Interval: time.Minute * 5})
	collectors = append(collectors, &IntervalCollector{F: c_omreport_storage_vdisk, Interval: time.Minute * 5})
	collectors = append(collectors, &IntervalCollector{F: c_omreport_storage_controller, Interval: time.Minute * 5})
	collectors = append(collectors, &IntervalCollector{F: c_omreport_storage_battery, Interval: time.Minute * 5})
	collectors = append(collectors, &IntervalCollector{F: c_omreport_ps, Interval: time.Minute * 5})
	collectors = append(collectors, &IntervalCollector{F: c_omreport_ps_amps, Interval: time.Minute * 5})
	collectors = append(collectors, &IntervalCollector{F: c_omreport_ps_volts, Interval: time.Minute * 5})
}

func c_omreport_chassis() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) != 2 || fields[0] == "SEVERITY" {
			return
		}
		sev := 0
		if fields[0] != "Ok" {
			sev = 1
		}
		component := strings.Replace(fields[1], " ", "_", -1)
		Add(&md, "hw.chassis", sev, opentsdb.TagSet{"component": component})
	}, "omreport", "chassis", "-fmt", "ssv")
	return md
}

func c_omreport_system() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) != 2 || fields[0] == "SEVERITY" {
			return
		}
		sev := 0
		if fields[0] != "Ok" {
			sev = 1
		}
		component := strings.Replace(fields[1], " ", "_", -1)
		Add(&md, "hw.system", sev, opentsdb.TagSet{"component": component})
	}, "omreport", "system", "-fmt", "ssv")
	return md
}

func c_omreport_storage_enclosure() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) < 3 || fields[0] == "ID" {
			return
		}
		sev := 0
		if fields[1] != "Ok" && fields[1] != "Non-Critical" {
			sev = 1
		}
		id := strings.Replace(fields[0], ":", "_", -1)
		Add(&md, "hw.storage.enclosure", sev, opentsdb.TagSet{"id": id})
	}, "omreport", "storage", "enclosure", "-fmt", "ssv")
	return md
}

func c_omreport_storage_vdisk() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) < 3 || fields[0] == "ID" {
			return
		}
		sev := 0
		if fields[1] != "Ok" && fields[1] != "Non-Critical" {
			sev = 1
		}
		id := strings.Replace(fields[0], ":", "_", -1)
		Add(&md, "hw.storage.vdisk", sev, opentsdb.TagSet{"id": id})
	}, "omreport", "storage", "vdisk", "-fmt", "ssv")
	return md
}

func c_omreport_ps() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) < 3 || fields[0] == "Index" {
			return
		}
		sev := 0
		if fields[1] != "Ok" {
			sev = 1
		}
		id := strings.Replace(fields[0], ":", "_", -1)
		Add(&md, "hw.ps", sev, opentsdb.TagSet{"id": id})
	}, "omreport", "chassis", "pwrsupplies", "-fmt", "ssv")
	return md
}

func c_omreport_ps_amps() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) != 2 || !strings.Contains(fields[0], "Current") {
			return
		}
		i_fields := strings.Split(fields[0], "Current")
		v_fields := strings.Fields(fields[1])
		if len(i_fields) < 2 && len(v_fields) < 2 {
			return
		}
		id := strings.Replace(i_fields[0], " ", "", -1)
		Add(&md, "hw.ps.current", v_fields[0], opentsdb.TagSet{"id": id})
	}, "omreport", "chassis", "pwrmonitoring", "-fmt", "ssv")
	return md
}

func c_omreport_ps_volts() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) != 8 || !strings.Contains(fields[2], "Voltage") || fields[3] == "[N/A]" {
			return
		}
		i_fields := strings.Split(fields[2], "Voltage")
		v_fields := strings.Fields(fields[3])
		if len(i_fields) < 2 && len(v_fields) < 2 {
			return
		}
		id := strings.Replace(i_fields[0], " ", "", -1)
		Add(&md, "hw.ps.volts", v_fields[0], opentsdb.TagSet{"id": id})
	}, "omreport", "chassis", "volts", "-fmt", "ssv")
	return md
}

func c_omreport_storage_battery() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) < 3 || fields[0] == "ID" {
			return
		}
		sev := 0
		if fields[1] != "Ok" && fields[1] != "Non-Critical" {
			sev = 1
		}
		id := strings.Replace(fields[0], ":", "_", -1)
		Add(&md, "hw.storage.battery", sev, opentsdb.TagSet{"id": id})
	}, "omreport", "storage", "battery", "-fmt", "ssv")
	return md
}

func c_omreport_storage_controller() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) < 3 || fields[0] == "ID" {
			return
		}
		sev := 0
		if fields[1] != "Ok" && fields[1] != "Non-Critical" {
			sev = 1
		}
		c_omreport_storage_pdisk(fields[0], &md)
		id := strings.Replace(fields[0], ":", "_", -1)
		Add(&md, "hw.storage.controller", sev, opentsdb.TagSet{"id": id})
	}, "omreport", "storage", "controller", "-fmt", "ssv")
	return md
}

// c_omreport_storage_pdisk is called from the controller func, since it needs the encapsulating id.
func c_omreport_storage_pdisk(id string, md *opentsdb.MultiDataPoint) {
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) < 3 || fields[0] == "ID" {
			return
		}
		sev := 0
		if fields[1] != "Ok" && fields[1] != "Non-Critical" {
			sev = 1
		}
		//Need to find out what the various ID formats might be
		id := strings.Replace(fields[0], ":", "_", -1)
		Add(md, "hw.storage.pdisk", sev, opentsdb.TagSet{"id": id})
	}, "omreport", "storage", "pdisk", "controller="+id, "-fmt", "ssv")
}
<|endoftext|>


<commit_before>/*
 * Copyright © 2015-2018 Aeneas Rekkas <aeneas+oss@aeneas.io>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * @author		Aeneas Rekkas <aeneas+oss@aeneas.io>
 * @copyright 	2015-2018 Aeneas Rekkas <aeneas+oss@aeneas.io>
 * @license 	Apache-2.0
 */

package server

import (
	"fmt"
	"net/url"
	"time"

	"github.com/gorilla/sessions"
	"github.com/julienschmidt/httprouter"
	"github.com/ory/fosite"
	"github.com/ory/fosite/compose"
	"github.com/ory/herodot"
	"github.com/ory/hydra/client"
	"github.com/ory/hydra/config"
	"github.com/ory/hydra/consent"
	"github.com/ory/hydra/jwk"
	"github.com/ory/hydra/oauth2"
	"github.com/ory/hydra/pkg"
	"github.com/ory/sqlcon"
)

func injectFositeStore(c *config.Config, clients client.Manager) {
	var ctx = c.Context()
	var store pkg.FositeStorer

	switch con := ctx.Connection.(type) {
	case *config.MemoryConnection:
		store = oauth2.NewFositeMemoryStore(clients, c.GetAccessTokenLifespan())
		break
	case *sqlcon.SQLConnection:
		store = oauth2.NewFositeSQLStore(clients, con.GetDatabase(), c.GetLogger(), c.GetAccessTokenLifespan())
		break
	case *config.PluginConnection:
		var err error
		if store, err = con.NewOAuth2Manager(clients); err != nil {
			c.GetLogger().Fatalf("Could not load client manager plugin %s", err)
		}
		break
	default:
		panic("Unknown connection type.")
	}

	ctx.FositeStore = store
}

func newOAuth2Provider(c *config.Config) (fosite.OAuth2Provider, string) {
	var ctx = c.Context()
	var store = ctx.FositeStore

	privateKey, err := createOrGetJWK(c, oauth2.OpenIDConnectKeyName, "private")
	if err != nil {
		c.GetLogger().WithError(err).Fatalf(`Could not fetch private signing key for OpenID Connect - did you forget to run "hydra migrate sql" or forget to set the SYSTEM_SECRET?`)
	}

	publicKey, err := createOrGetJWK(c, oauth2.OpenIDConnectKeyName, "public")
	if err != nil {
		c.GetLogger().WithError(err).Fatalf(`Could not fetch public signing key for OpenID Connect - did you forget to run "hydra migrate sql" or forget to set the SYSTEM_SECRET?`)
	}

	fc := &compose.Config{
		AccessTokenLifespan:            c.GetAccessTokenLifespan(),
		AuthorizeCodeLifespan:          c.GetAuthCodeLifespan(),
		IDTokenLifespan:                c.GetIDTokenLifespan(),
		HashCost:                       c.BCryptWorkFactor,
		ScopeStrategy:                  c.GetScopeStrategy(),
		SendDebugMessagesToClients:     c.SendOAuth2DebugMessagesToClients,
		EnforcePKCE:                    false,
		EnablePKCEPlainChallengeMethod: false,
	}

	return compose.Compose(
		fc,
		store,
		&compose.CommonStrategy{
			CoreStrategy:               compose.NewOAuth2HMACStrategy(fc, c.GetSystemSecret()),
			OpenIDConnectTokenStrategy: compose.NewOpenIDConnectStrategy(jwk.MustRSAPrivate(privateKey)),
		},
		nil,
		compose.OAuth2AuthorizeExplicitFactory,
		compose.OAuth2AuthorizeImplicitFactory,
		compose.OAuth2ClientCredentialsGrantFactory,
		compose.OAuth2RefreshTokenGrantFactory,
		compose.OAuth2PKCEFactory,
		compose.OpenIDConnectExplicitFactory,
		compose.OpenIDConnectHybridFactory,
		compose.OpenIDConnectImplicitFactory,
		compose.OpenIDConnectRefreshFactory,
		compose.OAuth2TokenRevocationFactory,
		compose.OAuth2TokenIntrospectionFactory,
	), publicKey.KeyID
}

func setDefaultConsentURL(s string, c *config.Config, path string) string {
	if s != "" {
		return s
	}
	proto := "https"
	if c.ForceHTTP {
		proto = "http"
	}
	host := "localhost"
	if c.BindHost != "" {
		host = c.BindHost
	}
	return fmt.Sprintf("%s://%s:%d/%s", proto, host, c.BindPort, path)
}

//func newOAuth2Handler(c *config.Config, router *httprouter.Router, cm oauth2.ConsentRequestManager, o fosite.OAuth2Provider, idTokenKeyID string) *oauth2.Handler {
func newOAuth2Handler(c *config.Config, router *httprouter.Router, cm consent.Manager, o fosite.OAuth2Provider, idTokenKeyID string) *oauth2.Handler {
	c.ConsentURL = setDefaultConsentURL(c.ConsentURL, c, "oauth2/fallbacks/consent")
	c.LoginURL = setDefaultConsentURL(c.LoginURL, c, "oauth2/fallbacks/consent")
	c.ErrorURL = setDefaultConsentURL(c.ErrorURL, c, "oauth2/fallbacks/error")

	errorURL, err := url.Parse(c.ErrorURL)
	pkg.Must(err, "Could not parse error url %s.", errorURL)

	handler := &oauth2.Handler{
		ScopesSupported:  c.OpenIDDiscoveryScopesSupported,
		UserinfoEndpoint: c.OpenIDDiscoveryUserinfoEndpoint,
		ClaimsSupported:  c.OpenIDDiscoveryClaimsSupported,
		ForcedHTTP:       c.ForceHTTP,
		OAuth2:           o,
		ScopeStrategy:    c.GetScopeStrategy(),
		Consent: &consent.DefaultStrategy{
			RequestMaxAge:     time.Minute * 15,
			AuthenticationURL: c.LoginURL,
			ConsentURL:        c.ConsentURL,
			IssuerURL:         c.Issuer,
			OAuth2AuthURL:     "/oauth2/auth",
			M:                 cm,
			CookieStore:       sessions.NewCookieStore(c.GetCookieSecret()),
			ScopeStrategy:     c.GetScopeStrategy(),
			RunsHTTPS:         !c.ForceHTTP,
		},
		//Consent: &oauth2.DefaultConsentStrategy{
		//	Issuer:                   c.Issuer,
		//	ConsentManager:           c.Context().ConsentManager,
		//	DefaultChallengeLifespan: c.GetChallengeTokenLifespan(),
		//	DefaultIDTokenLifespan:   c.GetIDTokenLifespan(),
		//	KeyID: idTokenKeyID,
		//},
		Storage:             c.Context().FositeStore,
		ErrorURL:            *errorURL,
		H:                   herodot.NewJSONWriter(c.GetLogger()),
		AccessTokenLifespan: c.GetAccessTokenLifespan(),
		CookieStore:         sessions.NewCookieStore(c.GetCookieSecret()),
		Issuer:              c.Issuer,
		L:                   c.GetLogger(),
		IDTokenPublicKeyID:  idTokenKeyID,
		IDTokenLifespan:     c.GetIDTokenLifespan(),
	}

	handler.SetRoutes(router)
	return handler
}
<commit_msg>cmd: Properly initializes consent strategy<commit_after>/*
 * Copyright © 2015-2018 Aeneas Rekkas <aeneas+oss@aeneas.io>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * @author		Aeneas Rekkas <aeneas+oss@aeneas.io>
 * @copyright 	2015-2018 Aeneas Rekkas <aeneas+oss@aeneas.io>
 * @license 	Apache-2.0
 */

package server

import (
	"fmt"
	"net/url"
	"time"

	"github.com/gorilla/sessions"
	"github.com/julienschmidt/httprouter"
	"github.com/ory/fosite"
	"github.com/ory/fosite/compose"
	"github.com/ory/fosite/handler/openid"
	"github.com/ory/herodot"
	"github.com/ory/hydra/client"
	"github.com/ory/hydra/config"
	"github.com/ory/hydra/consent"
	"github.com/ory/hydra/jwk"
	"github.com/ory/hydra/oauth2"
	"github.com/ory/hydra/pkg"
	"github.com/ory/sqlcon"
)

func injectFositeStore(c *config.Config, clients client.Manager) {
	var ctx = c.Context()
	var store pkg.FositeStorer

	switch con := ctx.Connection.(type) {
	case *config.MemoryConnection:
		store = oauth2.NewFositeMemoryStore(clients, c.GetAccessTokenLifespan())
		break
	case *sqlcon.SQLConnection:
		store = oauth2.NewFositeSQLStore(clients, con.GetDatabase(), c.GetLogger(), c.GetAccessTokenLifespan())
		break
	case *config.PluginConnection:
		var err error
		if store, err = con.NewOAuth2Manager(clients); err != nil {
			c.GetLogger().Fatalf("Could not load client manager plugin %s", err)
		}
		break
	default:
		panic("Unknown connection type.")
	}

	ctx.FositeStore = store
}

func newOAuth2Provider(c *config.Config) (fosite.OAuth2Provider, string) {
	var ctx = c.Context()
	var store = ctx.FositeStore

	privateKey, err := createOrGetJWK(c, oauth2.OpenIDConnectKeyName, "private")
	if err != nil {
		c.GetLogger().WithError(err).Fatalf(`Could not fetch private signing key for OpenID Connect - did you forget to run "hydra migrate sql" or forget to set the SYSTEM_SECRET?`)
	}

	publicKey, err := createOrGetJWK(c, oauth2.OpenIDConnectKeyName, "public")
	if err != nil {
		c.GetLogger().WithError(err).Fatalf(`Could not fetch public signing key for OpenID Connect - did you forget to run "hydra migrate sql" or forget to set the SYSTEM_SECRET?`)
	}

	fc := &compose.Config{
		AccessTokenLifespan:            c.GetAccessTokenLifespan(),
		AuthorizeCodeLifespan:          c.GetAuthCodeLifespan(),
		IDTokenLifespan:                c.GetIDTokenLifespan(),
		HashCost:                       c.BCryptWorkFactor,
		ScopeStrategy:                  c.GetScopeStrategy(),
		SendDebugMessagesToClients:     c.SendOAuth2DebugMessagesToClients,
		EnforcePKCE:                    false,
		EnablePKCEPlainChallengeMethod: false,
	}

	return compose.Compose(
		fc,
		store,
		&compose.CommonStrategy{
			CoreStrategy:               compose.NewOAuth2HMACStrategy(fc, c.GetSystemSecret()),
			OpenIDConnectTokenStrategy: compose.NewOpenIDConnectStrategy(jwk.MustRSAPrivate(privateKey)),
		},
		nil,
		compose.OAuth2AuthorizeExplicitFactory,
		compose.OAuth2AuthorizeImplicitFactory,
		compose.OAuth2ClientCredentialsGrantFactory,
		compose.OAuth2RefreshTokenGrantFactory,
		compose.OAuth2PKCEFactory,
		compose.OpenIDConnectExplicitFactory,
		compose.OpenIDConnectHybridFactory,
		compose.OpenIDConnectImplicitFactory,
		compose.OpenIDConnectRefreshFactory,
		compose.OAuth2TokenRevocationFactory,
		compose.OAuth2TokenIntrospectionFactory,
	), publicKey.KeyID
}

func setDefaultConsentURL(s string, c *config.Config, path string) string {
	if s != "" {
		return s
	}
	proto := "https"
	if c.ForceHTTP {
		proto = "http"
	}
	host := "localhost"
	if c.BindHost != "" {
		host = c.BindHost
	}
	return fmt.Sprintf("%s://%s:%d/%s", proto, host, c.BindPort, path)
}

//func newOAuth2Handler(c *config.Config, router *httprouter.Router, cm oauth2.ConsentRequestManager, o fosite.OAuth2Provider, idTokenKeyID string) *oauth2.Handler {
func newOAuth2Handler(c *config.Config, router *httprouter.Router, cm consent.Manager, o fosite.OAuth2Provider, idTokenKeyID string) *oauth2.Handler {
	c.ConsentURL = setDefaultConsentURL(c.ConsentURL, c, "oauth2/fallbacks/consent")
	c.LoginURL = setDefaultConsentURL(c.LoginURL, c, "oauth2/fallbacks/consent")
	c.ErrorURL = setDefaultConsentURL(c.ErrorURL, c, "oauth2/fallbacks/error")

	errorURL, err := url.Parse(c.ErrorURL)
	pkg.Must(err, "Could not parse error url %s.", errorURL)

	privateKey, err := createOrGetJWK(c, oauth2.OpenIDConnectKeyName, "private")
	if err != nil {
		c.GetLogger().WithError(err).Fatalf(`Could not fetch private signing key for OpenID Connect - did you forget to run "hydra migrate sql" or forget to set the SYSTEM_SECRET?`)
	}

	jwtStrategy := compose.NewOpenIDConnectStrategy(jwk.MustRSAPrivate(privateKey))

	handler := &oauth2.Handler{
		ScopesSupported:  c.OpenIDDiscoveryScopesSupported,
		UserinfoEndpoint: c.OpenIDDiscoveryUserinfoEndpoint,
		ClaimsSupported:  c.OpenIDDiscoveryClaimsSupported,
		ForcedHTTP:       c.ForceHTTP,
		OAuth2:           o,
		ScopeStrategy:    c.GetScopeStrategy(),
		Consent: consent.NewStrategy(
			c.LoginURL, c.ConsentURL, c.Issuer,
			"/oauth2/auth", cm,
			sessions.NewCookieStore(c.GetCookieSecret()), c.GetScopeStrategy(),
			!c.ForceHTTP, time.Minute*15,
			jwtStrategy,
			openid.NewOpenIDConnectRequestValidator(nil, jwtStrategy),
		),
		Storage:             c.Context().FositeStore,
		ErrorURL:            *errorURL,
		H:                   herodot.NewJSONWriter(c.GetLogger()),
		AccessTokenLifespan: c.GetAccessTokenLifespan(),
		CookieStore:         sessions.NewCookieStore(c.GetCookieSecret()),
		Issuer:              c.Issuer,
		L:                   c.GetLogger(),
		IDTokenPublicKeyID:  idTokenKeyID,
		IDTokenLifespan:     c.GetIDTokenLifespan(),
	}

	handler.SetRoutes(router)
	return handler
}
<|endoftext|>