<commit_before>package services

import (
	"fmt"

	v1 "kubevirt.io/api/core/v1"

	"kubevirt.io/kubevirt/pkg/virt-controller/watch/topology"
)

type NodeSelectorRenderer struct {
	cpuFeatureLabels []string
	cpuModelLabel    string
	hasDedicatedCPU  bool
	hyperv           bool
	podNodeSelectors map[string]string
	tscFrequency     *int64
	vmiFeatures      *v1.Features
}

type NodeSelectorRendererOption func(renderer *NodeSelectorRenderer)

func NewNodeSelectorRenderer(
	vmiNodeSelectors map[string]string,
	clusterWideConfNodeSelectors map[string]string,
	opts ...NodeSelectorRendererOption,
) *NodeSelectorRenderer {
	podNodeSelectors := map[string]string{v1.NodeSchedulable: "true"}
	copySelectors(clusterWideConfNodeSelectors, podNodeSelectors)
	copySelectors(vmiNodeSelectors, podNodeSelectors)

	nodeSelectorRenderer := &NodeSelectorRenderer{podNodeSelectors: podNodeSelectors}
	for _, opt := range opts {
		opt(nodeSelectorRenderer)
	}
	return nodeSelectorRenderer
}

func (nsr *NodeSelectorRenderer) Render() map[string]string {
	if nsr.hasDedicatedCPU {
		nsr.enableSelectorLabel(v1.CPUManager)
	}
	if nsr.hyperv {
		copySelectors(hypervNodeSelectors(nsr.vmiFeatures), nsr.podNodeSelectors)
	}
	if nsr.cpuModelLabel != "" && nsr.cpuModelLabel != cpuModelLabel(v1.CPUModeHostModel) && nsr.cpuModelLabel != cpuModelLabel(v1.CPUModeHostPassthrough) {
		nsr.enableSelectorLabel(nsr.cpuModelLabel)
	}
	for _, cpuFeatureLabel := range nsr.cpuFeatureLabels {
		nsr.enableSelectorLabel(cpuFeatureLabel)
	}

	if nsr.isManualTSCFrequencyRequired() {
		nsr.enableSelectorLabel(topology.ToTSCSchedulableLabel(*nsr.tscFrequency))
	}

	return nsr.podNodeSelectors
}

func (nsr *NodeSelectorRenderer) enableSelectorLabel(label string) {
	nsr.podNodeSelectors[label] = "true"
}

func (nsr *NodeSelectorRenderer) isManualTSCFrequencyRequired() bool {
	return nsr.tscFrequency != nil
}

func WithDedicatedCPU() NodeSelectorRendererOption {
	return func(renderer *NodeSelectorRenderer) {
		renderer.hasDedicatedCPU = true
	}
}

func copySelectors(src map[string]string, dst map[string]string) {
	for k, v := range src {
		dst[k] = v
	}
}

func WithHyperv(features *v1.Features) NodeSelectorRendererOption {
	return func(renderer *NodeSelectorRenderer) {
		renderer.hyperv = true
		renderer.vmiFeatures = features
	}
}

func WithModelAndFeatureLabels(modelLabel string, cpuFeatureLabels ...string) NodeSelectorRendererOption {
	return func(renderer *NodeSelectorRenderer) {
		renderer.cpuFeatureLabels = cpuFeatureLabels
		renderer.cpuModelLabel = modelLabel
	}
}

func WithTSCTimer(tscFrequency *int64) NodeSelectorRendererOption {
	return func(renderer *NodeSelectorRenderer) {
		renderer.tscFrequency = tscFrequency
	}
}

func CPUModelLabelFromCPUModel(vmi *v1.VirtualMachineInstance) (label string, err error) {
	if vmi.Spec.Domain.CPU == nil || vmi.Spec.Domain.CPU.Model == "" {
		err = fmt.Errorf("Cannot create CPU Model label, vmi spec is mising CPU model")
		return
	}
	label = cpuModelLabel(vmi.Spec.Domain.CPU.Model)
	return
}

func cpuModelLabel(cpuModel string) string {
	return NFD_CPU_MODEL_PREFIX + cpuModel
}

func CPUFeatureLabelsFromCPUFeatures(vmi *v1.VirtualMachineInstance) []string {
	var labels []string
	if vmi.Spec.Domain.CPU != nil && vmi.Spec.Domain.CPU.Features != nil {
		for _, feature := range vmi.Spec.Domain.CPU.Features {
			if feature.Policy == "" || feature.Policy == "require" {
				labels = append(labels, NFD_CPU_FEATURE_PREFIX+feature.Name)
			}
		}
	}
	return labels
}

func hypervNodeSelectors(vmiFeatures *v1.Features) map[string]string {
	nodeSelectors := make(map[string]string)
	if vmiFeatures == nil || vmiFeatures.Hyperv == nil {
		return nodeSelectors
	}

	for _, hv := range makeHVFeatureLabelTable(vmiFeatures) {
		if isFeatureStateEnabled(hv.Feature) {
			nodeSelectors[NFD_KVM_INFO_PREFIX+hv.Label] = "true"
		}
	}

	if vmiFeatures.Hyperv.EVMCS != nil {
		nodeSelectors[v1.CPUModelVendorLabel+IntelVendorName] = "true"
	}

	return nodeSelectors
}

type hvFeatureLabel struct {
	Feature *v1.FeatureState
	Label   string
}

// makeHVFeatureLabelTable creates the mapping table between the VMI hyperv state and the label names.
// The table needs pointers to v1.FeatureHyperv struct, so it has to be generated and can't be a
// static var
func makeHVFeatureLabelTable(vmiFeatures *v1.Features) []hvFeatureLabel {
	// The following HyperV features don't require support from the host kernel, according to inspection
	// of the QEMU sources (4.0 - adb3321bfd)
	// VAPIC, Relaxed, Spinlocks, VendorID
	// VPIndex, SyNIC: depend on both MSR and capability
	// IPI, TLBFlush: depend on KVM Capabilities
	// Runtime, Reset, SyNICTimer, Frequencies, Reenlightenment: depend on KVM MSRs availability
	// EVMCS: depends on KVM capability, but the only way to know that is enable it, QEMU doesn't do
	// any check before that, so we leave it out
	//
	// see also https://schd.ws/hosted_files/devconfcz2019/cf/vkuznets_enlightening_kvm_devconf2019.pdf
	// to learn about dependencies between enlightenments

	hyperv := vmiFeatures.Hyperv // shortcut

	syNICTimer := &v1.FeatureState{}
	if hyperv.SyNICTimer != nil {
		syNICTimer.Enabled = hyperv.SyNICTimer.Enabled
	}

	return []hvFeatureLabel{
		{
			Feature: hyperv.VPIndex,
			Label:   "vpindex",
		},
		{
			Feature: hyperv.Runtime,
			Label:   "runtime",
		},
		{
			Feature: hyperv.Reset,
			Label:   "reset",
		},
		{
			// TODO: SyNIC depends on vp-index on QEMU level. We should enforce this constraint.
			Feature: hyperv.SyNIC,
			Label:   "synic",
		},
		{
			// TODO: SyNICTimer depends on SyNIC and Relaxed. We should enforce this constraint.
			Feature: syNICTimer,
			Label:   "synictimer",
		},
		{
			Feature: hyperv.Frequencies,
			Label:   "frequencies",
		},
		{
			Feature: hyperv.Reenlightenment,
			Label:   "reenlightenment",
		},
		{
			Feature: hyperv.TLBFlush,
			Label:   "tlbflush",
		},
		{
			Feature: hyperv.IPI,
			Label:   "ipi",
		},
	}
}
<commit_msg>Fix Evmcs Hyperv bug. Currently whenever EVMCS HyperV feature Is not nil in the vm spec than we require intel vendor. Even if we set to vmi.Spec.Domain.Features.Hyperv.EVMCS.Enabled to be false.<commit_after>package services

import (
	"fmt"

	v1 "kubevirt.io/api/core/v1"

	"kubevirt.io/kubevirt/pkg/virt-controller/watch/topology"
)

type NodeSelectorRenderer struct {
	cpuFeatureLabels []string
	cpuModelLabel    string
	hasDedicatedCPU  bool
	hyperv           bool
	podNodeSelectors map[string]string
	tscFrequency     *int64
	vmiFeatures      *v1.Features
}

type NodeSelectorRendererOption func(renderer *NodeSelectorRenderer)

func NewNodeSelectorRenderer(
	vmiNodeSelectors map[string]string,
	clusterWideConfNodeSelectors map[string]string,
	opts ...NodeSelectorRendererOption,
) *NodeSelectorRenderer {
	podNodeSelectors := map[string]string{v1.NodeSchedulable: "true"}
	copySelectors(clusterWideConfNodeSelectors, podNodeSelectors)
	copySelectors(vmiNodeSelectors, podNodeSelectors)

	nodeSelectorRenderer := &NodeSelectorRenderer{podNodeSelectors: podNodeSelectors}
	for _, opt := range opts {
		opt(nodeSelectorRenderer)
	}
	return nodeSelectorRenderer
}

func (nsr *NodeSelectorRenderer) Render() map[string]string {
	if nsr.hasDedicatedCPU {
		nsr.enableSelectorLabel(v1.CPUManager)
	}
	if nsr.hyperv {
		copySelectors(hypervNodeSelectors(nsr.vmiFeatures), nsr.podNodeSelectors)
	}
	if nsr.cpuModelLabel != "" && nsr.cpuModelLabel != cpuModelLabel(v1.CPUModeHostModel) && nsr.cpuModelLabel != cpuModelLabel(v1.CPUModeHostPassthrough) {
		nsr.enableSelectorLabel(nsr.cpuModelLabel)
	}
	for _, cpuFeatureLabel := range nsr.cpuFeatureLabels {
		nsr.enableSelectorLabel(cpuFeatureLabel)
	}

	if nsr.isManualTSCFrequencyRequired() {
		nsr.enableSelectorLabel(topology.ToTSCSchedulableLabel(*nsr.tscFrequency))
	}

	return nsr.podNodeSelectors
}

func (nsr *NodeSelectorRenderer) enableSelectorLabel(label string) {
	nsr.podNodeSelectors[label] = "true"
}

func (nsr *NodeSelectorRenderer) isManualTSCFrequencyRequired() bool {
	return nsr.tscFrequency != nil
}

func WithDedicatedCPU() NodeSelectorRendererOption {
	return func(renderer *NodeSelectorRenderer) {
		renderer.hasDedicatedCPU = true
	}
}

func copySelectors(src map[string]string, dst map[string]string) {
	for k, v := range src {
		dst[k] = v
	}
}

func WithHyperv(features *v1.Features) NodeSelectorRendererOption {
	return func(renderer *NodeSelectorRenderer) {
		renderer.hyperv = true
		renderer.vmiFeatures = features
	}
}

func WithModelAndFeatureLabels(modelLabel string, cpuFeatureLabels ...string) NodeSelectorRendererOption {
	return func(renderer *NodeSelectorRenderer) {
		renderer.cpuFeatureLabels = cpuFeatureLabels
		renderer.cpuModelLabel = modelLabel
	}
}

func WithTSCTimer(tscFrequency *int64) NodeSelectorRendererOption {
	return func(renderer *NodeSelectorRenderer) {
		renderer.tscFrequency = tscFrequency
	}
}

func CPUModelLabelFromCPUModel(vmi *v1.VirtualMachineInstance) (label string, err error) {
	if vmi.Spec.Domain.CPU == nil || vmi.Spec.Domain.CPU.Model == "" {
		err = fmt.Errorf("Cannot create CPU Model label, vmi spec is mising CPU model")
		return
	}
	label = cpuModelLabel(vmi.Spec.Domain.CPU.Model)
	return
}

func cpuModelLabel(cpuModel string) string {
	return NFD_CPU_MODEL_PREFIX + cpuModel
}

func CPUFeatureLabelsFromCPUFeatures(vmi *v1.VirtualMachineInstance) []string {
	var labels []string
	if vmi.Spec.Domain.CPU != nil && vmi.Spec.Domain.CPU.Features != nil {
		for _, feature := range vmi.Spec.Domain.CPU.Features {
			if feature.Policy == "" || feature.Policy == "require" {
				labels = append(labels, NFD_CPU_FEATURE_PREFIX+feature.Name)
			}
		}
	}
	return labels
}

func hypervNodeSelectors(vmiFeatures *v1.Features) map[string]string {
	nodeSelectors := make(map[string]string)
	if vmiFeatures == nil || vmiFeatures.Hyperv == nil {
		return nodeSelectors
	}

	for _, hv := range makeHVFeatureLabelTable(vmiFeatures) {
		if isFeatureStateEnabled(hv.Feature) {
			nodeSelectors[NFD_KVM_INFO_PREFIX+hv.Label] = "true"
		}
	}

	if vmiFeatures.Hyperv.EVMCS != nil && (vmiFeatures.Hyperv.EVMCS.Enabled == nil || (*vmiFeatures.Hyperv.EVMCS.Enabled) == true) {
		nodeSelectors[v1.CPUModelVendorLabel+IntelVendorName] = "true"
	}

	return nodeSelectors
}

type hvFeatureLabel struct {
	Feature *v1.FeatureState
	Label   string
}

// makeHVFeatureLabelTable creates the mapping table between the VMI hyperv state and the label names.
// The table needs pointers to v1.FeatureHyperv struct, so it has to be generated and can't be a
// static var
func makeHVFeatureLabelTable(vmiFeatures *v1.Features) []hvFeatureLabel {
	// The following HyperV features don't require support from the host kernel, according to inspection
	// of the QEMU sources (4.0 - adb3321bfd)
	// VAPIC, Relaxed, Spinlocks, VendorID
	// VPIndex, SyNIC: depend on both MSR and capability
	// IPI, TLBFlush: depend on KVM Capabilities
	// Runtime, Reset, SyNICTimer, Frequencies, Reenlightenment: depend on KVM MSRs availability
	// EVMCS: depends on KVM capability, but the only way to know that is enable it, QEMU doesn't do
	// any check before that, so we leave it out
	//
	// see also https://schd.ws/hosted_files/devconfcz2019/cf/vkuznets_enlightening_kvm_devconf2019.pdf
	// to learn about dependencies between enlightenments

	hyperv := vmiFeatures.Hyperv // shortcut

	syNICTimer := &v1.FeatureState{}
	if hyperv.SyNICTimer != nil {
		syNICTimer.Enabled = hyperv.SyNICTimer.Enabled
	}

	return []hvFeatureLabel{
		{
			Feature: hyperv.VPIndex,
			Label:   "vpindex",
		},
		{
			Feature: hyperv.Runtime,
			Label:   "runtime",
		},
		{
			Feature: hyperv.Reset,
			Label:   "reset",
		},
		{
			// TODO: SyNIC depends on vp-index on QEMU level. We should enforce this constraint.
			Feature: hyperv.SyNIC,
			Label:   "synic",
		},
		{
			// TODO: SyNICTimer depends on SyNIC and Relaxed. We should enforce this constraint.
			Feature: syNICTimer,
			Label:   "synictimer",
		},
		{
			Feature: hyperv.Frequencies,
			Label:   "frequencies",
		},
		{
			Feature: hyperv.Reenlightenment,
			Label:   "reenlightenment",
		},
		{
			Feature: hyperv.TLBFlush,
			Label:   "tlbflush",
		},
		{
			Feature: hyperv.IPI,
			Label:   "ipi",
		},
	}
}
<|endoftext|>


<commit_before>package server

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/google/go-github/github"
	"golang.org/x/net/context"

	datakit "github.com/docker/datakit/api/go"
)

type GithubHeaders struct {
	GitHubEvent    string
	GitHubDelivery string
	HubSignature   string
}

func (h *Server) PRDir(e github.PullRequestEvent) ([]string, error) {
	n := e.Number
	if n == nil {
		return nil, fmt.Errorf("Invalid PR number")
	}

	if e.Repo.Owner.Login == nil {
		return nil, fmt.Errorf("Empty user")
	}
	user := *e.Repo.Owner.Login

	if e.Repo.Name == nil {
		return nil, fmt.Errorf("Empty repo")
	}
	repo := *e.Repo.Name

	h.logger.Debugf("user=%s, repo=%s", user, repo)
	prDir := []string{user, repo, "prs", strconv.Itoa(*n)}
	return prDir, nil
}

func (h *Server) HandlePullRequestEvent(g GithubHeaders, e github.PullRequestEvent) error {

	// do nothing if the pull request's head didn't change
	if e.Action == nil {
		return nil
	}
	h.logger.Debugf("action: %s", *e.Action)
	switch *e.Action {
	case "opened":
	case "closed":
	case "reopened":
	case "synchronized":
	default:
		return nil
	}

	// maybe we want to move Dial in the parent function
	ctx := context.Background()
	client, err := datakit.Dial(ctx, h.proto, h.address)
	if err != nil {
		return err
	}

	// create a new transaction in the DB
	tr, err := datakit.NewTransaction(ctx, client, h.branch, h.branch+"-"+g.GitHubDelivery)
	if err != nil {
		return err
	}

	dir, err := h.PRDir(e)
	if err != nil {
		return err
	}

	if *e.Action == "closed" {
		// we only store open PRs
		tr.Remove(ctx, dir)
	} else {
		// the PR's head has changed, so store the new head
		head := e.PullRequest.Head.SHA
		if head == nil {
			return fmt.Errorf("PR %d has an invalid head", *e.Number)
		}
		tr.Write(ctx, append(dir, "head"), *head)
	}

	// commit the changes to the hook's branch
	err = tr.Commit(ctx, fmt.Sprintf("Pull request event: %s", g.GitHubDelivery))
	if err != nil {
		return err
	}

	client.Close(ctx)
	return nil
}

func (h *Server) CommitDir(e github.StatusEvent) ([]string, error) {

	if e.Repo.Owner.Login == nil {
		return nil, fmt.Errorf("Empty user")
	}
	user := *e.Repo.Owner.Login

	if e.Repo.Name == nil {
		return nil, fmt.Errorf("Empty repo")
	}
	repo := *e.Repo.Name

	if e.SHA == nil {
		return nil, fmt.Errorf("Empty SHA")
	}
	sha := *e.SHA

	h.logger.Debugf("user=%s, repo=%s", user, repo)
	commitDir := []string{user, repo, "commits", sha}
	return commitDir, nil
}

func (h *Server) HandleStatusEvent(g GithubHeaders, e github.StatusEvent) error {

	// maybe we want to move Dial in the parent function
	ctx := context.Background()
	client, err := datakit.Dial(ctx, h.proto, h.address)
	if err != nil {
		return err
	}

	// create a new transaction in the DB
	tr, err := datakit.NewTransaction(ctx, client, h.branch, h.branch+"-"+g.GitHubDelivery)
	if err != nil {
		return err
	}

	// project the event in the the filesystem
	dir, err := h.CommitDir(e)
	if err != nil {
		return err
	}

	// the status name, e.g. 'continuous-integration/appveyor/pr'
	context := e.Context
	if context == nil {
		return fmt.Errorf("Status without a valid context")
	}
	dir = append(dir, strings.Split(*context, "/")...)

	if e.Description != nil {
		tr.Write(ctx, append(dir, "description"), *e.Description)
	}
	if e.State != nil {
		tr.Write(ctx, append(dir, "state"), *e.State)
	}
	if e.TargetURL != nil {
		tr.Write(ctx, append(dir, "target_url"), *e.TargetURL)
	}

	// commit the changes to the hook's branch
	err = tr.Commit(ctx, fmt.Sprintf("Status event: %s", g.GitHubDelivery))
	if err != nil {
		return err
	}

	client.Close(ctx)
	return nil
}
<commit_msg>Fix typo in GitHub event handler<commit_after>package server

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/google/go-github/github"
	"golang.org/x/net/context"

	datakit "github.com/docker/datakit/api/go"
)

type GithubHeaders struct {
	GitHubEvent    string
	GitHubDelivery string
	HubSignature   string
}

func (h *Server) PRDir(e github.PullRequestEvent) ([]string, error) {
	n := e.Number
	if n == nil {
		return nil, fmt.Errorf("Invalid PR number")
	}

	if e.Repo.Owner.Login == nil {
		return nil, fmt.Errorf("Empty user")
	}
	user := *e.Repo.Owner.Login

	if e.Repo.Name == nil {
		return nil, fmt.Errorf("Empty repo")
	}
	repo := *e.Repo.Name

	h.logger.Debugf("user=%s, repo=%s", user, repo)
	prDir := []string{user, repo, "prs", strconv.Itoa(*n)}
	return prDir, nil
}

func (h *Server) HandlePullRequestEvent(g GithubHeaders, e github.PullRequestEvent) error {

	// do nothing if the pull request's head didn't change
	if e.Action == nil {
		return nil
	}
	h.logger.Debugf("action: %s", *e.Action)
	switch *e.Action {
	case "opened":
	case "closed":
	case "reopened":
	case "synchronize":
	default:
		return nil
	}

	// maybe we want to move Dial in the parent function
	ctx := context.Background()
	client, err := datakit.Dial(ctx, h.proto, h.address)
	if err != nil {
		return err
	}

	// create a new transaction in the DB
	tr, err := datakit.NewTransaction(ctx, client, h.branch, h.branch+"-"+g.GitHubDelivery)
	if err != nil {
		return err
	}

	dir, err := h.PRDir(e)
	if err != nil {
		return err
	}

	if *e.Action == "closed" {
		// we only store open PRs
		tr.Remove(ctx, dir)
	} else {
		// the PR's head has changed, so store the new head
		head := e.PullRequest.Head.SHA
		if head == nil {
			return fmt.Errorf("PR %d has an invalid head", *e.Number)
		}
		tr.Write(ctx, append(dir, "head"), *head)
	}

	// commit the changes to the hook's branch
	err = tr.Commit(ctx, fmt.Sprintf("Pull request event: %s", g.GitHubDelivery))
	if err != nil {
		return err
	}

	client.Close(ctx)
	return nil
}

func (h *Server) CommitDir(e github.StatusEvent) ([]string, error) {

	if e.Repo.Owner.Login == nil {
		return nil, fmt.Errorf("Empty user")
	}
	user := *e.Repo.Owner.Login

	if e.Repo.Name == nil {
		return nil, fmt.Errorf("Empty repo")
	}
	repo := *e.Repo.Name

	if e.SHA == nil {
		return nil, fmt.Errorf("Empty SHA")
	}
	sha := *e.SHA

	h.logger.Debugf("user=%s, repo=%s", user, repo)
	commitDir := []string{user, repo, "commits", sha}
	return commitDir, nil
}

func (h *Server) HandleStatusEvent(g GithubHeaders, e github.StatusEvent) error {

	// maybe we want to move Dial in the parent function
	ctx := context.Background()
	client, err := datakit.Dial(ctx, h.proto, h.address)
	if err != nil {
		return err
	}
	defer client.Close(ctx)

	// create a new transaction in the DB
	tr, err := datakit.NewTransaction(ctx, client, h.branch, h.branch+"-"+g.GitHubDelivery)
	if err != nil {
		return err
	}

	// project the event in the the filesystem
	dir, err := h.CommitDir(e)
	if err != nil {
		return err
	}

	// the status name, e.g. 'continuous-integration/appveyor/pr'
	context := e.Context
	if context == nil {
		return fmt.Errorf("Status without a valid context")
	}
	dir = append(dir, strings.Split(*context, "/")...)

	if e.Description != nil {
		tr.Write(ctx, append(dir, "description"), *e.Description)
	}
	if e.State != nil {
		tr.Write(ctx, append(dir, "state"), *e.State)
	}
	if e.TargetURL != nil {
		tr.Write(ctx, append(dir, "target_url"), *e.TargetURL)
	}

	// commit the changes to the hook's branch
	err = tr.Commit(ctx, fmt.Sprintf("Status event: %s", g.GitHubDelivery))
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package types

import (
	"fmt"

	"github.com/keel-hq/keel/util/image"
)

// Credentials - registry credentials
type Credentials struct {
	Username, Password string
}

// TrackedImage - tracked image data+metadata
type TrackedImage struct {
	Image        *image.Reference  `json:"image"`
	Trigger      TriggerType       `json:"trigger"`
	PollSchedule string            `json:"poll_schedule"`
	Provider     string            `json:"provider"`
	Namespace    string            `json:"namespace"`
	Secrets      []string          `json:"secrets"`
	Meta         map[string]string `json:"meta"` // metadata supplied by providers
	// a list of pre-release tags, ie: 1.0.0-dev, 1.5.0-prod get translated into
	// dev, prod
	// combined semver tags
	Tags   []string `json:"tags"`
	Policy Policy   `json:"policy"`
}

type Policy interface {
	ShouldUpdate(current, new string) (bool, error)
	Name() string
}

func (i TrackedImage) String() string {
	return fmt.Sprintf("namespace:%s,image:%s:%s,provider:%s,trigger:%s,sched:%s,secrets:%s", i.Namespace, i.Image.Repository(), i.Image.Tag(), i.Provider, i.Trigger, i.PollSchedule, i.Secrets)
}
<commit_msg>camel case<commit_after>package types

import (
	"fmt"

	"github.com/keel-hq/keel/util/image"
)

// Credentials - registry credentials
type Credentials struct {
	Username, Password string
}

// TrackedImage - tracked image data+metadata
type TrackedImage struct {
	Image        *image.Reference  `json:"image"`
	Trigger      TriggerType       `json:"trigger"`
	PollSchedule string            `json:"pollSchedule"`
	Provider     string            `json:"provider"`
	Namespace    string            `json:"namespace"`
	Secrets      []string          `json:"secrets"`
	Meta         map[string]string `json:"meta"` // metadata supplied by providers
	// a list of pre-release tags, ie: 1.0.0-dev, 1.5.0-prod get translated into
	// dev, prod
	// combined semver tags
	Tags   []string `json:"tags"`
	Policy Policy   `json:"policy"`
}

type Policy interface {
	ShouldUpdate(current, new string) (bool, error)
	Name() string
}

func (i TrackedImage) String() string {
	return fmt.Sprintf("namespace:%s,image:%s:%s,provider:%s,trigger:%s,sched:%s,secrets:%s", i.Namespace, i.Image.Repository(), i.Image.Tag(), i.Provider, i.Trigger, i.PollSchedule, i.Secrets)
}
<|endoftext|>


<commit_before>package cfinternal

import (
	"encoding/json"
	"strings"
	"time"

	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gexec"
)

type starter interface {
	Start(string, ...string) (*Session, error)
}

func ApiRequest(cmdStarter starter, method, endpoint string, response interface{}, timeout time.Duration, data ...string) {
	args := []string{
		"curl",
		endpoint,
		"-X", method,
	}

	dataArg := strings.Join(data, "")
	if len(dataArg) > 0 {
		args = append(args, "-d", dataArg)
	}

	request, err := cmdStarter.Start("cf", args...)
	Expect(err).NotTo(HaveOccurred())

	request.Wait(timeout)
	Expect(request).To(Exit(0))

	if response != nil {
		err := json.Unmarshal(request.Out.Contents(), response)
		Expect(err).ToNot(HaveOccurred())
	}
}
<commit_msg>Expectations in ApiRequest use ExpectWithOffset<commit_after>package cfinternal

import (
	"encoding/json"
	"strings"
	"time"

	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gexec"
)

type starter interface {
	Start(string, ...string) (*Session, error)
}

func ApiRequest(cmdStarter starter, method, endpoint string, response interface{}, timeout time.Duration, data ...string) {
	args := []string{
		"curl",
		endpoint,
		"-X", method,
	}

	dataArg := strings.Join(data, "")
	if len(dataArg) > 0 {
		args = append(args, "-d", dataArg)
	}

	request, err := cmdStarter.Start("cf", args...)
	ExpectWithOffset(2, err).NotTo(HaveOccurred())

	request.Wait(timeout)
	ExpectWithOffset(2, request).To(Exit(0))

	if response != nil {
		err := json.Unmarshal(request.Out.Contents(), response)
		ExpectWithOffset(2, err).ToNot(HaveOccurred())
	}
}
<|endoftext|>


<commit_before>package standard

import (
	"io"
	"net"
	"net/http"

	"github.com/webx-top/echo/engine"
	"github.com/webx-top/echo/logger"
)

type (
	Response struct {
		config    *engine.Config
		response  http.ResponseWriter
		request   *http.Request
		header    engine.Header
		status    int
		size      int64
		committed bool
		writer    io.Writer
		logger    logger.Logger
		body      []byte
	}
)

func NewResponse(w http.ResponseWriter, r *http.Request, l logger.Logger) *Response {
	return &Response{
		response: w,
		request:  r,
		header:   &Header{header: w.Header()},
		writer:   w,
		logger:   l,
	}
}

func (r *Response) Header() engine.Header {
	return r.header
}

func (r *Response) WriteHeader(code int) {
	if r.committed {
		r.logger.Warn("response already committed")
		return
	}
	r.status = code
	r.response.WriteHeader(code)
	r.committed = true
}

func (r *Response) Write(b []byte) (n int, err error) {
	r.body = b
	n, err = r.writer.Write(b)
	r.size += int64(n)
	return
}

func (r *Response) Status() int {
	return r.status
}

func (r *Response) Size() int64 {
	return r.size
}

func (r *Response) Committed() bool {
	return r.committed
}

func (r *Response) SetWriter(w io.Writer) {
	r.writer = w
}

func (r *Response) Writer() io.Writer {
	return r.writer
}

func (r *Response) Object() interface{} {
	return r.response
}

func (r *Response) Error(errMsg string, args ...int) {
	if len(args) > 0 {
		r.status = args[0]
	} else {
		r.status = http.StatusInternalServerError
	}
	r.response.Write(engine.Str2bytes(errMsg))
	r.response.WriteHeader(r.status)
}

func (r *Response) reset(w http.ResponseWriter, req *http.Request, h engine.Header) {
	r.response = w
	r.request = req
	r.header = h
	r.status = http.StatusOK
	r.size = 0
	r.committed = false
	r.writer = w
	r.body = nil
}

func (r *Response) Hijack(fn func(net.Conn)) {
	conn, bufrw, err := r.response.(http.Hijacker).Hijack()
	if err != nil {
		r.logger.Error(err)
	}
	_ = bufrw
	fn(conn)
	conn.Close()
}

func (r *Response) Body() []byte {
	return r.body
}

func (r *Response) Redirect(url string, code int) {
	http.Redirect(r.response, r.request, url, code)
}

func (r *Response) NotFound() {
	http.Error(r.response, http.StatusText(http.StatusNotFound), http.StatusNotFound)
}

func (r *Response) SetCookie(cookie *http.Cookie) {
	r.header.Set("Set-Cookie", cookie.String())
}

func (r *Response) ServeFile(file string) {
	http.ServeFile(r.response, r.request, file)
}

func (r *Response) StdResponseWriter() http.ResponseWriter {
	return r.response
}
<commit_msg>fixed bug<commit_after>package standard

import (
	"io"
	"net"
	"net/http"

	"github.com/webx-top/echo/engine"
	"github.com/webx-top/echo/logger"
)

type (
	Response struct {
		config    *engine.Config
		response  http.ResponseWriter
		request   *http.Request
		header    engine.Header
		status    int
		size      int64
		committed bool
		writer    io.Writer
		logger    logger.Logger
		body      []byte
	}
)

func NewResponse(w http.ResponseWriter, r *http.Request, l logger.Logger) *Response {
	return &Response{
		response: w,
		request:  r,
		header:   &Header{header: w.Header()},
		writer:   w,
		logger:   l,
	}
}

func (r *Response) Header() engine.Header {
	return r.header
}

func (r *Response) WriteHeader(code int) {
	if r.committed {
		r.logger.Warn("response already committed")
		return
	}
	r.status = code
	r.response.WriteHeader(code)
	r.committed = true
}

func (r *Response) Write(b []byte) (n int, err error) {
	r.body = b
	n, err = r.writer.Write(b)
	r.size += int64(n)
	return
}

func (r *Response) Status() int {
	return r.status
}

func (r *Response) Size() int64 {
	return r.size
}

func (r *Response) Committed() bool {
	return r.committed
}

func (r *Response) SetWriter(w io.Writer) {
	r.writer = w
}

func (r *Response) Writer() io.Writer {
	return r.writer
}

func (r *Response) Object() interface{} {
	return r.response
}

func (r *Response) Error(errMsg string, args ...int) {
	if len(args) > 0 {
		r.status = args[0]
	} else {
		r.status = http.StatusInternalServerError
	}
	r.Write(engine.Str2bytes(errMsg))
	r.WriteHeader(r.status)
}

func (r *Response) reset(w http.ResponseWriter, req *http.Request, h engine.Header) {
	r.response = w
	r.request = req
	r.header = h
	r.status = http.StatusOK
	r.size = 0
	r.committed = false
	r.writer = w
	r.body = nil
}

func (r *Response) Hijack(fn func(net.Conn)) {
	conn, bufrw, err := r.response.(http.Hijacker).Hijack()
	if err != nil {
		r.logger.Error(err)
	}
	_ = bufrw
	fn(conn)
	conn.Close()
}

func (r *Response) Body() []byte {
	return r.body
}

func (r *Response) Redirect(url string, code int) {
	http.Redirect(r.response, r.request, url, code)
}

func (r *Response) NotFound() {
	http.Error(r.response, http.StatusText(http.StatusNotFound), http.StatusNotFound)
}

func (r *Response) SetCookie(cookie *http.Cookie) {
	r.header.Set("Set-Cookie", cookie.String())
}

func (r *Response) ServeFile(file string) {
	http.ServeFile(r.response, r.request, file)
}

func (r *Response) StdResponseWriter() http.ResponseWriter {
	return r.response
}
<|endoftext|>


<commit_before>package schema2struct

import (
	"fmt"
	"string"

	"github.com/xitongsys/parquet-go/parquet"
)

func ParquetTypeToParquetTypeStr(pT *parquet.ParquetType, cT *parquet.ConvertedType) (string, string) {
	var pTStr, cTStr string
	switch *pT {
	case parquet.Type_BOOLEAN:
		pTStr = "BOOLEAN"
	case parquet.Type_INT32:
		pTStr = "INT32"
	case parquet.Type_INT64:
		pTStr = "INT64"
	case parquet.Type_INT96:
		pTStr = "INT96"
	case parquet.Type_FLOAT:
		pTStr = "FLOAT"
	case parquet.Type_DOUBLE:
		pTStr = "DOUBLE"
	case parquet.Type_BYTE_ARRAY:
		pTStr = "BYTE_ARRAY"
	case parquet.Type_FIXED_LEN_BYTE_ARRAY:
		pTStr = "FIXED_LEN_BYTE_ARRAY"
	}
	switch *cT {
	case parquet.ConvertedType_UTF8:
		cTStr = "UTF8"
	case parquet.ConvertedType_INT_8:
		cTStr = "INT_8"
	case parquet.ConvertedType_INT_16:
		cTStr = "INT_16"
	case parquet.ConvertedType_INT_32:
		cTStr = "INT_32"
	case parquet.ConvertedType_INT_64:
		cTStr = "INT_64"
	case parquet.ConvertedType_UINT_8:
		cTStr = "UINT_8"
	case parquet.ConvertedType_UINT_16:
		cTStr = "UINT_16"
	case parquet.ConvertedType_UINT_32:
		cTStr = "UINT_32"
	case parquet.ConvertedType_UINT_64:
		cTStr = "UINT_64"
	case parquet.ConvertedType_DATE:
		cTStr = "DATE"
	case parquet.ConvertedType_TIME_MILLIS:
		cTStr = "TIME_MILLIS"
	case parquet.ConvertedType_TIME_MICROS:
		cTStr = "TIME_MICROS"
	case parquet.ConvertedType_TIMESTAMP_MILLIS:
		cTStr = "TIMESTAMP_MILLIS"
	case parquet.ConvertedType_TIMESTAMP_MICROS:
		cTStr = "TIMESTAMP_MICROS"
	case parquet.ConvertedType_INTERVAL:
		cTStr = "INTERVAL"
	case parquet.ConvertedType_DECIMAL:
		cTStr = "DECIMAL"
	case parquet.ConvertedType_MAP:
		cTStr = "MAP"
	case parquet.ConvertedType_LIST:
		cTStr = "LIST"
	}
	return pTStr, cTStr
}

func ParquetTypeToGoTypeStr(pT *parquet.ParquetType, cT *parquet.ConvertedType) string {
	res := ""
	switch *pT {
	case parquet.Type_BOOLEAN:
		res = "bool"
	case parquet.Type_INT32:
		res = "int32"
	case parquet.Type_INT64:
		res = "int64"
	case parquet.Type_INT96:
		res = "string"
	case parquet.Type_FLOAT:
		res = "float32"
	case parquet.Type_DOUBLE:
		res = "float64"
	case parquet.Type_BYTE_ARRAY:
		res = "string"
	case parquet.Type_FIXED_LEN_BYTE_ARRAY:
		res = "string"
	}
	if cT != nil {
		switch *cT {
		case parquet.ConvertedType_UINT_8:
			res = "uint32"
		case parquet.ConvertedType_UINT_16:
			res = "uint32"
		case parquet.ConvertedType_UINT_32:
			res = "uint32"
		case parquet.ConvertedType_UINT_64:
			res = "uint64"
		}
	}
}

type Node struct {
	SE       *parquet.SchemaElement
	Children []*Node
}

func NewNode(schema *parquet.SchemaElement) *Node {
	node := &(Node{
		SE:       schema,
		Children: []*Node{},
	})
	return node
}

func (self *Node) OutputJsonSchema() string {
	return ""
}

func (self *Node) OutputStruct() string {
	name := self.SE.GetName()
	res := name
	pT, cT := self.SE.GetType(), self.SE.GetConvertedType()
	pTStr, cTStr := ParquetTypeToParquetTypeStr(pT, cT)
	rTStr := ""
	if self.SE.GetRepetitionType == parquet.FieldRepetitionType_OPTIONAL {
		rTStr = "*"
	} else if self.SE.GetConvertedType == parquet.FieldRepetitionType_REPEATED {
		rTStr = "[]"
	}

	if pT == nil && cT == nil {
		res += " struct {\n"
		for _, cNode := range self.Children {
			res += "\t" + cNode.OutputStruct() + "\n"
		}
		res += "}"

	} else if cT != nil && *cT == parquet.ConvertedType_MAP {
		keyNode := self.Children[0].Children[0]
		keyPT, keyCT := keyNode.SE.GetType(), keyNode.SE.GetConvertedType()
		keyGoTypeStr := ParquetTypeToGoTypeStr(pT, cT)
		valNode := self.Children[0].Children[1]
		res += " " + "map[" + keyGoTypeStr + "]" + valNode.OutputStruct()

	} else if cT != nil && *cT == parquet.ConvertedType_LIST {
		cNode := self.Children[0].Children[0]
		res += " " + rTStr + " " + cNode.OutputStruct()

	} else {
		goTypeStr := ParquetTypeToGoTypeStr(pT, cT)
		res += " " + rTStr + " " + goTypeStr
	}
	return res

}

func CreateTree(schemas []*parquet.SchemaElement) *Node {
	ln := len(schemas)
	pos := 0
	stack := make([]*Node, 0)
	root := NewNode(schemas[0])
	stack = append(stack, root)
	pos++

	for len(stack) > 0 {
		node := stack[len(stack)-1]
		numChildren := int(node.SE.GetNumChildren())
		lc := len(node.Children)
		if lc < numChildren {
			newNode := NewNode(schemas[pos])
			node.Children = append(node.Children, newNode)
			stack = append(stack, newNode)
			pos++
		} else {
			stack = stack[:len(stack)-1]
		}
	}
	return root

}
<commit_msg>adding pt<commit_after>package SchemaOutput

import (
	"fmt"
	"string"

	"github.com/xitongsys/parquet-go/parquet"
)

func ParquetTypeToParquetTypeStr(pT *parquet.ParquetType, cT *parquet.ConvertedType) (string, string) {
	var pTStr, cTStr string
	switch *pT {
	case parquet.Type_BOOLEAN:
		pTStr = "BOOLEAN"
	case parquet.Type_INT32:
		pTStr = "INT32"
	case parquet.Type_INT64:
		pTStr = "INT64"
	case parquet.Type_INT96:
		pTStr = "INT96"
	case parquet.Type_FLOAT:
		pTStr = "FLOAT"
	case parquet.Type_DOUBLE:
		pTStr = "DOUBLE"
	case parquet.Type_BYTE_ARRAY:
		pTStr = "BYTE_ARRAY"
	case parquet.Type_FIXED_LEN_BYTE_ARRAY:
		pTStr = "FIXED_LEN_BYTE_ARRAY"
	}
	switch *cT {
	case parquet.ConvertedType_UTF8:
		cTStr = "UTF8"
	case parquet.ConvertedType_INT_8:
		cTStr = "INT_8"
	case parquet.ConvertedType_INT_16:
		cTStr = "INT_16"
	case parquet.ConvertedType_INT_32:
		cTStr = "INT_32"
	case parquet.ConvertedType_INT_64:
		cTStr = "INT_64"
	case parquet.ConvertedType_UINT_8:
		cTStr = "UINT_8"
	case parquet.ConvertedType_UINT_16:
		cTStr = "UINT_16"
	case parquet.ConvertedType_UINT_32:
		cTStr = "UINT_32"
	case parquet.ConvertedType_UINT_64:
		cTStr = "UINT_64"
	case parquet.ConvertedType_DATE:
		cTStr = "DATE"
	case parquet.ConvertedType_TIME_MILLIS:
		cTStr = "TIME_MILLIS"
	case parquet.ConvertedType_TIME_MICROS:
		cTStr = "TIME_MICROS"
	case parquet.ConvertedType_TIMESTAMP_MILLIS:
		cTStr = "TIMESTAMP_MILLIS"
	case parquet.ConvertedType_TIMESTAMP_MICROS:
		cTStr = "TIMESTAMP_MICROS"
	case parquet.ConvertedType_INTERVAL:
		cTStr = "INTERVAL"
	case parquet.ConvertedType_DECIMAL:
		cTStr = "DECIMAL"
	case parquet.ConvertedType_MAP:
		cTStr = "MAP"
	case parquet.ConvertedType_LIST:
		cTStr = "LIST"
	}
	return pTStr, cTStr
}

func ParquetTypeToGoTypeStr(pT *parquet.ParquetType, cT *parquet.ConvertedType) string {
	res := ""
	switch *pT {
	case parquet.Type_BOOLEAN:
		res = "bool"
	case parquet.Type_INT32:
		res = "int32"
	case parquet.Type_INT64:
		res = "int64"
	case parquet.Type_INT96:
		res = "string"
	case parquet.Type_FLOAT:
		res = "float32"
	case parquet.Type_DOUBLE:
		res = "float64"
	case parquet.Type_BYTE_ARRAY:
		res = "string"
	case parquet.Type_FIXED_LEN_BYTE_ARRAY:
		res = "string"
	}
	if cT != nil {
		switch *cT {
		case parquet.ConvertedType_UINT_8:
			res = "uint32"
		case parquet.ConvertedType_UINT_16:
			res = "uint32"
		case parquet.ConvertedType_UINT_32:
			res = "uint32"
		case parquet.ConvertedType_UINT_64:
			res = "uint64"
		}
	}
}

type Node struct {
	SE       *parquet.SchemaElement
	Children []*Node
}

func NewNode(schema *parquet.SchemaElement) *Node {
	node := &(Node{
		SE:       schema,
		Children: []*Node{},
	})
	return node
}

func (self *Node) OutputJsonSchema() string {
	res := "{\"Tag\":"
	pT, cT := self.SE.GetType(), self.SE.GetConvertedType()
	rTStr := "REQUIRED"
	if self.SE.GetRepetitionType() == parquet.FieldRepetitionType_OPTIONAL {
		rTStr = "OPTIONAL"
	} else if self.SE.GetRepetitionType() == parquet.FieldRepetitionType_REPEATED {
		rTStr = "REPEATED"
	}

	pTStr, cTStr := ParquetTypeToParquetTypeStr(pT, cT)
	tagStr := "\"name=%s, type=%s, repetitiontype=%s\""
	name := self.SE.GetName()

	if len(self.Children) == 0 {
		if *pT == parquet.Type_FIXED_LEN_BYTE_ARRAY && cT == nil {
			length := self.SE.GetTypeLength()
			tagStr = "\"name=%s, type=%s, length=%d, repetitiontype=%s\""
			res += fmt.Sprintf(tagStr, name, pTStr, length, rTStr) + "}"

		} else if *cT == parquet.ConvertedType_DECIMAL {
			scale, precision := self.SE.GetScale(), self.SE.GetPrecision()
			if *pT == parquet.Type_FIXED_LEN_BYTE_ARRAY {
				length := self.SE.GetTypeLength()
				tagStr = "\"name=%s, type=%s, basetype=%s, scale=%d, precision=%s, length=%d, repetitiontype=%s\""
				res += fmt.Sprintf(tagStr, name, cTStr, pTStr, scale, precision, length, rTStr) + "}"
			} else {
				tagStr = "\"name=%s, type=%s, basetype=%s, scale=%d, precision=%s, repetitiontype\""
				res += fmt.Sprintf(tagStr, name, cTStr, pTStr, scale, precision, rTStr) + "}"
			}

		} else {
			typeStr := pTStr
			if cT != nil {
				typeStr = pTStr
			}
			res += fmt.Sprintf(tagStr, name, typeStr, rTStr) + "}"

		}
	} else {

		if cT != nil {
			tagStr = "\"name=%s, type=%s\""
			res += fmt.Sprintf(tagStr, name, cTStr)
		} else {
			tagStr = "\"name=%s\""
			res += fmt.Sprintf(tagStr, name)
		}
		res += ",\n"
		res += "\"Fields\":[\n"
		for _, cNode := range self.Children {
			res += "\t" + cNode.OutputJsonSchema() + "\n"
		}

		res += "]\n"
		res += "}"

	}
	return res
}

func (self *Node) OutputStruct() string {
	name := self.SE.GetName()
	res := name
	pT, cT := self.SE.GetType(), self.SE.GetConvertedType()
	rTStr := ""
	if self.SE.GetRepetitionType == parquet.FieldRepetitionType_OPTIONAL {
		rTStr = "*"
	} else if self.SE.GetConvertedType == parquet.FieldRepetitionType_REPEATED {
		rTStr = "[]"
	}

	if pT == nil && cT == nil {
		res += " struct {\n"
		for _, cNode := range self.Children {
			res += "\t" + cNode.OutputStruct() + "\n"
		}
		res += "}"

	} else if cT != nil && *cT == parquet.ConvertedType_MAP {
		keyNode := self.Children[0].Children[0]
		keyPT, keyCT := keyNode.SE.GetType(), keyNode.SE.GetConvertedType()
		keyGoTypeStr := ParquetTypeToGoTypeStr(pT, cT)
		valNode := self.Children[0].Children[1]
		res += " " + "map[" + keyGoTypeStr + "]" + valNode.OutputStruct()

	} else if cT != nil && *cT == parquet.ConvertedType_LIST {
		cNode := self.Children[0].Children[0]
		res += " " + rTStr + " " + cNode.OutputStruct()

	} else {
		goTypeStr := ParquetTypeToGoTypeStr(pT, cT)
		res += " " + rTStr + " " + goTypeStr
	}
	return res

}

func CreateTree(schemas []*parquet.SchemaElement) *Node {
	ln := len(schemas)
	pos := 0
	stack := make([]*Node, 0)
	root := NewNode(schemas[0])
	stack = append(stack, root)
	pos++

	for len(stack) > 0 {
		node := stack[len(stack)-1]
		numChildren := int(node.SE.GetNumChildren())
		lc := len(node.Children)
		if lc < numChildren {
			newNode := NewNode(schemas[pos])
			node.Children = append(node.Children, newNode)
			stack = append(stack, newNode)
			pos++
		} else {
			stack = stack[:len(stack)-1]
		}
	}
	return root

}
<|endoftext|>


<commit_before>package ns1

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strings"

	"gopkg.in/ns1/ns1-go.v2/rest"
	"gopkg.in/ns1/ns1-go.v2/rest/model/dns"
	"gopkg.in/ns1/ns1-go.v2/rest/model/filter"

	"github.com/StackExchange/dnscontrol/v3/models"
	"github.com/StackExchange/dnscontrol/v3/pkg/diff"
	"github.com/StackExchange/dnscontrol/v3/providers"
)

var docNotes = providers.DocumentationNotes{
	providers.CanUseAlias:            providers.Can(),
	providers.CanUseCAA:              providers.Can(),
	providers.CanUsePTR:              providers.Can(),
	providers.DocCreateDomains:       providers.Cannot(),
	providers.DocDualHost:            providers.Can(),
	providers.DocOfficiallySupported: providers.Cannot(),
}

func init() {
	fns := providers.DspFuncs{
		Initializer:   newProvider,
		RecordAuditor: AuditRecords,
	}
	providers.RegisterDomainServiceProviderType("NS1", fns, providers.CanUseSRV, docNotes)
	providers.RegisterCustomRecordType("NS1_URLFWD", "NS1", "URLFWD")
}

type nsone struct {
	*rest.Client
}

func newProvider(creds map[string]string, meta json.RawMessage) (providers.DNSServiceProvider, error) {
	if creds["api_token"] == "" {
		return nil, fmt.Errorf("api_token required for ns1")
	}
	return &nsone{rest.NewClient(http.DefaultClient, rest.SetAPIKey(creds["api_token"]))}, nil
}

func (n *nsone) GetNameservers(domain string) ([]*models.Nameserver, error) {
	z, _, err := n.Zones.Get(domain)
	if err != nil {
		return nil, err
	}
	return models.ToNameservers(z.DNSServers)
}

// GetZoneRecords gets the records of a zone and returns them in RecordConfig format.
func (n *nsone) GetZoneRecords(domain string) (models.Records, error) {
	return nil, fmt.Errorf("not implemented")
	// This enables the get-zones subcommand.
	// Implement this by extracting the code from GetDomainCorrections into
	// a single function.  For most providers this should be relatively easy.
}

func (n *nsone) GetDomainCorrections(dc *models.DomainConfig) ([]*models.Correction, error) {
	dc.Punycode()
	//dc.CombineMXs()
	z, _, err := n.Zones.Get(dc.Name)
	if err != nil {
		return nil, err
	}

	found := models.Records{}
	for _, r := range z.Records {
		zrs, err := convert(r, dc.Name)
		if err != nil {
			return nil, err
		}
		found = append(found, zrs...)
	}
	foundGrouped := found.GroupedByKey()
	desiredGrouped := dc.Records.GroupedByKey()

	//  Normalize
	models.PostProcessRecords(found)

	differ := diff.New(dc)
	changedGroups, err := differ.ChangedGroups(found)
	if err != nil {
		return nil, err
	}
	corrections := []*models.Correction{}
	// each name/type is given to the api as a unit.
	for k, descs := range changedGroups {
		key := k

		desc := strings.Join(descs, "\n")
		_, current := foundGrouped[k]
		recs, wanted := desiredGrouped[k]
		if wanted && !current {
			// pure addition
			corrections = append(corrections, &models.Correction{
				Msg: desc,
				F:   func() error { return n.add(recs, dc.Name) },
			})
		} else if current && !wanted {
			// pure deletion
			corrections = append(corrections, &models.Correction{
				Msg: desc,
				F:   func() error { return n.remove(key, dc.Name) },
			})
		} else {
			// modification
			corrections = append(corrections, &models.Correction{
				Msg: desc,
				F:   func() error { return n.modify(recs, dc.Name) },
			})
		}
	}
	return corrections, nil
}

func (n *nsone) add(recs models.Records, domain string) error {
	_, err := n.Records.Create(buildRecord(recs, domain, ""))
	return err
}

func (n *nsone) remove(key models.RecordKey, domain string) error {
	_, err := n.Records.Delete(domain, key.NameFQDN, key.Type)
	return err
}

func (n *nsone) modify(recs models.Records, domain string) error {
	_, err := n.Records.Update(buildRecord(recs, domain, ""))
	return err
}

func buildRecord(recs models.Records, domain string, id string) *dns.Record {
	r := recs[0]
	rec := &dns.Record{
		Domain:  r.GetLabelFQDN(),
		Type:    r.Type,
		ID:      id,
		TTL:     int(r.TTL),
		Zone:    domain,
		Filters: []*filter.Filter{}, // Work through a bug in the NS1 API library that causes 400 Input validation failed (Value None for field '<obj>.filters' is not of type array)
	}
	for _, r := range recs {
		if r.Type == "MX" {
			rec.AddAnswer(&dns.Answer{Rdata: strings.Split(fmt.Sprintf("%d %v", r.MxPreference, r.GetTargetField()), " ")})
		} else if r.Type == "TXT" {
			rec.AddAnswer(&dns.Answer{Rdata: r.TxtStrings})
		} else if r.Type == "CAA" {
			rec.AddAnswer(&dns.Answer{Rdata: strings.Split(fmt.Sprintf("%v %s %s", r.CaaFlag, r.CaaTag, r.GetTargetField()), " ")})
		} else if r.Type == "SRV" {
			rec.AddAnswer(&dns.Answer{Rdata: strings.Split(fmt.Sprintf("%d %d %d %v", r.SrvPriority, r.SrvWeight, r.SrvPort, r.GetTargetField()), " ")})
		} else {
			rec.AddAnswer(&dns.Answer{Rdata: strings.Split(r.GetTargetField(), " ")})
		}
	}
	return rec
}

func convert(zr *dns.ZoneRecord, domain string) ([]*models.RecordConfig, error) {
	found := []*models.RecordConfig{}
	for _, ans := range zr.ShortAns {
		rec := &models.RecordConfig{
			TTL:      uint32(zr.TTL),
			Original: zr,
		}
		rec.SetLabelFromFQDN(zr.Domain, domain)
		switch rtype := zr.Type; rtype {
		case "ALIAS":
			rec.Type = rtype
			if err := rec.SetTarget(ans); err != nil {
				panic(fmt.Errorf("unparsable %s record received from ns1: %w", rtype, err))
			}
		case "URLFWD":
			rec.Type = rtype
			if err := rec.SetTarget(ans); err != nil {
				panic(fmt.Errorf("unparsable %s record received from ns1: %w", rtype, err))
			}
		default:
			if err := rec.PopulateFromString(rtype, ans, domain); err != nil {
				panic(fmt.Errorf("unparsable record received from ns1: %w", err))
			}
		}
		found = append(found, rec)
	}
	return found, nil
}
<commit_msg>ns1: Implement EnsureDomainExists (#1157)<commit_after>package ns1

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strings"

	"gopkg.in/ns1/ns1-go.v2/rest"
	"gopkg.in/ns1/ns1-go.v2/rest/model/dns"
	"gopkg.in/ns1/ns1-go.v2/rest/model/filter"

	"github.com/StackExchange/dnscontrol/v3/models"
	"github.com/StackExchange/dnscontrol/v3/pkg/diff"
	"github.com/StackExchange/dnscontrol/v3/providers"
)

var docNotes = providers.DocumentationNotes{
	providers.CanUseAlias:            providers.Can(),
	providers.CanUseCAA:              providers.Can(),
	providers.CanUsePTR:              providers.Can(),
	providers.DocCreateDomains:       providers.Can(),
	providers.DocDualHost:            providers.Can(),
	providers.DocOfficiallySupported: providers.Cannot(),
}

func init() {
	fns := providers.DspFuncs{
		Initializer:   newProvider,
		RecordAuditor: AuditRecords,
	}
	providers.RegisterDomainServiceProviderType("NS1", fns, providers.CanUseSRV, docNotes)
	providers.RegisterCustomRecordType("NS1_URLFWD", "NS1", "URLFWD")
}

type nsone struct {
	*rest.Client
}

func newProvider(creds map[string]string, meta json.RawMessage) (providers.DNSServiceProvider, error) {
	if creds["api_token"] == "" {
		return nil, fmt.Errorf("api_token required for ns1")
	}
	return &nsone{rest.NewClient(http.DefaultClient, rest.SetAPIKey(creds["api_token"]))}, nil
}

func (n *nsone) EnsureDomainExists(domain string) error {
	// This enables the create-domains subcommand

	zone := dns.NewZone(domain)
	_, err := n.Zones.Create(zone)

	if err == rest.ErrZoneExists {
		// if domain exists already, just return nil, nothing to do here.
		return nil
	}

	return err
}

func (n *nsone) GetNameservers(domain string) ([]*models.Nameserver, error) {
	z, _, err := n.Zones.Get(domain)
	if err != nil {
		return nil, err
	}
	return models.ToNameservers(z.DNSServers)
}

// GetZoneRecords gets the records of a zone and returns them in RecordConfig format.
func (n *nsone) GetZoneRecords(domain string) (models.Records, error) {
	return nil, fmt.Errorf("not implemented")
	// This enables the get-zones subcommand.
	// Implement this by extracting the code from GetDomainCorrections into
	// a single function.  For most providers this should be relatively easy.
}

func (n *nsone) GetDomainCorrections(dc *models.DomainConfig) ([]*models.Correction, error) {
	dc.Punycode()
	//dc.CombineMXs()
	z, _, err := n.Zones.Get(dc.Name)
	if err != nil {
		return nil, err
	}

	found := models.Records{}
	for _, r := range z.Records {
		zrs, err := convert(r, dc.Name)
		if err != nil {
			return nil, err
		}
		found = append(found, zrs...)
	}
	foundGrouped := found.GroupedByKey()
	desiredGrouped := dc.Records.GroupedByKey()

	//  Normalize
	models.PostProcessRecords(found)

	differ := diff.New(dc)
	changedGroups, err := differ.ChangedGroups(found)
	if err != nil {
		return nil, err
	}
	corrections := []*models.Correction{}
	// each name/type is given to the api as a unit.
	for k, descs := range changedGroups {
		key := k

		desc := strings.Join(descs, "\n")
		_, current := foundGrouped[k]
		recs, wanted := desiredGrouped[k]
		if wanted && !current {
			// pure addition
			corrections = append(corrections, &models.Correction{
				Msg: desc,
				F:   func() error { return n.add(recs, dc.Name) },
			})
		} else if current && !wanted {
			// pure deletion
			corrections = append(corrections, &models.Correction{
				Msg: desc,
				F:   func() error { return n.remove(key, dc.Name) },
			})
		} else {
			// modification
			corrections = append(corrections, &models.Correction{
				Msg: desc,
				F:   func() error { return n.modify(recs, dc.Name) },
			})
		}
	}
	return corrections, nil
}

func (n *nsone) add(recs models.Records, domain string) error {
	_, err := n.Records.Create(buildRecord(recs, domain, ""))
	return err
}

func (n *nsone) remove(key models.RecordKey, domain string) error {
	_, err := n.Records.Delete(domain, key.NameFQDN, key.Type)
	return err
}

func (n *nsone) modify(recs models.Records, domain string) error {
	_, err := n.Records.Update(buildRecord(recs, domain, ""))
	return err
}

func buildRecord(recs models.Records, domain string, id string) *dns.Record {
	r := recs[0]
	rec := &dns.Record{
		Domain:  r.GetLabelFQDN(),
		Type:    r.Type,
		ID:      id,
		TTL:     int(r.TTL),
		Zone:    domain,
		Filters: []*filter.Filter{}, // Work through a bug in the NS1 API library that causes 400 Input validation failed (Value None for field '<obj>.filters' is not of type array)
	}
	for _, r := range recs {
		if r.Type == "MX" {
			rec.AddAnswer(&dns.Answer{Rdata: strings.Split(fmt.Sprintf("%d %v", r.MxPreference, r.GetTargetField()), " ")})
		} else if r.Type == "TXT" {
			rec.AddAnswer(&dns.Answer{Rdata: r.TxtStrings})
		} else if r.Type == "CAA" {
			rec.AddAnswer(&dns.Answer{Rdata: strings.Split(fmt.Sprintf("%v %s %s", r.CaaFlag, r.CaaTag, r.GetTargetField()), " ")})
		} else if r.Type == "SRV" {
			rec.AddAnswer(&dns.Answer{Rdata: strings.Split(fmt.Sprintf("%d %d %d %v", r.SrvPriority, r.SrvWeight, r.SrvPort, r.GetTargetField()), " ")})
		} else {
			rec.AddAnswer(&dns.Answer{Rdata: strings.Split(r.GetTargetField(), " ")})
		}
	}
	return rec
}

func convert(zr *dns.ZoneRecord, domain string) ([]*models.RecordConfig, error) {
	found := []*models.RecordConfig{}
	for _, ans := range zr.ShortAns {
		rec := &models.RecordConfig{
			TTL:      uint32(zr.TTL),
			Original: zr,
		}
		rec.SetLabelFromFQDN(zr.Domain, domain)
		switch rtype := zr.Type; rtype {
		case "ALIAS":
			rec.Type = rtype
			if err := rec.SetTarget(ans); err != nil {
				panic(fmt.Errorf("unparsable %s record received from ns1: %w", rtype, err))
			}
		case "URLFWD":
			rec.Type = rtype
			if err := rec.SetTarget(ans); err != nil {
				panic(fmt.Errorf("unparsable %s record received from ns1: %w", rtype, err))
			}
		default:
			if err := rec.PopulateFromString(rtype, ans, domain); err != nil {
				panic(fmt.Errorf("unparsable record received from ns1: %w", err))
			}
		}
		found = append(found, rec)
	}
	return found, nil
}
<|endoftext|>


<commit_before>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
)

// AddNodeHandler calls cluster.Register registering a node into it.
func AddNodeHandler(w http.ResponseWriter, r *http.Request) error {
	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return err
	}
	params := map[string]string{}
	err = json.Unmarshal(b, &params)
	if err != nil {
		return err
	}
	var scheduler segregatedScheduler
	return scheduler.Register(params)
}

func RemoveNodeHandler(w http.ResponseWriter, r *http.Request) error {
	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return err
	}
	params := map[string]string{}
	err = json.Unmarshal(b, &params)
	if err != nil {
		return err
	}
	var scheduler segregatedScheduler
	return scheduler.Unregister(params)
}
<commit_msg>provision/docker/handler.go: created helper function to unmarshal body<commit_after>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"
)

// AddNodeHandler calls cluster.Register registering a node into it.
func AddNodeHandler(w http.ResponseWriter, r *http.Request) error {
	params, err := unmarshal(r.Body)
	if err != nil {
		return err
	}
	var scheduler segregatedScheduler
	return scheduler.Register(params)
}

func RemoveNodeHandler(w http.ResponseWriter, r *http.Request) error {
	params, err := unmarshal(r.Body)
	if err != nil {
		return err
	}
	var scheduler segregatedScheduler
	return scheduler.Unregister(params)
}

func unmarshal(body io.ReadCloser) (map[string]string, error) {
	b, err := ioutil.ReadAll(body)
	if err != nil {
		return nil, err
	}
	params := map[string]string{}
	err = json.Unmarshal(b, &params)
	if err != nil {
		return nil, err
	}
	return params, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	// "io"
	"flag"
	"io/ioutil"
	"strings"
	"time"
)

var verbose *bool

type getArgs struct {
	hostname string
	port int
	path string
	auth string
	urls bool
	verbose bool
	timeout int
}

func vLogger(msg string, args ...interface{}) {

	if *verbose {
		fmt.Fprintf(os.Stderr, msg, args...)
	}

}

func get(args getArgs) (rv bool, err error) {

	defer func() {
		if err := recover(); err != nil {
			return
		}
	}()

	rv = true

	vLogger("fetching:hostname:%s:\n", args.hostname)

	res := &http.Response{}

	if args.urls {

		url := args.hostname
		res, err = http.Head(url)
		defer res.Body.Close()

		if err != nil {
			fmt.Println(err.Error())
			rv = false
			return
		}
		_, err = ioutil.ReadAll(res.Body)

	} else {

		client := &http.Client{Timeout: time.Duration(args.timeout) * time.Second}

		// had to allocate this or the SetBasicAuth will panic
		headers := make(map[string][]string)
		hostPort := fmt.Sprintf("%s:%d", args.hostname, args.port)

		vLogger("adding hostPort:%s:%d:path:%s:\n", args.hostname, args.port, args.path)

		req := &http.Request{
			Method: "HEAD",
			// Host:  hostPort,
			URL: &url.URL{
				Host:   hostPort,
				Scheme: "http",
				Opaque: args.path,
			},
			Header: headers,
		}

		if args.auth != "" {

			up := strings.SplitN(args.auth, ":", 2)

			vLogger("Doing auth with:username:%s:password:%s:", up[0], up[1])
			req.SetBasicAuth(up[0], up[1])

		}

		if args.verbose {

			dump, _ := httputil.DumpRequestOut(req, true)
			vLogger("%s", dump)

		}

		res, err = client.Do(req)
		if err != nil {
			fmt.Println(err.Error())
			rv = false
			return
		}

		defer res.Body.Close()
		_, err = ioutil.ReadAll(res.Body)

	}

	if args.verbose {

		fmt.Println(res.Status)
		for k, v := range res.Header {
			fmt.Println(k+":", v)
		}

	}

	if res.StatusCode != http.StatusOK {
		rv = false
	}

	return
}

func main() {

	status := "OK"
	rv := 0
	name := "Bulk HTTP Check"
	bad := 0
	total := 0

	// this needs improvement. the number of spaces here has to equal the number of chars in the badHosts append line suffix
	badHosts := []byte("  ")

	//verbose := flag.Bool("v", false, "verbose output")
	verbose = flag.Bool("v", false, "verbose output")
	warn := flag.Int("w", 10, "warning level - number of non-200s or percentage of non-200s (default is numeric not percentage)")
	crit := flag.Int("c", 20, "critical level - number of non-200s or percentage of non-200s (default is numeric not percentage)")
	timeout := flag.Int("t", 2, "timeout in seconds - don't wait.  Do Head requests and don't wait.")
	pct := flag.Bool("pct", false, "interpret warming and critical levels are percentages")
	path := flag.String("path", "", "optional path to append to the input lines including the leading slash - these will not be urlencoded. This is ignored is the urls option is given.")
	file := flag.String("file", "", "input data source: a filename or '-' for STDIN.")
	port := flag.Int("port", 80, "optional port for the http request - ignored if urls is specified")
	urls := flag.Bool("urls", false, "Assume the input data is full urls - its normally a list of hostnames")
	auth := flag.String("auth", "", "Do basic auth with this username:passwd - ignored if urls is specified - make this use .netrc instead")
	checkName := flag.String("name", "", "a name to be included in the check output to distinguish the check output")
	silence := flag.Bool("silence", false, "don't make a huge list of all failing checks")

	flag.Usage = func() {

		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		fmt.Fprintf(os.Stderr, `
	Read hostnames from a file or STDIN and do a single nagios check over
	them all.  Just check for 200s.  Warning and Critical are either
	percentages of the total, or a regular numeric thresholds.

	The output contains the hostname of any non-200 reporting hosts (see -silence).

	Skip input lines that are commented out with shell style comments
	like /^#/.

	Do Head requests since we don't care about the content.  Make this
	optional some day.

	The -path is appended to the hostnames to make full URLs for the checks.

	If the -urls option is specified, then the input is assumed a complete URL, like http://$hostname:$port/$path.

	Examples:

	./someCommand |  ./check_http_bulk  -w 1 -c 2 -path '/api/aliveness-test/%%2F/' -port 15672 -file - -auth zup:nuch

	./check_http_bulk -urls -file urls.txt

`)

		flag.PrintDefaults()
	}

	flag.Parse()

	if len(flag.Args()) > 0 {

		flag.Usage()
		os.Exit(3)

	}

	// it urls is specified, the input is full urls to be used enmasse and to be url encoded
	if *urls {
		*path = ""
	}

	if *checkName != "" {
		name = *checkName
	}

	defer func() {
		if err := recover(); err != nil {
			fmt.Println(name+" Unknown: ", err)
			os.Exit(3)
		}
	}()

	if file == nil || *file == "" {
		flag.Usage()
		os.Exit(3)
	}

	inputSource := os.Stdin

	if (*file)[0] != "-"[0] {

		var err error

		inputSource, err = os.Open(*file)

		if err != nil {

			fmt.Printf("Couldn't open the specified input file:%s:error:%v:\n\n", name, err)
			flag.Usage()
			os.Exit(3)

		}

	}

	scanner := bufio.NewScanner(inputSource)
	for scanner.Scan() {

		hostname := scanner.Text()

		if len(hostname) == 0 {

			vLogger("skipping blank:\n" )

			continue
		}

		if hostname[0] == "#"[0] {

			vLogger("skipping:%s:\n", hostname)

			continue
		}

		total++

		vLogger("working on:%s:\n", hostname)

		goodCheck, err := get(getArgs{hostname: hostname, port: *port, path: *path, auth: *auth, urls: *urls, verbose: *verbose, timeout: *timeout})
		if err != nil {

			fmt.Printf("%s get error: %T %s %#v\n", name, err, err, err)
			if ! *silence {
				badHosts = append(badHosts, hostname...)
				badHosts = append(badHosts, ", "...)
			}
			bad++

			continue

		}

		if !goodCheck {
			if ! *silence {
				badHosts = append(badHosts, hostname...)
				badHosts = append(badHosts, ", "...)
			}
			bad++
		}

	}
	if err := scanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, "reading standard input:", err)
		status = "Unknown"
		rv = 3
	}

	if *pct {

		ratio := int(float64(bad)/float64(total)*100)

		vLogger("ratio:%d:\n", ratio)

		if ratio >= *crit {
			status = "Critical"
			rv = 2
		} else if ratio >= *warn {
			status = "Warning"
			rv = 1
		}

	} else {

		if bad >= *crit {
			status = "Critical"
			rv = 2
		} else if bad >= *warn {
			status = "Warning"
			rv = 1
		}

	}

	fmt.Printf("%s %s: %d of %d failed|%s\n", name, status, bad, total, badHosts[:len(badHosts)-2])
	os.Exit(rv)
}
<commit_msg>use channels to do the http calls in workers - but its not concurrent yet<commit_after>package main

import (
	"bufio"
	"fmt"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	// "io"
	"flag"
	"io/ioutil"
	"strings"
	"time"
)

var verbose *bool

type getArgs struct {
	hostname string
	port int
	path string
	auth string
	urls bool
	verbose bool
	timeout int
	resultChan chan getReply
}

type getReply struct {
	err interface{}
	rv bool
}

func vLogger(msg string, args ...interface{}) {

	if *verbose {
		fmt.Fprintf(os.Stderr, msg, args...)
	}

}

// func get(args getArgs) (rv bool, err error) {
func get(request chan *getArgs) {

	var rv bool
	var err error

	for args := range request {

		defer func() {
			if err := recover(); err != nil {
				// return
				args.resultChan <- getReply{err: err}
			}
		}()

		rv = true

		vLogger("fetching:hostname:%s:\n", args.hostname)

		res := &http.Response{}

		if args.urls {

			url := args.hostname
			res, err = http.Head(url)
			defer res.Body.Close()

			if err != nil {
				fmt.Println(err.Error())
				rv = false
				args.resultChan <- getReply{rv: rv, err: err}
				// return
			}
			_, err = ioutil.ReadAll(res.Body)

		} else {

			client := &http.Client{Timeout: time.Duration(args.timeout) * time.Second}

			// had to allocate this or the SetBasicAuth will panic
			headers := make(map[string][]string)
			hostPort := fmt.Sprintf("%s:%d", args.hostname, args.port)

			vLogger("adding hostPort:%s:%d:path:%s:\n", args.hostname, args.port, args.path)

			req := &http.Request{
				Method: "HEAD",
				// Host:  hostPort,
				URL: &url.URL{
					Host:   hostPort,
					Scheme: "http",
					Opaque: args.path,
				},
				Header: headers,
			}

			if args.auth != "" {

				up := strings.SplitN(args.auth, ":", 2)

				vLogger("Doing auth with:username:%s:password:%s:", up[0], up[1])
				req.SetBasicAuth(up[0], up[1])

			}

			if args.verbose {

				dump, _ := httputil.DumpRequestOut(req, true)
				vLogger("%s", dump)

			}

			res, err = client.Do(req)
			if err != nil {
				fmt.Println(err.Error())
				rv = false
				args.resultChan <- getReply{rv: rv, err: err}
				// return
			}

			defer res.Body.Close()
			_, err = ioutil.ReadAll(res.Body)

		}

		if args.verbose {

			fmt.Println(res.Status)
			for k, v := range res.Header {
				fmt.Println(k+":", v)
			}

		}

		if res.StatusCode != http.StatusOK {
			// rv = false
			args.resultChan <- getReply{rv: rv}
		}

		args.resultChan <- getReply{rv: rv}

	}

	// return

}

func WorkerPool(n int) chan *getArgs {
    requests := make(chan *getArgs)

    for i:=0; i<n; i++ {
        go get(requests)
    }

    return requests
}

func main() {

	status := "OK"
	rv := 0
	name := "Bulk HTTP Check"
	bad := 0
	total := 0

	// this needs improvement. the number of spaces here has to equal the number of chars in the badHosts append line suffix
	badHosts := []byte("  ")

	//verbose := flag.Bool("v", false, "verbose output")
	verbose = flag.Bool("v", false, "verbose output")
	warn := flag.Int("w", 10, "warning level - number of non-200s or percentage of non-200s (default is numeric not percentage)")
	crit := flag.Int("c", 20, "critical level - number of non-200s or percentage of non-200s (default is numeric not percentage)")
	timeout := flag.Int("t", 2, "timeout in seconds - don't wait.  Do Head requests and don't wait.")
	pct := flag.Bool("pct", false, "interpret warming and critical levels are percentages")
	path := flag.String("path", "", "optional path to append to the input lines including the leading slash - these will not be urlencoded. This is ignored is the urls option is given.")
	file := flag.String("file", "", "input data source: a filename or '-' for STDIN.")
	port := flag.Int("port", 80, "optional port for the http request - ignored if urls is specified")
	urls := flag.Bool("urls", false, "Assume the input data is full urls - its normally a list of hostnames")
	auth := flag.String("auth", "", "Do basic auth with this username:passwd - ignored if urls is specified - make this use .netrc instead")
	checkName := flag.String("name", "", "a name to be included in the check output to distinguish the check output")
	silence := flag.Bool("silence", false, "don't make a huge list of all failing checks")

	flag.Usage = func() {

		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		fmt.Fprintf(os.Stderr, `
	Read hostnames from a file or STDIN and do a single nagios check over
	them all.  Just check for 200s.  Warning and Critical are either
	percentages of the total, or a regular numeric thresholds.

	The output contains the hostname of any non-200 reporting hosts (see -silence).

	Skip input lines that are commented out with shell style comments
	like /^#/.

	Do Head requests since we don't care about the content.  Make this
	optional some day.

	The -path is appended to the hostnames to make full URLs for the checks.

	If the -urls option is specified, then the input is assumed a complete URL, like http://$hostname:$port/$path.

	Examples:

	./someCommand |  ./check_http_bulk  -w 1 -c 2 -path '/api/aliveness-test/%%2F/' -port 15672 -file - -auth zup:nuch

	./check_http_bulk -urls -file urls.txt

`)

		flag.PrintDefaults()
	}

	flag.Parse()

	if len(flag.Args()) > 0 {

		flag.Usage()
		os.Exit(3)

	}

	// it urls is specified, the input is full urls to be used enmasse and to be url encoded
	if *urls {
		*path = ""
	}

	if *checkName != "" {
		name = *checkName
	}

	defer func() {
		if err := recover(); err != nil {
			fmt.Println(name+" Unknown: ", err)
			os.Exit(3)
		}
	}()

	if file == nil || *file == "" {
		flag.Usage()
		os.Exit(3)
	}

	inputSource := os.Stdin

	if (*file)[0] != "-"[0] {

		var err error

		inputSource, err = os.Open(*file)

		if err != nil {

			fmt.Printf("Couldn't open the specified input file:%s:error:%v:\n\n", name, err)
			flag.Usage()
			os.Exit(3)

		}

	}

    requests := WorkerPool(5)
	scanner := bufio.NewScanner(inputSource)
	for scanner.Scan() {

		hostname := scanner.Text()

		if len(hostname) == 0 {

			vLogger("skipping blank:\n" )

			continue
		}

		if hostname[0] == "#"[0] {

			vLogger("skipping:%s:\n", hostname)

			continue
		}

		total++

		vLogger("working on:%s:\n", hostname)

		// goodCheck, err := get(getArgs{hostname: hostname, port: *port, path: *path, auth: *auth, urls: *urls, verbose: *verbose, timeout: *timeout})
		// request := getArgs{hostname: hostname, port: *port, path: *path, auth: *auth, urls: *urls, verbose: *verbose, timeout: *timeout, resultChan: make(chan getReply)}
		request := &getArgs{hostname: hostname, port: *port, path: *path, auth: *auth, urls: *urls, verbose: *verbose, timeout: *timeout, resultChan: make(chan getReply)}
		requests <- request
		result := <- request.resultChan

		err := result.err
		goodCheck := result.rv

		if err != nil {

			fmt.Printf("%s get error: %T %s %#v\n", name, err, err, err)
			if ! *silence {
				badHosts = append(badHosts, hostname...)
				badHosts = append(badHosts, ", "...)
			}
			bad++

			continue

		}

		if !goodCheck {
			if ! *silence {
				badHosts = append(badHosts, hostname...)
				badHosts = append(badHosts, ", "...)
			}
			bad++
		}

	}
	if err := scanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, "reading standard input:", err)
		status = "Unknown"
		rv = 3
	}

	if *pct {

		ratio := int(float64(bad)/float64(total)*100)

		vLogger("ratio:%d:\n", ratio)

		if ratio >= *crit {
			status = "Critical"
			rv = 2
		} else if ratio >= *warn {
			status = "Warning"
			rv = 1
		}

	} else {

		if bad >= *crit {
			status = "Critical"
			rv = 2
		} else if bad >= *warn {
			status = "Warning"
			rv = 1
		}

	}

	fmt.Printf("%s %s: %d of %d failed|%s\n", name, status, bad, total, badHosts[:len(badHosts)-2])
	os.Exit(rv)
}
<|endoftext|>


<commit_before>package main

import "bufio"
import "encoding/hex"
import "github.com/vbauerster/mpb"
import "hash"
import "io"
import "os"

func checksumCount(path string) int {
    var count = 0

    for checksumAlgo := range checksumLookupTable {
        checksumPath := xattrRoot + checksumAlgo
        checksumValue, _ := GetxattrString(path, checksumPath)
        if len(checksumValue) > 0 {
            count += 1
        }
    }
    return count
}

func missingChecksums(path string) bool {
    return checksumCount(path) != len(checksumLookupTable)
}

func hasAllChecksums(path string) bool {
    return checksumCount(path) == len(checksumLookupTable)
}

func checksumPath(path string) (map[string]string, error) {
    var hashers = map[string]hash.Hash{}
    var hashes = map[string]string{}

    for checksumAlgo := range checksumLookupTable {
        hashers[checksumAlgo] = checksumLookupTable[checksumAlgo].New()
    }

    fp, err := os.Open(path)
    if err != nil {
        return nil, err
    }

    defer fp.Close()

    fpStat, _ := fp.Stat()
    fileSize := fpStat.Size()
    var totalRead = 0
    var bar *mpb.Bar
    var rp *bufio.Reader

    if progressBar != nil {
        bar = progressBar.AddBar(fileSize).
                PrependName(path, 0, mpb.DwidthSync|mpb.DidentRight).
                PrependCounters("%3s / %3s", mpb.UnitBytes, 18, mpb.DwidthSync|mpb.DextraSpace).
                AppendPercentage(3, 0)
                defer progressBar.RemoveBar(bar)
        rp = bufio.NewReader(bar.ProxyReader(fp))
    } else {
        rp = bufio.NewReader(fp)
    }

    buffer := make([]byte, bufferSize)

    for {
        amountRead, err := rp.Read(buffer)
        if err != nil {
            if err == io.EOF {
                break
            }
            return nil, err
        }
        totalRead += amountRead
        if bar != nil {
            bar.Incr(amountRead)
        }

        for checksumAlgo := range checksumLookupTable {
            hashers[checksumAlgo].Write(buffer[:amountRead])
        }
    }

    for checksumAlgo := range checksumLookupTable {
        hashes[checksumAlgo] = hex.EncodeToString(hashers[checksumAlgo].Sum(nil))
        Trace.Printf("%v: Computed hash %v: %v\n", path, checksumAlgo, hashes[checksumAlgo])
    }
    return hashes, nil
}
<commit_msg>Spacing<commit_after>package main

import "bufio"
import "encoding/hex"
import "github.com/vbauerster/mpb"
import "hash"
import "io"
import "os"

func checksumCount(path string) int {
    var count = 0

    for checksumAlgo := range checksumLookupTable {
        checksumPath := xattrRoot + checksumAlgo
        checksumValue, _ := GetxattrString(path, checksumPath)
        if len(checksumValue) > 0 {
            count += 1
        }
    }
    return count
}

func missingChecksums(path string) bool {
    return checksumCount(path) != len(checksumLookupTable)
}

func hasAllChecksums(path string) bool {
    return checksumCount(path) == len(checksumLookupTable)
}

func checksumPath(path string) (map[string]string, error) {
    var hashers = map[string]hash.Hash{}
    var hashes = map[string]string{}

    for checksumAlgo := range checksumLookupTable {
        hashers[checksumAlgo] = checksumLookupTable[checksumAlgo].New()
    }

    fp, err := os.Open(path)
    if err != nil {
        return nil, err
    }

    defer fp.Close()

    fpStat, _ := fp.Stat()
    fileSize := fpStat.Size()
    var totalRead = 0
    var bar *mpb.Bar
    var rp *bufio.Reader

    if progressBar != nil {
        bar = progressBar.AddBar(fileSize).
                PrependName(path, 0, mpb.DwidthSync|mpb.DidentRight).
                PrependCounters("%3s / %3s", mpb.UnitBytes, 18, mpb.DwidthSync|mpb.DextraSpace).
                AppendPercentage(3, 0)
        defer progressBar.RemoveBar(bar)
        rp = bufio.NewReader(bar.ProxyReader(fp))
    } else {
        rp = bufio.NewReader(fp)
    }

    buffer := make([]byte, bufferSize)

    for {
        amountRead, err := rp.Read(buffer)
        if err != nil {
            if err == io.EOF {
                break
            }
            return nil, err
        }
        totalRead += amountRead
        if bar != nil {
            bar.Incr(amountRead)
        }

        for checksumAlgo := range checksumLookupTable {
            hashers[checksumAlgo].Write(buffer[:amountRead])
        }
    }

    for checksumAlgo := range checksumLookupTable {
        hashes[checksumAlgo] = hex.EncodeToString(hashers[checksumAlgo].Sum(nil))
        Trace.Printf("%v: Computed hash %v: %v\n", path, checksumAlgo, hashes[checksumAlgo])
    }
    return hashes, nil
}
<|endoftext|>


<commit_before>package kafka

import (
	"bufio"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/Symantec/scotty/lib/retry"
	"github.com/Symantec/scotty/pstore"
	"github.com/Symantec/tricorder/go/tricorder/duration"
	"github.com/Symantec/tricorder/go/tricorder/types"
	"github.com/Symantec/tricorder/go/tricorder/units"
	"github.com/optiopay/kafka"
	"github.com/optiopay/kafka/proto"
	"io"
	"os"
	"strings"
	"time"
)

const (
	kVersion   = "@version"
	kTimestamp = "@timestamp"
	kValue     = "value"
	kName      = "name"
	kHost      = "host"
	kTenantId  = "tenant_id"
	kApiKey    = "apikey"
)

const (
	kTimeFormat = "2006-01-02T15:04:05.000Z"
)

const (
	kVersionNum = "1"
)

var (
	supportedTypes = map[types.Type]bool{
		types.Bool:       true,
		types.Int8:       true,
		types.Int16:      true,
		types.Int32:      true,
		types.Int64:      true,
		types.Uint8:      true,
		types.Uint16:     true,
		types.Uint32:     true,
		types.Uint64:     true,
		types.Float32:    true,
		types.Float64:    true,
		types.GoTime:     true,
		types.GoDuration: true,
	}
)

var (
	kFakeSerializer = &recordSerializerType{
		TenantId: "aTenantId",
		ApiKey:   "anApiKey",
	}
	kFakeWriter = &fakeWriter{
		serializer: kFakeSerializer,
	}
)

func isTypeSupported(t types.Type) bool {
	return supportedTypes[t]
}

func asFloat64(r *pstore.Record) float64 {
	switch r.Kind {
	case types.Bool:
		if r.Value.(bool) {
			return 1.0
		}
		return 0.0
	case types.Int8:
		return float64(r.Value.(int8))
	case types.Int16:
		return float64(r.Value.(int16))
	case types.Int32:
		return float64(r.Value.(int32))
	case types.Int64:
		return float64(r.Value.(int64))
	case types.Uint8:
		return float64(r.Value.(uint8))
	case types.Uint16:
		return float64(r.Value.(uint16))
	case types.Uint32:
		return float64(r.Value.(uint32))
	case types.Uint64:
		return float64(r.Value.(uint64))
	case types.Float32:
		return float64(r.Value.(float32))
	case types.Float64:
		return r.Value.(float64)
	case types.GoTime:
		return duration.TimeToFloat(r.Value.(time.Time))
	case types.GoDuration:
		return duration.ToFloat(
			r.Value.(time.Duration)) * units.FromSeconds(
			r.Unit)
	default:
		panic("Unsupported type")

	}
}

// TODO: Remove once we know the grafana bug involving duplicate timestamps
// is fixed.
type pathAndMillisType struct {
	Path   string
	Millis int64
}

// TODO: Remove once we know the grafana bug involving duplicate timestamps
// is fixed.
func newPathAndMillisType(record *pstore.Record) pathAndMillisType {
	return pathAndMillisType{
		Path:   record.Path,
		Millis: int64(duration.TimeToFloat(record.Timestamp) * 1000.0)}
}

// TODO: Remove once we know the grafana bug involving duplicate timestamps
// is fixed.
type uniqueMetricsWriter struct {
	pstore.RecordWriter
}

// fixDuplicates is a workaround for a Grafana bug.
// In Grafana, two metric values cannot have the same name and timestamp even
// if they are for different endpoints. Otherwise, race conditions cause
// one of the metric values to stomp out the other yielding an incomplete
// data set.
// fixDuplicates returns a slice like records except that if two records
// have the same metric name and timestamp, fixDuplicates adds 1ms to one
// of the timestamps to avoid race conditions in Grafana.
// If records requires no modifications, fixDuplicates returns it unchanged.
// Otherwise, fixDuplicates returns a copy of the records slice with the
// needed modifications leaving the original records slice unchanged.
func fixDuplicates(records []pstore.Record) (result []pstore.Record) {
	// TODO: Remove once we know the grafana bug involving duplicate timestamps
	// is fixed.
	result = records
	copied := false
	pathAndTimeExists := make(map[pathAndMillisType]bool)
	for i := range records {
		pathAndMillis := newPathAndMillisType(&result[i])
		for pathAndTimeExists[pathAndMillis] {
			if !copied {
				result = make([]pstore.Record, len(records))
				copy(result, records)
				copied = true
			}
			result[i].Timestamp = result[i].Timestamp.Add(time.Millisecond)
			pathAndMillis = newPathAndMillisType(&result[i])
		}
		pathAndTimeExists[pathAndMillis] = true
	}
	return
}

// TODO: Remove once we know the grafana bug involving duplicate timestamps
// is fixed.
func (u uniqueMetricsWriter) Write(records []pstore.Record) (err error) {
	return u.RecordWriter.Write(fixDuplicates(records))
}

type fakeWriter struct {
	serializer *recordSerializerType
	file       *os.File
	buffer     *bufio.Writer
}

func newFakeWriter() pstore.LimitedRecordWriter {
	return kFakeWriter
}

func newFakeWriterToPath(path string) (pstore.LimitedRecordWriter, error) {
	file, err := os.Create(path)
	if err != nil {
		return nil, err
	}
	return &fakeWriter{
		serializer: kFakeSerializer,
		file:       file,
		buffer:     bufio.NewWriter(file),
	}, nil
}

func (f *fakeWriter) IsTypeSupported(t types.Type) bool {
	return IsTypeSupported(t)
}

func (f *fakeWriter) Write(records []pstore.Record) (err error) {
	for i := range records {
		var payload []byte
		payload, err = f.serializer.Serialize(&records[i])
		if err != nil {
			return
		}
		f.printLine(string(payload))
		f.printLine()
	}
	f.printLine()
	return
}

func (f *fakeWriter) printLine(args ...interface{}) {
	if f.buffer == nil {
		fmt.Println(args...)
	} else {
		fmt.Fprintln(f.buffer, args...)
		f.buffer.Flush()
		f.file.Sync()
	}
}

type writer struct {
	topic      string
	serializer recordSerializerType
	clientId   string
	endpoints  []string
	retry      retry.Retry
	broker     *kafka.Broker
	producer   kafka.DistributingProducer
}

func newWriter(c Config) (
	result pstore.LimitedRecordWriter, err error) {
	if len(c.Endpoints) == 0 || c.Topic == "" || c.ClientId == "" || c.TenantId == "" || c.ApiKey == "" {
		err = errors.New(
			"endpoint, topic, clientId, tenantId, and apiKey keys required")
		return
	}
	var awriter writer
	awriter.topic = c.Topic
	awriter.serializer.TenantId = c.TenantId
	awriter.serializer.ApiKey = c.ApiKey
	awriter.clientId = c.ClientId
	awriter.endpoints = c.Endpoints
	result = &awriter
	return
}

func (w *writer) ensureInitialized() error {
	return w.retry.Do(func() (err error) {
		w.broker, err = kafka.Dial(w.endpoints, kafka.NewBrokerConf(w.clientId))
		if err != nil {
			return
		}
		var count int32
		count, err = w.broker.PartitionCount(w.topic)
		if err != nil {
			return
		}
		conf := kafka.NewProducerConf()
		conf.RequiredAcks = proto.RequiredAcksLocal
		producer := w.broker.Producer(conf)
		w.producer = newChannelProducer(producer, count)
		return
	})
}

func (w *writer) IsTypeSupported(t types.Type) bool {
	return IsTypeSupported(t)
}

func (w *writer) Write(records []pstore.Record) (err error) {
	msgs := make([]*proto.Message, len(records))
	for i := range records {
		var payload []byte
		payload, err = w.serializer.Serialize(&records[i])
		if err != nil {
			return
		}
		msgs[i] = &proto.Message{Value: payload}
	}
	if err = w.ensureInitialized(); err != nil {
		return
	}
	_, err = w.producer.Distribute(w.topic, msgs...)
	return
}

// recordSerializerType serializes a record to bytes for kafka.
type recordSerializerType struct {
	TenantId string
	ApiKey   string
}

func (s *recordSerializerType) Serialize(r *pstore.Record) ([]byte, error) {
	return LMMSerialiseAsBytes(r, s.TenantId, s.ApiKey, true)
}

func lmmJSONPayload(
	r *pstore.Record,
	tenantId,
	apiKey string,
	slashesToUnderscores bool) map[string]interface{} {
	record := map[string]interface{}{
		kVersion:   kVersionNum,
		kTenantId:  tenantId,
		kApiKey:    apiKey,
		kTimestamp: r.Timestamp.Format(kTimeFormat),
		kName:      r.Path,
		kHost:      r.HostName}
	if slashesToUnderscores {
		record[kName] = strings.Replace(r.Path, "/", "_", -1)
	}
	if r.Kind == types.String {
		record[kValue] = 0.0
		record["stringValue"] = r.Value.(string)
	} else {
		record[kValue] = ToFloat64(r)
	}
	for k, v := range r.Tags {
		record[k] = v
	}
	return record
}

func lmmSerialise(
	r *pstore.Record,
	tenantId,
	apiKey string,
	slashesToUnderscores bool,
	w io.Writer) error {
	encoder := json.NewEncoder(w)
	return encoder.Encode(
		lmmJSONPayload(r, tenantId, apiKey, slashesToUnderscores))
}
<commit_msg>Only send first 100 characters of strings to lmm.<commit_after>package kafka

import (
	"bufio"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/Symantec/scotty/lib/retry"
	"github.com/Symantec/scotty/pstore"
	"github.com/Symantec/tricorder/go/tricorder/duration"
	"github.com/Symantec/tricorder/go/tricorder/types"
	"github.com/Symantec/tricorder/go/tricorder/units"
	"github.com/optiopay/kafka"
	"github.com/optiopay/kafka/proto"
	"io"
	"os"
	"strings"
	"time"
)

const (
	kVersion   = "@version"
	kTimestamp = "@timestamp"
	kValue     = "value"
	kName      = "name"
	kHost      = "host"
	kTenantId  = "tenant_id"
	kApiKey    = "apikey"
	kMaxStrLen = 100
)

const (
	kTimeFormat = "2006-01-02T15:04:05.000Z"
)

const (
	kVersionNum = "1"
)

var (
	supportedTypes = map[types.Type]bool{
		types.Bool:       true,
		types.Int8:       true,
		types.Int16:      true,
		types.Int32:      true,
		types.Int64:      true,
		types.Uint8:      true,
		types.Uint16:     true,
		types.Uint32:     true,
		types.Uint64:     true,
		types.Float32:    true,
		types.Float64:    true,
		types.GoTime:     true,
		types.GoDuration: true,
	}
)

var (
	kFakeSerializer = &recordSerializerType{
		TenantId: "aTenantId",
		ApiKey:   "anApiKey",
	}
	kFakeWriter = &fakeWriter{
		serializer: kFakeSerializer,
	}
)

func isTypeSupported(t types.Type) bool {
	return supportedTypes[t]
}

func asFloat64(r *pstore.Record) float64 {
	switch r.Kind {
	case types.Bool:
		if r.Value.(bool) {
			return 1.0
		}
		return 0.0
	case types.Int8:
		return float64(r.Value.(int8))
	case types.Int16:
		return float64(r.Value.(int16))
	case types.Int32:
		return float64(r.Value.(int32))
	case types.Int64:
		return float64(r.Value.(int64))
	case types.Uint8:
		return float64(r.Value.(uint8))
	case types.Uint16:
		return float64(r.Value.(uint16))
	case types.Uint32:
		return float64(r.Value.(uint32))
	case types.Uint64:
		return float64(r.Value.(uint64))
	case types.Float32:
		return float64(r.Value.(float32))
	case types.Float64:
		return r.Value.(float64)
	case types.GoTime:
		return duration.TimeToFloat(r.Value.(time.Time))
	case types.GoDuration:
		return duration.ToFloat(
			r.Value.(time.Duration)) * units.FromSeconds(
			r.Unit)
	default:
		panic("Unsupported type")

	}
}

// TODO: Remove once we know the grafana bug involving duplicate timestamps
// is fixed.
type pathAndMillisType struct {
	Path   string
	Millis int64
}

// TODO: Remove once we know the grafana bug involving duplicate timestamps
// is fixed.
func newPathAndMillisType(record *pstore.Record) pathAndMillisType {
	return pathAndMillisType{
		Path:   record.Path,
		Millis: int64(duration.TimeToFloat(record.Timestamp) * 1000.0)}
}

// TODO: Remove once we know the grafana bug involving duplicate timestamps
// is fixed.
type uniqueMetricsWriter struct {
	pstore.RecordWriter
}

// fixDuplicates is a workaround for a Grafana bug.
// In Grafana, two metric values cannot have the same name and timestamp even
// if they are for different endpoints. Otherwise, race conditions cause
// one of the metric values to stomp out the other yielding an incomplete
// data set.
// fixDuplicates returns a slice like records except that if two records
// have the same metric name and timestamp, fixDuplicates adds 1ms to one
// of the timestamps to avoid race conditions in Grafana.
// If records requires no modifications, fixDuplicates returns it unchanged.
// Otherwise, fixDuplicates returns a copy of the records slice with the
// needed modifications leaving the original records slice unchanged.
func fixDuplicates(records []pstore.Record) (result []pstore.Record) {
	// TODO: Remove once we know the grafana bug involving duplicate timestamps
	// is fixed.
	result = records
	copied := false
	pathAndTimeExists := make(map[pathAndMillisType]bool)
	for i := range records {
		pathAndMillis := newPathAndMillisType(&result[i])
		for pathAndTimeExists[pathAndMillis] {
			if !copied {
				result = make([]pstore.Record, len(records))
				copy(result, records)
				copied = true
			}
			result[i].Timestamp = result[i].Timestamp.Add(time.Millisecond)
			pathAndMillis = newPathAndMillisType(&result[i])
		}
		pathAndTimeExists[pathAndMillis] = true
	}
	return
}

// TODO: Remove once we know the grafana bug involving duplicate timestamps
// is fixed.
func (u uniqueMetricsWriter) Write(records []pstore.Record) (err error) {
	return u.RecordWriter.Write(fixDuplicates(records))
}

type fakeWriter struct {
	serializer *recordSerializerType
	file       *os.File
	buffer     *bufio.Writer
}

func newFakeWriter() pstore.LimitedRecordWriter {
	return kFakeWriter
}

func newFakeWriterToPath(path string) (pstore.LimitedRecordWriter, error) {
	file, err := os.Create(path)
	if err != nil {
		return nil, err
	}
	return &fakeWriter{
		serializer: kFakeSerializer,
		file:       file,
		buffer:     bufio.NewWriter(file),
	}, nil
}

func (f *fakeWriter) IsTypeSupported(t types.Type) bool {
	return IsTypeSupported(t)
}

func (f *fakeWriter) Write(records []pstore.Record) (err error) {
	for i := range records {
		var payload []byte
		payload, err = f.serializer.Serialize(&records[i])
		if err != nil {
			return
		}
		f.printLine(string(payload))
		f.printLine()
	}
	f.printLine()
	return
}

func (f *fakeWriter) printLine(args ...interface{}) {
	if f.buffer == nil {
		fmt.Println(args...)
	} else {
		fmt.Fprintln(f.buffer, args...)
		f.buffer.Flush()
		f.file.Sync()
	}
}

type writer struct {
	topic      string
	serializer recordSerializerType
	clientId   string
	endpoints  []string
	retry      retry.Retry
	broker     *kafka.Broker
	producer   kafka.DistributingProducer
}

func newWriter(c Config) (
	result pstore.LimitedRecordWriter, err error) {
	if len(c.Endpoints) == 0 || c.Topic == "" || c.ClientId == "" || c.TenantId == "" || c.ApiKey == "" {
		err = errors.New(
			"endpoint, topic, clientId, tenantId, and apiKey keys required")
		return
	}
	var awriter writer
	awriter.topic = c.Topic
	awriter.serializer.TenantId = c.TenantId
	awriter.serializer.ApiKey = c.ApiKey
	awriter.clientId = c.ClientId
	awriter.endpoints = c.Endpoints
	result = &awriter
	return
}

func (w *writer) ensureInitialized() error {
	return w.retry.Do(func() (err error) {
		w.broker, err = kafka.Dial(w.endpoints, kafka.NewBrokerConf(w.clientId))
		if err != nil {
			return
		}
		var count int32
		count, err = w.broker.PartitionCount(w.topic)
		if err != nil {
			return
		}
		conf := kafka.NewProducerConf()
		conf.RequiredAcks = proto.RequiredAcksLocal
		producer := w.broker.Producer(conf)
		w.producer = newChannelProducer(producer, count)
		return
	})
}

func (w *writer) IsTypeSupported(t types.Type) bool {
	return IsTypeSupported(t)
}

func (w *writer) Write(records []pstore.Record) (err error) {
	msgs := make([]*proto.Message, len(records))
	for i := range records {
		var payload []byte
		payload, err = w.serializer.Serialize(&records[i])
		if err != nil {
			return
		}
		msgs[i] = &proto.Message{Value: payload}
	}
	if err = w.ensureInitialized(); err != nil {
		return
	}
	_, err = w.producer.Distribute(w.topic, msgs...)
	return
}

// recordSerializerType serializes a record to bytes for kafka.
type recordSerializerType struct {
	TenantId string
	ApiKey   string
}

func (s *recordSerializerType) Serialize(r *pstore.Record) ([]byte, error) {
	return LMMSerialiseAsBytes(r, s.TenantId, s.ApiKey, true)
}

func lmmJSONPayload(
	r *pstore.Record,
	tenantId,
	apiKey string,
	slashesToUnderscores bool) map[string]interface{} {
	record := map[string]interface{}{
		kVersion:   kVersionNum,
		kTenantId:  tenantId,
		kApiKey:    apiKey,
		kTimestamp: r.Timestamp.Format(kTimeFormat),
		kName:      r.Path,
		kHost:      r.HostName}
	if slashesToUnderscores {
		record[kName] = strings.Replace(r.Path, "/", "_", -1)
	}
	if r.Kind == types.String {
		record[kValue] = 0.0
		str := r.Value.(string)
		if len(str) > kMaxStrLen {
			str = str[:kMaxStrLen]
		}
		record["stringValue"] = str
	} else {
		record[kValue] = ToFloat64(r)
	}
	for k, v := range r.Tags {
		record[k] = v
	}
	return record
}

func lmmSerialise(
	r *pstore.Record,
	tenantId,
	apiKey string,
	slashesToUnderscores bool,
	w io.Writer) error {
	encoder := json.NewEncoder(w)
	return encoder.Encode(
		lmmJSONPayload(r, tenantId, apiKey, slashesToUnderscores))
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tabletmanager

import (
	"os"
	"path"
	"sync"
	"time"

	"golang.org/x/net/context"
	"vitess.io/vitess/go/mysql"
	"vitess.io/vitess/go/timer"
	"vitess.io/vitess/go/vt/log"
	"vitess.io/vitess/go/vt/mysqlctl"
	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
	"vitess.io/vitess/go/vt/topo"
)

const (
	// replicationStoppedFile is the name of the file whose existence informs
	// vttablet to NOT try to repair replication.
	replicationStoppedFile = "do_not_replicate"
)

// replManager runs a poller to ensure mysql is replicating from
// the master. If necessary, it invokes tm.repairReplication to get it
// fixed. On state change, SetTabletType must be called before changing
// the tabletserver state. This will ensure that replication is fixed
// upfront, allowing tabletserver to start off healthy.
type replManager struct {
	ctx          context.Context
	tm           *TabletManager
	markerFile   string
	ticks        *timer.Timer
	firstFailure bool

	// replStopped is tri-state.
	// A nil value signifies that the value is not set.
	mu          sync.Mutex
	replStopped *bool
}

func newReplManager(ctx context.Context, tm *TabletManager, interval time.Duration) *replManager {
	return &replManager{
		ctx:        ctx,
		tm:         tm,
		markerFile: markerFile(tm.Cnf),
		ticks:      timer.NewTimer(interval),
	}
}

func (rm *replManager) SetTabletType(tabletType topodatapb.TabletType) {
	if *mysqlctl.DisableActiveReparents {
		return
	}
	if !topo.IsReplicaType(tabletType) {
		rm.ticks.Stop()
		return
	}
	if rm.replicationStopped() {
		// Stop just to be safe.
		rm.ticks.Stop()
		log.Info("Replication Manager: stopped")
		return
	}
	if rm.ticks.Running() {
		return
	}
	log.Info("Replication Manager: starting")
	// Run an immediate check to fix replication if it was broken.
	// A higher caller may already have te action lock. So, we use
	// a code path that avoids it.
	rm.checkActionLocked()
	rm.ticks.Start(rm.check)
}

func (rm *replManager) check() {
	// We need to obtain the action lock if we're going to fix
	// replication
	if err := rm.tm.lock(rm.ctx); err != nil {
		return
	}
	defer rm.tm.unlock()
	rm.checkActionLocked()
}

func (rm *replManager) checkActionLocked() {
	status, err := rm.tm.MysqlDaemon.ReplicationStatus()
	if err != nil {
		if err != mysql.ErrNotReplica {
			return
		}
	} else {
		// If only one of the threads is stopped, it's probably
		// intentional. So, we don't repair replication.
		if status.SQLThreadRunning || status.IOThreadRunning {
			return
		}
	}

	if rm.firstFailure {
		log.Infof("Replication is stopped, reconnecting to master.")
	}
	ctx, cancel := context.WithTimeout(rm.ctx, 5*time.Second)
	defer cancel()
	if err := rm.tm.repairReplication(ctx); err != nil {
		if rm.firstFailure {
			rm.firstFailure = false
			log.Infof("Failed to reconnect to master: %v, will keep retrying.", err)
		}
		return
	}
	log.Info("Successfully reconnected to master.")
	rm.firstFailure = true
}

// setReplicationStopped performs a best effort attempt of
// remembering a decision to stop replication.
func (rm *replManager) setReplicationStopped(stopped bool) {
	if *mysqlctl.DisableActiveReparents {
		return
	}

	rm.mu.Lock()
	defer rm.mu.Unlock()

	rm.replStopped = &stopped

	if stopped {
		rm.ticks.Stop()
	} else {
		rm.ticks.Start(rm.check)
	}

	if rm.markerFile == "" {
		return
	}
	if stopped {
		if file, err := os.Create(rm.markerFile); err == nil {
			file.Close()
		}
	} else {
		os.Remove(rm.markerFile)
	}
}

func (rm *replManager) replicationStopped() bool {
	rm.mu.Lock()
	defer rm.mu.Unlock()

	if rm.replStopped != nil {
		// Return cached value.
		return *rm.replStopped
	}
	if rm.markerFile == "" {
		return false
	}

	_, err := os.Stat(rm.markerFile)
	replicationStopped := err == nil
	rm.replStopped = &replicationStopped
	return replicationStopped
}

func markerFile(cnf *mysqlctl.Mycnf) string {
	if cnf == nil {
		return ""
	}
	return path.Join(cnf.TabletDir(), replicationStoppedFile)
}
<commit_msg>tm: minor logging fix<commit_after>/*
Copyright 2020 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tabletmanager

import (
	"os"
	"path"
	"sync"
	"time"

	"golang.org/x/net/context"
	"vitess.io/vitess/go/mysql"
	"vitess.io/vitess/go/timer"
	"vitess.io/vitess/go/vt/log"
	"vitess.io/vitess/go/vt/mysqlctl"
	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
	"vitess.io/vitess/go/vt/topo"
)

const (
	// replicationStoppedFile is the name of the file whose existence informs
	// vttablet to NOT try to repair replication.
	replicationStoppedFile = "do_not_replicate"
)

// replManager runs a poller to ensure mysql is replicating from
// the master. If necessary, it invokes tm.repairReplication to get it
// fixed. On state change, SetTabletType must be called before changing
// the tabletserver state. This will ensure that replication is fixed
// upfront, allowing tabletserver to start off healthy.
type replManager struct {
	ctx        context.Context
	tm         *TabletManager
	markerFile string
	ticks      *timer.Timer
	failed     bool

	// replStopped is tri-state.
	// A nil value signifies that the value is not set.
	mu          sync.Mutex
	replStopped *bool
}

func newReplManager(ctx context.Context, tm *TabletManager, interval time.Duration) *replManager {
	return &replManager{
		ctx:        ctx,
		tm:         tm,
		markerFile: markerFile(tm.Cnf),
		ticks:      timer.NewTimer(interval),
	}
}

func (rm *replManager) SetTabletType(tabletType topodatapb.TabletType) {
	if *mysqlctl.DisableActiveReparents {
		return
	}
	if !topo.IsReplicaType(tabletType) {
		rm.ticks.Stop()
		return
	}
	if rm.replicationStopped() {
		// Stop just to be safe.
		rm.ticks.Stop()
		log.Info("Replication Manager: stopped")
		return
	}
	if rm.ticks.Running() {
		return
	}
	log.Info("Replication Manager: starting")
	// Run an immediate check to fix replication if it was broken.
	// A higher caller may already have te action lock. So, we use
	// a code path that avoids it.
	rm.checkActionLocked()
	rm.ticks.Start(rm.check)
}

func (rm *replManager) check() {
	// We need to obtain the action lock if we're going to fix
	// replication
	if err := rm.tm.lock(rm.ctx); err != nil {
		return
	}
	defer rm.tm.unlock()
	rm.checkActionLocked()
}

func (rm *replManager) checkActionLocked() {
	status, err := rm.tm.MysqlDaemon.ReplicationStatus()
	if err != nil {
		if err != mysql.ErrNotReplica {
			return
		}
	} else {
		// If only one of the threads is stopped, it's probably
		// intentional. So, we don't repair replication.
		if status.SQLThreadRunning || status.IOThreadRunning {
			return
		}
	}

	if !rm.failed {
		log.Infof("Replication is stopped, reconnecting to master.")
	}
	ctx, cancel := context.WithTimeout(rm.ctx, 5*time.Second)
	defer cancel()
	if err := rm.tm.repairReplication(ctx); err != nil {
		if !rm.failed {
			rm.failed = true
			log.Infof("Failed to reconnect to master: %v, will keep retrying.", err)
		}
		return
	}
	log.Info("Successfully reconnected to master.")
	rm.failed = false
}

// setReplicationStopped performs a best effort attempt of
// remembering a decision to stop replication.
func (rm *replManager) setReplicationStopped(stopped bool) {
	if *mysqlctl.DisableActiveReparents {
		return
	}

	rm.mu.Lock()
	defer rm.mu.Unlock()

	rm.replStopped = &stopped

	if stopped {
		rm.ticks.Stop()
	} else {
		rm.ticks.Start(rm.check)
	}

	if rm.markerFile == "" {
		return
	}
	if stopped {
		if file, err := os.Create(rm.markerFile); err == nil {
			file.Close()
		}
	} else {
		os.Remove(rm.markerFile)
	}
}

func (rm *replManager) replicationStopped() bool {
	rm.mu.Lock()
	defer rm.mu.Unlock()

	if rm.replStopped != nil {
		// Return cached value.
		return *rm.replStopped
	}
	if rm.markerFile == "" {
		return false
	}

	_, err := os.Stat(rm.markerFile)
	replicationStopped := err == nil
	rm.replStopped = &replicationStopped
	return replicationStopped
}

func markerFile(cnf *mysqlctl.Mycnf) string {
	if cnf == nil {
		return ""
	}
	return path.Join(cnf.TabletDir(), replicationStoppedFile)
}
<|endoftext|>