<commit_before>package integration

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"testing"
	"time"

	"github.com/google/shlex"
	"github.com/pkg/errors"
)

func init() {
	register(&oci{})

	// the rootless uid is defined in hack/dockerfiles/test.Dockerfile
	if s := os.Getenv("BUILDKIT_INTEGRATION_ROOTLESS_IDPAIR"); s != "" {
		var uid, gid int
		if _, err := fmt.Sscanf(s, "%d:%d", &uid, &gid); err != nil {
			panic(errors.Errorf("unexpected BUILDKIT_INTEGRATION_ROOTLESS_IDPAIR: %q", s))
		}
		register(&oci{uid: uid, gid: gid})
	}
}

type oci struct {
	uid int
	gid int
}

func (s *oci) Name() string {
	if s.uid != 0 {
		return "oci-rootless"
	}
	return "oci"
}

func (s *oci) New() (Sandbox, func() error, error) {
	if err := lookupBinary("buildkitd"); err != nil {
		return nil, nil, err
	}
	if err := requireRoot(); err != nil {
		return nil, nil, err
	}
	logs := map[string]*bytes.Buffer{}
	buildkitdArgs := []string{"buildkitd", "--oci-worker=true", "--containerd-worker=false"}
	if s.uid != 0 {
		if s.gid == 0 {
			return nil, nil, errors.Errorf("unsupported id pair: uid=%d, gid=%d", s.uid, s.gid)
		}
		// TODO: make sure the user exists and subuid/subgid are configured.
		buildkitdArgs = append([]string{"sudo", "-u", fmt.Sprintf("#%d", s.uid), "-i", "--", "rootlesskit"}, buildkitdArgs...)
	}
	buildkitdSock, stop, err := runBuildkitd(buildkitdArgs, logs, s.uid, s.gid)
	if err != nil {
		return nil, nil, err
	}

	deferF := &multiCloser{}
	deferF.append(stop)

	return &sandbox{address: buildkitdSock, logs: logs, cleanup: deferF, rootless: s.uid != 0}, deferF.F(), nil
}

type sandbox struct {
	address  string
	logs     map[string]*bytes.Buffer
	cleanup  *multiCloser
	rootless bool
}

func (sb *sandbox) Address() string {
	return sb.address
}

func (sb *sandbox) PrintLogs(t *testing.T) {
	for name, l := range sb.logs {
		t.Log(name)
		s := bufio.NewScanner(l)
		for s.Scan() {
			t.Log(s.Text())
		}
	}
}

func (sb *sandbox) NewRegistry() (string, error) {
	url, cl, err := newRegistry()
	if err != nil {
		return "", err
	}
	sb.cleanup.append(cl)
	return url, nil
}

func (sb *sandbox) Cmd(args ...string) *exec.Cmd {
	if len(args) == 1 {
		if split, err := shlex.Split(args[0]); err == nil {
			args = split
		}
	}
	cmd := exec.Command("buildctl", args...)
	cmd.Env = append(cmd.Env, os.Environ()...)
	cmd.Env = append(cmd.Env, "BUILDKIT_HOST="+sb.Address())
	return cmd
}

func (sb *sandbox) Rootless() bool {
	return sb.rootless
}

func runBuildkitd(args []string, logs map[string]*bytes.Buffer, uid, gid int) (address string, cl func() error, err error) {
	deferF := &multiCloser{}
	cl = deferF.F()

	defer func() {
		if err != nil {
			deferF.F()()
			cl = nil
		}
	}()

	tmpdir, err := ioutil.TempDir("", "bktest_buildkitd")
	if err != nil {
		return "", nil, err
	}
	if err := os.Chown(tmpdir, uid, gid); err != nil {
		return "", nil, err
	}
	deferF.append(func() error { return os.RemoveAll(tmpdir) })

	address = "unix://" + filepath.Join(tmpdir, "buildkitd.sock")
	if runtime.GOOS == "windows" {
		address = "//./pipe/buildkitd-" + filepath.Base(tmpdir)
	}

	args = append(args, "--root", tmpdir, "--addr", address, "--debug")
	cmd := exec.Command(args[0], args[1:]...)
	cmd.Env = append(os.Environ(), "BUILDKIT_DEBUG_EXEC_OUTPUT=1")

	if stop, err := startCmd(cmd, logs); err != nil {
		return "", nil, err
	} else {
		deferF.append(stop)
	}

	if err := waitUnix(address, 5*time.Second); err != nil {
		return "", nil, err
	}

	return
}
<commit_msg>integration: skip rootless if unsupported<commit_after>package integration

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"testing"
	"time"

	"github.com/google/shlex"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

func init() {
	register(&oci{})

	// the rootless uid is defined in hack/dockerfiles/test.Dockerfile
	if s := os.Getenv("BUILDKIT_INTEGRATION_ROOTLESS_IDPAIR"); s != "" {
		var uid, gid int
		if _, err := fmt.Sscanf(s, "%d:%d", &uid, &gid); err != nil {
			logrus.Fatalf("unexpected BUILDKIT_INTEGRATION_ROOTLESS_IDPAIR: %q", s)
		}
		if rootlessSupported(uid) {
			register(&oci{uid: uid, gid: gid})
		}
	}
}

type oci struct {
	uid int
	gid int
}

func (s *oci) Name() string {
	if s.uid != 0 {
		return "oci-rootless"
	}
	return "oci"
}

func (s *oci) New() (Sandbox, func() error, error) {
	if err := lookupBinary("buildkitd"); err != nil {
		return nil, nil, err
	}
	if err := requireRoot(); err != nil {
		return nil, nil, err
	}
	logs := map[string]*bytes.Buffer{}
	buildkitdArgs := []string{"buildkitd", "--oci-worker=true", "--containerd-worker=false"}
	if s.uid != 0 {
		if s.gid == 0 {
			return nil, nil, errors.Errorf("unsupported id pair: uid=%d, gid=%d", s.uid, s.gid)
		}
		// TODO: make sure the user exists and subuid/subgid are configured.
		buildkitdArgs = append([]string{"sudo", "-u", fmt.Sprintf("#%d", s.uid), "-i", "--", "rootlesskit"}, buildkitdArgs...)
	}
	buildkitdSock, stop, err := runBuildkitd(buildkitdArgs, logs, s.uid, s.gid)
	if err != nil {
		return nil, nil, err
	}

	deferF := &multiCloser{}
	deferF.append(stop)

	return &sandbox{address: buildkitdSock, logs: logs, cleanup: deferF, rootless: s.uid != 0}, deferF.F(), nil
}

type sandbox struct {
	address  string
	logs     map[string]*bytes.Buffer
	cleanup  *multiCloser
	rootless bool
}

func (sb *sandbox) Address() string {
	return sb.address
}

func (sb *sandbox) PrintLogs(t *testing.T) {
	for name, l := range sb.logs {
		t.Log(name)
		s := bufio.NewScanner(l)
		for s.Scan() {
			t.Log(s.Text())
		}
	}
}

func (sb *sandbox) NewRegistry() (string, error) {
	url, cl, err := newRegistry()
	if err != nil {
		return "", err
	}
	sb.cleanup.append(cl)
	return url, nil
}

func (sb *sandbox) Cmd(args ...string) *exec.Cmd {
	if len(args) == 1 {
		if split, err := shlex.Split(args[0]); err == nil {
			args = split
		}
	}
	cmd := exec.Command("buildctl", args...)
	cmd.Env = append(cmd.Env, os.Environ()...)
	cmd.Env = append(cmd.Env, "BUILDKIT_HOST="+sb.Address())
	return cmd
}

func (sb *sandbox) Rootless() bool {
	return sb.rootless
}

func runBuildkitd(args []string, logs map[string]*bytes.Buffer, uid, gid int) (address string, cl func() error, err error) {
	deferF := &multiCloser{}
	cl = deferF.F()

	defer func() {
		if err != nil {
			deferF.F()()
			cl = nil
		}
	}()

	tmpdir, err := ioutil.TempDir("", "bktest_buildkitd")
	if err != nil {
		return "", nil, err
	}
	if err := os.Chown(tmpdir, uid, gid); err != nil {
		return "", nil, err
	}
	deferF.append(func() error { return os.RemoveAll(tmpdir) })

	address = "unix://" + filepath.Join(tmpdir, "buildkitd.sock")
	if runtime.GOOS == "windows" {
		address = "//./pipe/buildkitd-" + filepath.Base(tmpdir)
	}

	args = append(args, "--root", tmpdir, "--addr", address, "--debug")
	cmd := exec.Command(args[0], args[1:]...)
	cmd.Env = append(os.Environ(), "BUILDKIT_DEBUG_EXEC_OUTPUT=1")

	if stop, err := startCmd(cmd, logs); err != nil {
		return "", nil, err
	} else {
		deferF.append(stop)
	}

	if err := waitUnix(address, 5*time.Second); err != nil {
		return "", nil, err
	}

	return
}

func rootlessSupported(uid int) bool {
	cmd := exec.Command("sudo", "-u", fmt.Sprintf("#%d", uid), "-i", "--", "unshare", "-U", "true")
	b, err := cmd.CombinedOutput()
	if err != nil {
		logrus.Warnf("rootless mode is not supported on this host: %v (%s)", err, string(b))
		return false
	}
	return true
}
<|endoftext|>


<commit_before>// Copyright 2019, OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package translation

const (
	// DefaultTranslationRulesYaml defines default translation rules that will be applied to metrics if
	// config.SendCompatibleMetrics set to true and config.TranslationRules not specified explicitly.
	// Keep it in YAML format to be able to easily copy and paste it in config if modifications needed.
	DefaultTranslationRulesYaml = `
translation_rules:

- action: rename_dimension_keys
  mapping:

    # dimensions
    container.image.name: container_image
    k8s.container.name: container_spec_name
    k8s.cluster.name: kubernetes_cluster
    k8s.daemonset.name: kubernetes_name
    k8s.daemonset.uid: kubernetes_uid
    k8s.deployment.name: kubernetes_name
    k8s.deployment.uid: kubernetes_uid
    k8s.hpa.name: kubernetes_name
    k8s.hpa.uid: kubernetes_uid
    k8s.namespace.name: kubernetes_namespace
    k8s.node.name: kubernetes_node
    k8s.node.uid: kubernetes_node_uid
    k8s.pod.name: kubernetes_pod_name
    k8s.pod.uid: kubernetes_pod_uid
    k8s.replicaset.name: kubernetes_name
    k8s.replicaset.uid: kubernetes_uid
    k8s.replicationcontroller.name: kubernetes_name
    k8s.replicationcontroller.uid: kubernetes_uid
    k8s.resourcequota.name: quota_name
    k8s.resourcequota.uid: kubernetes_uid
    k8s.statefulset.name: kubernetes_name
    k8s.statefulset.uid: kubernetes_uid

    # properties
    cronjob_uid: cronJob_uid
    cronjob: cronJob
    daemonset_uid: daemonSet_uid
    daemonset: daemonSet
    k8s.workload.kind: kubernetes_workload
    k8s.workload.name: kubernetes_workload_name
    replicaset_uid: replicaSet_uid
    replicaset: replicaSet
    statefulset_uid: statefulSet_uid
    statefulset: statefulSet

- action: rename_metrics
  mapping:

    # kubeletstats receiver metrics
    container.cpu.time: container_cpu_utilization
    container.filesystem.available: container_fs_available_bytes
    container.filesystem.capacity: container_fs_capacity_bytes
    container.filesystem.usage: container_fs_usage_bytes
    container.memory.available: container_memory_available_bytes
    container.memory.major_page_faults: container_memory_major_page_faults
    container.memory.page_faults: container_memory_page_faults
    container.memory.rss: container_memory_rss_bytes
    container.memory.usage: container_memory_usage_bytes
    container.memory.working_set: container_memory_working_set_bytes

    # k8s cluster receiver metrics
    k8s/container/cpu/limit: kubernetes.container_cpu_limit
    k8s/container/cpu/request: kubernetes.container_cpu_request
    k8s/container/ephemeral-storage/limit: kubernetes.container_ephemeral_storage_limit
    k8s/container/ephemeral-storage/request: kubernetes.container_ephemeral_storage_request
    k8s/container/memory/limit: kubernetes.container_memory_limit
    k8s/container/memory/request: kubernetes.container_memory_request
    k8s/container/ready: kubernetes.container_ready
    k8s/container/restarts: kubernetes.container_restart_count
    k8s/cronjob/active_jobs: kubernetes.cronjob.active
    k8s/daemon_set/current_scheduled_nodes: kubernetes.daemon_set.current_scheduled
    k8s/daemon_set/desired_scheduled_nodes: kubernetes.daemon_set.desired_scheduled
    k8s/daemon_set/misscheduled_nodes: kubernetes.daemon_set.misscheduled
    k8s/daemon_set/ready_nodes: kubernetes.daemon_set.ready
    k8s/deployment/available: kubernetes.deployment.available
    k8s/deployment/desired: kubernetes.deployment.desired
    k8s/job/active_pods: kubernetes.job.active
    k8s/job/desired_successful_pods: kubernetes.job.completions
    k8s/job/failed_pods: kubernetes.job.failed
    k8s/job/max_parallel_pods: kubernetes.job.parallelism
    k8s/job/successful_pods: kubernetes.job.succeeded
    k8s/hpa/current_replicas: kubernetes.hpa.status.current_replicas
    k8s/hpa/desired_replicas: kubernetes.hpa.status.desired_replicas
    k8s/hpa/max_replicas: kubernetes.hpa.spec.max_replicas
    k8s/hpa/min_replicas: kubernetes.hpa.spec.min_replicas
    k8s/namespace/phase: kubernetes.namespace_phase
    k8s/node/condition_memory_pressure: kubernetes.node_memory_pressure
    k8s/node/condition_network_unavailable: kubernetes.node_network_unavailable
    k8s/node/condition_out_of_disk: kubernetes.node_out_of_disk
    k8s/node/condition_p_i_d_pressure: kubernetes.node_p_i_d_pressure
    k8s/node/condition_ready: kubernetes.node_ready
    k8s/pod/phase: kubernetes.pod_phase
    k8s/replica_set/available: kubernetes.replica_set.available
    k8s/replica_set/desired: kubernetes.replica_set.desired
    k8s/replication_controller/available: kubernetes.replication_controller.available
    k8s/replication_controller/desired: kubernetes.replication_controller.desired
    k8s/resource_quota/hard_limt: kubernetes.resource_quota_hard
    k8s/resource_quota/used: kubernetes.resource_quota_used
    k8s/stateful_set/current_pods: kubernetes.stateful_set.current
    k8s/stateful_set/desired_pods: kubernetes.stateful_set.desired
    k8s/stateful_set/ready_pods: kubernetes.stateful_set.ready
    k8s/stateful_set/updated_pods: kubernetes.stateful_set.updated

    # load metrics
    system.cpu.load_average.15m: load.longterm
    system.cpu.load_average.5m: load.midterm
    system.cpu.load_average.1m: load.shortterm

# container network metrics
- action: split_metric
  metric_name: k8s.pod.network.io
  dimension_key: direction
  mapping:
    receive: pod_network_receive_bytes_total
    transmit: pod_network_transmit_bytes_total
- action: split_metric
  metric_name: k8s.pod.network.errors
  dimension_key: direction
  mapping:
    receive: pod_network_receive_errors_total
    transmit: pod_network_transmit_errors_total

# compute cpu utilization
- action: delta_metric
  mapping:
    system.cpu.time: system.cpu.delta
- action: copy_metrics
  mapping:
    system.cpu.delta: system.cpu.usage
  dimension_key: state
  dimension_values:
    interrupt: true
    nice: true
    softirq: true
    steal: true
    system: true
    user: true
    wait: true
- action: aggregate_metric
  metric_name: system.cpu.usage
  aggregation_method: sum
  without_dimensions:
  - state
  - cpu
- action: copy_metrics
  mapping:
    system.cpu.delta: system.cpu.total
- action: aggregate_metric
  metric_name: system.cpu.total
  aggregation_method: sum
  without_dimensions:
  - state
  - cpu
- action: calculate_new_metric
  metric_name: cpu.utilization
  operand1_metric: system.cpu.usage
  operand2_metric: system.cpu.total
  operator: /

# convert cpu metrics
- action: split_metric
  metric_name: system.cpu.time
  dimension_key: state
  mapping:
    idle: cpu.idle
    interrupt: cpu.interrupt
    system: cpu.system
    user: cpu.user
    steal: cpu.steal
    wait: cpu.wait
    softirq: cpu.softirq
    nice: cpu.nice
- action: multiply_float
  scale_factors_float:
    container_cpu_utilization: 100
    cpu.idle: 100
    cpu.interrupt: 100
    cpu.system: 100
    cpu.user: 100
    cpu.steal: 100
    cpu.wait: 100
    cpu.softirq: 100
    cpu.nice: 100
- action: convert_values
  types_mapping:
    container_cpu_utilization: int
    cpu.idle: int
    cpu.interrupt: int
    cpu.system: int
    cpu.user: int
    cpu.steal: int
    cpu.wait: int
    cpu.softirq: int
    cpu.nice: int

# compute cpu.num_processors
- action: copy_metrics
  mapping:
    cpu.idle: cpu.num_processors
- action: aggregate_metric
  metric_name: cpu.num_processors
  aggregation_method: count
  without_dimensions:
  - cpu

# compute memory.total
- action: copy_metrics
  mapping:
    system.memory.usage: memory.total
  dimension_key: state
  dimension_values:
    buffered: true
    cached: true
    free: true
    used: true
- action: aggregate_metric
  metric_name: memory.total
  aggregation_method: sum
  without_dimensions:
  - state

# convert memory metrics
- action: split_metric
  metric_name: system.memory.usage
  dimension_key: state
  mapping:
    buffered: memory.buffered
    cached: memory.cached
    free: memory.free
    slab_reclaimable: memory.slab_recl
    slab_unreclaimable: memory.slab_unrecl
    used: memory.used

# calculate disk.total
- action: copy_metrics
  mapping:
    system.filesystem.usage: disk.total
- action: aggregate_metric
  metric_name: disk.total
  aggregation_method: sum
  without_dimensions:
    - state

# calculate disk.summary_total
- action: copy_metrics
  mapping:
    system.filesystem.usage: disk.summary_total
- action: aggregate_metric
  metric_name: disk.summary_total
  aggregation_method: sum
  without_dimensions:
    - state
    - device

# convert filesystem metrics
- action: split_metric
  metric_name: system.filesystem.usage
  dimension_key: state
  mapping:
    free: df_complex.free
    reserved: df_complex.reserved
    used: df_complex.used
- action: split_metric
  metric_name: system.filesystem.inodes.usage
  dimension_key: state
  mapping:
    free: df_inodes.free
    used: df_inodes.used

# df_complex.used_total
- action: copy_metrics
  mapping:
    df_complex.used: df_complex.used_total
- action: aggregate_metric
  metric_name: df_complex.used_total
  aggregation_method: sum
  without_dimensions:
  - device

# disk utilization
- action: calculate_new_metric
  metric_name: disk.utilization
  operand1_metric: df_complex.used
  operand2_metric: disk.total
  operator: /
- action: multiply_float
  scale_factors_float:
    disk.utilization: 100

- action: calculate_new_metric
  metric_name: disk.summary_utilization
  operand1_metric: df_complex.used_total
  operand2_metric: disk.summary_total
  operator: /
- action: multiply_float
  scale_factors_float:
    disk.summary_utilization: 100

# convert disk I/O metrics
- action: copy_metrics
  mapping:
    system.disk.ops: disk.ops
- action: aggregate_metric
  metric_name: disk.ops
  aggregation_method: sum
  without_dimensions:
   - direction
   - device
- action: delta_metric
  mapping:
    disk.ops: disk_ops.total
- action: rename_dimension_keys
  metric_names:
    system.disk.merged: true
    system.disk.io: true
    system.disk.ops: true
    system.disk.time: true
  mapping:
    device: disk
- action: split_metric
  metric_name: system.disk.merged
  dimension_key: direction
  mapping:
    read: disk_merged.read
    write: disk_merged.write
- action: split_metric
  metric_name: system.disk.io
  dimension_key: direction
  mapping:
    read: disk_octets.read
    write: disk_octets.write
- action: split_metric
  metric_name: system.disk.ops
  dimension_key: direction
  mapping:
    read: disk_ops.read
    write: disk_ops.write
- action: split_metric
  metric_name: system.disk.time
  dimension_key: direction
  mapping:
    read: disk_time.read
    write: disk_time.write
- action: delta_metric
  mapping:
    system.disk.pending_operations: disk_ops.pending

# convert network I/O metrics
- action: copy_metrics
  mapping:
    system.network.io: network.total
  dimension_key: direction
  dimension_values:
    receive: true
    transmit: true
- action: aggregate_metric
  metric_name: network.total
  aggregation_method: sum
  without_dimensions:
  - direction
  - interface
- action: split_metric
  metric_name: system.network.dropped_packets
  dimension_key: direction
  mapping:
    receive: if_dropped.rx
    transmit: if_dropped.tx
- action: split_metric
  metric_name: system.network.errors
  dimension_key: direction
  mapping:
    receive: if_errors.rx
    transmit: if_errors.tx
- action: split_metric
  metric_name: system.network.io
  dimension_key: direction
  mapping:
    receive: if_octets.rx
    transmit: if_octets.tx
- action: split_metric
  metric_name: system.network.packets
  dimension_key: direction
  mapping:
    receive: if_packets.rx
    transmit: if_packets.tx

# memory utilization
- action: calculate_new_metric
  metric_name: memory.utilization
  operand1_metric: memory.used
  operand2_metric: memory.total
  operator: /

- action: multiply_float
  scale_factors_float:
    memory.utilization: 100
    cpu.utilization: 100

# remove redundant metrics
- action: drop_metrics
  metric_names:
    df_complex.used_total: true
    disk.ops: true
    disk.summary_total: true
    disk.total: true
    system.cpu.usage: true
    system.cpu.total: true
    system.cpu.delta: true
`
)
<commit_msg>sfxexporter: Update k8s_cluster metric translations (#1121)<commit_after>// Copyright 2019, OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package translation

const (
	// DefaultTranslationRulesYaml defines default translation rules that will be applied to metrics if
	// config.SendCompatibleMetrics set to true and config.TranslationRules not specified explicitly.
	// Keep it in YAML format to be able to easily copy and paste it in config if modifications needed.
	DefaultTranslationRulesYaml = `
translation_rules:

- action: rename_dimension_keys
  mapping:

    # dimensions
    container.image.name: container_image
    k8s.container.name: container_spec_name
    k8s.cluster.name: kubernetes_cluster
    k8s.daemonset.name: kubernetes_name
    k8s.daemonset.uid: kubernetes_uid
    k8s.deployment.name: kubernetes_name
    k8s.deployment.uid: kubernetes_uid
    k8s.hpa.name: kubernetes_name
    k8s.hpa.uid: kubernetes_uid
    k8s.namespace.name: kubernetes_namespace
    k8s.node.name: kubernetes_node
    k8s.node.uid: kubernetes_node_uid
    k8s.pod.name: kubernetes_pod_name
    k8s.pod.uid: kubernetes_pod_uid
    k8s.replicaset.name: kubernetes_name
    k8s.replicaset.uid: kubernetes_uid
    k8s.replicationcontroller.name: kubernetes_name
    k8s.replicationcontroller.uid: kubernetes_uid
    k8s.resourcequota.name: quota_name
    k8s.resourcequota.uid: kubernetes_uid
    k8s.statefulset.name: kubernetes_name
    k8s.statefulset.uid: kubernetes_uid

    # properties
    cronjob_uid: cronJob_uid
    cronjob: cronJob
    daemonset_uid: daemonSet_uid
    daemonset: daemonSet
    k8s.workload.kind: kubernetes_workload
    k8s.workload.name: kubernetes_workload_name
    replicaset_uid: replicaSet_uid
    replicaset: replicaSet
    statefulset_uid: statefulSet_uid
    statefulset: statefulSet

- action: rename_metrics
  mapping:

    # kubeletstats receiver metrics
    container.cpu.time: container_cpu_utilization
    container.filesystem.available: container_fs_available_bytes
    container.filesystem.capacity: container_fs_capacity_bytes
    container.filesystem.usage: container_fs_usage_bytes
    container.memory.available: container_memory_available_bytes
    container.memory.major_page_faults: container_memory_major_page_faults
    container.memory.page_faults: container_memory_page_faults
    container.memory.rss: container_memory_rss_bytes
    container.memory.usage: container_memory_usage_bytes
    container.memory.working_set: container_memory_working_set_bytes

    # k8s cluster receiver metrics
    k8s.container.cpu_limit: kubernetes.container_cpu_limit
    k8s.container.cpu_request: kubernetes.container_cpu_request
    k8s.container.ephemeral-storage_limit: kubernetes.container_ephemeral_storage_limit
    k8s.container.ephemeral-storage_request: kubernetes.container_ephemeral_storage_request
    k8s.container.memory_limit: kubernetes.container_memory_limit
    k8s.container.memory_request: kubernetes.container_memory_request
    k8s.container.ready: kubernetes.container_ready
    k8s.container.restarts: kubernetes.container_restart_count
    k8s.cronjob.active_jobs: kubernetes.cronjob.active
    k8s.daemonset.current_scheduled_nodes: kubernetes.daemon_set.current_scheduled
    k8s.daemonset.desired_scheduled_nodes: kubernetes.daemon_set.desired_scheduled
    k8s.daemonset.misscheduled_nodes: kubernetes.daemon_set.misscheduled
    k8s.daemonset.ready_nodes: kubernetes.daemon_set.ready
    k8s.deployment.available: kubernetes.deployment.available
    k8s.deployment.desired: kubernetes.deployment.desired
    k8s.job.active_pods: kubernetes.job.active
    k8s.job.desired_successful_pods: kubernetes.job.completions
    k8s.job.failed_pods: kubernetes.job.failed
    k8s.job.max_parallel_pods: kubernetes.job.parallelism
    k8s.job.successful_pods: kubernetes.job.succeeded
    k8s.hpa.current_replicas: kubernetes.hpa.status.current_replicas
    k8s.hpa.desired_replicas: kubernetes.hpa.status.desired_replicas
    k8s.hpa.max_replicas: kubernetes.hpa.spec.max_replicas
    k8s.hpa.min_replicas: kubernetes.hpa.spec.min_replicas
    k8s.namespace.phase: kubernetes.namespace_phase
    k8s.node.condition_memory_pressure: kubernetes.node_memory_pressure
    k8s.node.condition_network_unavailable: kubernetes.node_network_unavailable
    k8s.node.condition_out_of_disk: kubernetes.node_out_of_disk
    k8s.node.condition_p_i_d_pressure: kubernetes.node_p_i_d_pressure
    k8s.node.condition_ready: kubernetes.node_ready
    k8s.pod.phase: kubernetes.pod_phase
    k8s.replicaset.available: kubernetes.replica_set.available
    k8s.replicaset.desired: kubernetes.replica_set.desired
    k8s.replication_controller.available: kubernetes.replication_controller.available
    k8s.replication_controller.desired: kubernetes.replication_controller.desired
    k8s.resource_quota.hard_limit: kubernetes.resource_quota_hard
    k8s.resource_quota.used: kubernetes.resource_quota_used
    k8s.statefulset.current_pods: kubernetes.stateful_set.current
    k8s.statefulset.desired_pods: kubernetes.stateful_set.desired
    k8s.statefulset.ready_pods: kubernetes.stateful_set.ready
    k8s.statefulset.updated_pods: kubernetes.stateful_set.updated

    # load metrics
    system.cpu.load_average.15m: load.longterm
    system.cpu.load_average.5m: load.midterm
    system.cpu.load_average.1m: load.shortterm

# container network metrics
- action: split_metric
  metric_name: k8s.pod.network.io
  dimension_key: direction
  mapping:
    receive: pod_network_receive_bytes_total
    transmit: pod_network_transmit_bytes_total
- action: split_metric
  metric_name: k8s.pod.network.errors
  dimension_key: direction
  mapping:
    receive: pod_network_receive_errors_total
    transmit: pod_network_transmit_errors_total

# compute cpu utilization
- action: delta_metric
  mapping:
    system.cpu.time: system.cpu.delta
- action: copy_metrics
  mapping:
    system.cpu.delta: system.cpu.usage
  dimension_key: state
  dimension_values:
    interrupt: true
    nice: true
    softirq: true
    steal: true
    system: true
    user: true
    wait: true
- action: aggregate_metric
  metric_name: system.cpu.usage
  aggregation_method: sum
  without_dimensions:
  - state
  - cpu
- action: copy_metrics
  mapping:
    system.cpu.delta: system.cpu.total
- action: aggregate_metric
  metric_name: system.cpu.total
  aggregation_method: sum
  without_dimensions:
  - state
  - cpu
- action: calculate_new_metric
  metric_name: cpu.utilization
  operand1_metric: system.cpu.usage
  operand2_metric: system.cpu.total
  operator: /

# convert cpu metrics
- action: split_metric
  metric_name: system.cpu.time
  dimension_key: state
  mapping:
    idle: cpu.idle
    interrupt: cpu.interrupt
    system: cpu.system
    user: cpu.user
    steal: cpu.steal
    wait: cpu.wait
    softirq: cpu.softirq
    nice: cpu.nice
- action: multiply_float
  scale_factors_float:
    container_cpu_utilization: 100
    cpu.idle: 100
    cpu.interrupt: 100
    cpu.system: 100
    cpu.user: 100
    cpu.steal: 100
    cpu.wait: 100
    cpu.softirq: 100
    cpu.nice: 100
- action: convert_values
  types_mapping:
    container_cpu_utilization: int
    cpu.idle: int
    cpu.interrupt: int
    cpu.system: int
    cpu.user: int
    cpu.steal: int
    cpu.wait: int
    cpu.softirq: int
    cpu.nice: int

# compute cpu.num_processors
- action: copy_metrics
  mapping:
    cpu.idle: cpu.num_processors
- action: aggregate_metric
  metric_name: cpu.num_processors
  aggregation_method: count
  without_dimensions:
  - cpu

# compute memory.total
- action: copy_metrics
  mapping:
    system.memory.usage: memory.total
  dimension_key: state
  dimension_values:
    buffered: true
    cached: true
    free: true
    used: true
- action: aggregate_metric
  metric_name: memory.total
  aggregation_method: sum
  without_dimensions:
  - state

# convert memory metrics
- action: split_metric
  metric_name: system.memory.usage
  dimension_key: state
  mapping:
    buffered: memory.buffered
    cached: memory.cached
    free: memory.free
    slab_reclaimable: memory.slab_recl
    slab_unreclaimable: memory.slab_unrecl
    used: memory.used

# calculate disk.total
- action: copy_metrics
  mapping:
    system.filesystem.usage: disk.total
- action: aggregate_metric
  metric_name: disk.total
  aggregation_method: sum
  without_dimensions:
    - state

# calculate disk.summary_total
- action: copy_metrics
  mapping:
    system.filesystem.usage: disk.summary_total
- action: aggregate_metric
  metric_name: disk.summary_total
  aggregation_method: sum
  without_dimensions:
    - state
    - device

# convert filesystem metrics
- action: split_metric
  metric_name: system.filesystem.usage
  dimension_key: state
  mapping:
    free: df_complex.free
    reserved: df_complex.reserved
    used: df_complex.used
- action: split_metric
  metric_name: system.filesystem.inodes.usage
  dimension_key: state
  mapping:
    free: df_inodes.free
    used: df_inodes.used

# df_complex.used_total
- action: copy_metrics
  mapping:
    df_complex.used: df_complex.used_total
- action: aggregate_metric
  metric_name: df_complex.used_total
  aggregation_method: sum
  without_dimensions:
  - device

# disk utilization
- action: calculate_new_metric
  metric_name: disk.utilization
  operand1_metric: df_complex.used
  operand2_metric: disk.total
  operator: /
- action: multiply_float
  scale_factors_float:
    disk.utilization: 100

- action: calculate_new_metric
  metric_name: disk.summary_utilization
  operand1_metric: df_complex.used_total
  operand2_metric: disk.summary_total
  operator: /
- action: multiply_float
  scale_factors_float:
    disk.summary_utilization: 100

# convert disk I/O metrics
- action: copy_metrics
  mapping:
    system.disk.ops: disk.ops
- action: aggregate_metric
  metric_name: disk.ops
  aggregation_method: sum
  without_dimensions:
   - direction
   - device
- action: delta_metric
  mapping:
    disk.ops: disk_ops.total
- action: rename_dimension_keys
  metric_names:
    system.disk.merged: true
    system.disk.io: true
    system.disk.ops: true
    system.disk.time: true
  mapping:
    device: disk
- action: split_metric
  metric_name: system.disk.merged
  dimension_key: direction
  mapping:
    read: disk_merged.read
    write: disk_merged.write
- action: split_metric
  metric_name: system.disk.io
  dimension_key: direction
  mapping:
    read: disk_octets.read
    write: disk_octets.write
- action: split_metric
  metric_name: system.disk.ops
  dimension_key: direction
  mapping:
    read: disk_ops.read
    write: disk_ops.write
- action: split_metric
  metric_name: system.disk.time
  dimension_key: direction
  mapping:
    read: disk_time.read
    write: disk_time.write
- action: delta_metric
  mapping:
    system.disk.pending_operations: disk_ops.pending

# convert network I/O metrics
- action: copy_metrics
  mapping:
    system.network.io: network.total
  dimension_key: direction
  dimension_values:
    receive: true
    transmit: true
- action: aggregate_metric
  metric_name: network.total
  aggregation_method: sum
  without_dimensions:
  - direction
  - interface
- action: split_metric
  metric_name: system.network.dropped_packets
  dimension_key: direction
  mapping:
    receive: if_dropped.rx
    transmit: if_dropped.tx
- action: split_metric
  metric_name: system.network.errors
  dimension_key: direction
  mapping:
    receive: if_errors.rx
    transmit: if_errors.tx
- action: split_metric
  metric_name: system.network.io
  dimension_key: direction
  mapping:
    receive: if_octets.rx
    transmit: if_octets.tx
- action: split_metric
  metric_name: system.network.packets
  dimension_key: direction
  mapping:
    receive: if_packets.rx
    transmit: if_packets.tx

# memory utilization
- action: calculate_new_metric
  metric_name: memory.utilization
  operand1_metric: memory.used
  operand2_metric: memory.total
  operator: /

- action: multiply_float
  scale_factors_float:
    memory.utilization: 100
    cpu.utilization: 100

# remove redundant metrics
- action: drop_metrics
  metric_names:
    df_complex.used_total: true
    disk.ops: true
    disk.summary_total: true
    disk.total: true
    system.cpu.usage: true
    system.cpu.total: true
    system.cpu.delta: true
`
)
<|endoftext|>


<commit_before>// Copyright 2016-2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os"
	"regexp"

	"github.com/cilium/cilium/pkg/endpoint"
	"github.com/cilium/cilium/pkg/policy"

	"github.com/spf13/cobra"
	"k8s.io/client-go/util/jsonpath"
)

func Fatalf(msg string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "Error: %s\n", fmt.Sprintf(msg, args...))
	os.Exit(1)
}

func Usagef(cmd *cobra.Command, msg string, args ...interface{}) {
	txt := fmt.Sprintf(msg, args...)
	fmt.Fprintf(os.Stderr, "Error: %s\n\n", txt)
	cmd.Help()
	os.Exit(1)
}

func requireEndpointID(cmd *cobra.Command, args []string) {
	if len(args) < 1 {
		Usagef(cmd, "Missing endpoint id argument")
	}

	if id := policy.ReservedIdentities[args[0]]; id == policy.IdentityUnknown {
		_, _, err := endpoint.ValidateID(args[0])

		if err != nil {
			Fatalf("Cannot parse endpoint id \"%s\": %s", args[0], err)
		}
	}
}

func requireEndpointIDorGlobal(cmd *cobra.Command, args []string) {
	if len(args) < 1 {
		Usagef(cmd, "Missing endpoint id or 'global' argument")
	}

	if args[0] != "global" {
		requireEndpointID(cmd, args)
	}
}

func requirePath(cmd *cobra.Command, args []string) {
	if len(args) < 1 {
		Usagef(cmd, "Missing path argument")
	}

	if args[0] == "" {
		Usagef(cmd, "Empty path argument")
	}
}

func requireServiceID(cmd *cobra.Command, args []string) {
	if len(args) < 1 {
		Usagef(cmd, "Missing service id argument")
	}

	if args[0] == "" {
		Usagef(cmd, "Empty service id argument")
	}
}

var dumpOutput string

//AddMultipleOutput adds the -o|--output option to any cmd to export to json
func AddMultipleOutput(cmd *cobra.Command) {
	cmd.Flags().StringVarP(&dumpOutput, "output", "o", "", "json| jsonpath='{}'")
}

//OutputPrinter receives an interface and dump the data using the --output flag.
//ATM only json or jsonpath. In the future yaml
func OutputPrinter(data interface{}) error {
	var re = regexp.MustCompile(`^jsonpath\=(.*)`)

	if dumpOutput == "json" {
		return dumpJSON(data, "")
	}

	if re.MatchString(dumpOutput) {
		return dumpJSON(data, re.ReplaceAllString(dumpOutput, "$1"))
	}

	return fmt.Errorf("Couldn't found output printer")
}

// dumpJSON dump the data variable to the stdout as json.
// If somethings fail, it'll return an error
// If jsonPath is passed, it'll run the json query over data var.
func dumpJSON(data interface{}, jsonPath string) error {

	if len(jsonPath) == 0 {
		result, err := json.Marshal(data)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Couldn't marshal to json: '%s'\n", err)
			return err
		}
		fmt.Println(string(result))
		return nil
	}

	parser := jsonpath.New("").AllowMissingKeys(true)
	if err := parser.Parse(jsonPath); err != nil {
		fmt.Fprintf(os.Stderr, "Couldn't parse jsonpath expression: '%s'\n", err)
		return err
	}

	buf := new(bytes.Buffer)
	if err := parser.Execute(buf, data); err != nil {
		fmt.Fprintf(os.Stderr, "Couldn't parse jsonpath expression: '%s'\n", err)
		return err

	}
	fmt.Println(buf.String())
	return nil
}
<commit_msg>cmd: Comment Fatalf/Usagef functions<commit_after>// Copyright 2016-2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os"
	"regexp"

	"github.com/cilium/cilium/pkg/endpoint"
	"github.com/cilium/cilium/pkg/policy"

	"github.com/spf13/cobra"
	"k8s.io/client-go/util/jsonpath"
)

// Fatalf prints the Printf formatted message to stderr and exits the program
// Note: os.Exit(1) is not recoverable
func Fatalf(msg string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "Error: %s\n", fmt.Sprintf(msg, args...))
	os.Exit(1)
}

// Usagef prints the Printf formatted message to stderr, prints usage help and
// exits the program
// Note: os.Exit(1) is not recoverable
func Usagef(cmd *cobra.Command, msg string, args ...interface{}) {
	txt := fmt.Sprintf(msg, args...)
	fmt.Fprintf(os.Stderr, "Error: %s\n\n", txt)
	cmd.Help()
	os.Exit(1)
}

func requireEndpointID(cmd *cobra.Command, args []string) {
	if len(args) < 1 {
		Usagef(cmd, "Missing endpoint id argument")
	}

	if id := policy.ReservedIdentities[args[0]]; id == policy.IdentityUnknown {
		_, _, err := endpoint.ValidateID(args[0])

		if err != nil {
			Fatalf("Cannot parse endpoint id \"%s\": %s", args[0], err)
		}
	}
}

func requireEndpointIDorGlobal(cmd *cobra.Command, args []string) {
	if len(args) < 1 {
		Usagef(cmd, "Missing endpoint id or 'global' argument")
	}

	if args[0] != "global" {
		requireEndpointID(cmd, args)
	}
}

func requirePath(cmd *cobra.Command, args []string) {
	if len(args) < 1 {
		Usagef(cmd, "Missing path argument")
	}

	if args[0] == "" {
		Usagef(cmd, "Empty path argument")
	}
}

func requireServiceID(cmd *cobra.Command, args []string) {
	if len(args) < 1 {
		Usagef(cmd, "Missing service id argument")
	}

	if args[0] == "" {
		Usagef(cmd, "Empty service id argument")
	}
}

var dumpOutput string

//AddMultipleOutput adds the -o|--output option to any cmd to export to json
func AddMultipleOutput(cmd *cobra.Command) {
	cmd.Flags().StringVarP(&dumpOutput, "output", "o", "", "json| jsonpath='{}'")
}

//OutputPrinter receives an interface and dump the data using the --output flag.
//ATM only json or jsonpath. In the future yaml
func OutputPrinter(data interface{}) error {
	var re = regexp.MustCompile(`^jsonpath\=(.*)`)

	if dumpOutput == "json" {
		return dumpJSON(data, "")
	}

	if re.MatchString(dumpOutput) {
		return dumpJSON(data, re.ReplaceAllString(dumpOutput, "$1"))
	}

	return fmt.Errorf("Couldn't found output printer")
}

// dumpJSON dump the data variable to the stdout as json.
// If somethings fail, it'll return an error
// If jsonPath is passed, it'll run the json query over data var.
func dumpJSON(data interface{}, jsonPath string) error {

	if len(jsonPath) == 0 {
		result, err := json.Marshal(data)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Couldn't marshal to json: '%s'\n", err)
			return err
		}
		fmt.Println(string(result))
		return nil
	}

	parser := jsonpath.New("").AllowMissingKeys(true)
	if err := parser.Parse(jsonPath); err != nil {
		fmt.Fprintf(os.Stderr, "Couldn't parse jsonpath expression: '%s'\n", err)
		return err
	}

	buf := new(bytes.Buffer)
	if err := parser.Execute(buf, data); err != nil {
		fmt.Fprintf(os.Stderr, "Couldn't parse jsonpath expression: '%s'\n", err)
		return err

	}
	fmt.Println(buf.String())
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2016 TiKV Project Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package pd

import (
	"context"
	"testing"
	"time"

	. "github.com/pingcap/check"
	"github.com/pingcap/errors"
	"github.com/pingcap/kvproto/pkg/pdpb"
	"github.com/tikv/pd/pkg/testutil"
	"go.uber.org/goleak"
	"google.golang.org/grpc"
)

func Test(t *testing.T) {
	TestingT(t)
}

func TestMain(m *testing.M) {
	goleak.VerifyTestMain(m, testutil.LeakOptions...)
}

var _ = Suite(&testClientSuite{})

type testClientSuite struct{}

func (s *testClientSuite) TestTsLessEqual(c *C) {
	c.Assert(tsLessEqual(9, 9, 9, 9), IsTrue)
	c.Assert(tsLessEqual(8, 9, 9, 8), IsTrue)
	c.Assert(tsLessEqual(9, 8, 8, 9), IsFalse)
	c.Assert(tsLessEqual(9, 8, 9, 6), IsFalse)
	c.Assert(tsLessEqual(9, 6, 9, 8), IsTrue)
}

func (s *testClientSuite) TestUpdateURLs(c *C) {
	members := []*pdpb.Member{
		{Name: "pd4", ClientUrls: []string{"tmp//pd4"}},
		{Name: "pd1", ClientUrls: []string{"tmp//pd1"}},
		{Name: "pd3", ClientUrls: []string{"tmp//pd3"}},
		{Name: "pd2", ClientUrls: []string{"tmp//pd2"}},
	}
	getURLs := func(ms []*pdpb.Member) (urls []string) {
		for _, m := range ms {
			urls = append(urls, m.GetClientUrls()[0])
		}
		return
	}
	cli := &baseClient{}
	cli.updateURLs(members[1:])
	c.Assert(cli.urls, DeepEquals, getURLs([]*pdpb.Member{members[1], members[3], members[2]}))
	cli.updateURLs(members[1:])
	c.Assert(cli.urls, DeepEquals, getURLs([]*pdpb.Member{members[1], members[3], members[2]}))
	cli.updateURLs(members)
	c.Assert(cli.urls, DeepEquals, getURLs([]*pdpb.Member{members[1], members[3], members[2], members[0]}))
}

var _ = Suite(&testClientCtxSuite{})

type testClientCtxSuite struct{}

func (s *testClientCtxSuite) TestClientCtx(c *C) {
	start := time.Now()
	ctx, cancel := context.WithTimeout(context.TODO(), time.Second*3)
	defer cancel()
	_, err := NewClientWithContext(ctx, []string{"127.0.0.1:8080"}, SecurityOption{})
	c.Assert(err, NotNil)
	c.Assert(time.Since(start), Less, time.Second*4)
}

func (s *testClientCtxSuite) TestClientWithRetry(c *C) {
	start := time.Now()
	_, err := NewClientWithContext(context.TODO(), []string{"127.0.0.1:8080"}, SecurityOption{}, WithMaxErrorRetry(5))
	c.Assert(err, NotNil)
	c.Assert(time.Since(start), Less, time.Second*6)
}

var _ = Suite(&testClientDialOptionSuite{})

type testClientDialOptionSuite struct{}

func (s *testClientDialOptionSuite) TestGRPCDialOption(c *C) {
	start := time.Now()
	ctx, cancel := context.WithTimeout(context.TODO(), 500*time.Millisecond)
	defer cancel()
	// nolint
	cli := &baseClient{
		urls:                 []string{"http://127.0.0.1:8080"},
		checkLeaderCh:        make(chan struct{}, 1),
		checkTSODispatcherCh: make(chan struct{}, 1),
		ctx:                  ctx,
		cancel:               cancel,
		security:             SecurityOption{},
		gRPCDialOptions:      []grpc.DialOption{grpc.WithBlock()},
	}

	err := cli.updateMember()
	c.Assert(err, NotNil)
	c.Assert(time.Since(start), Greater, 500*time.Millisecond)
}

var _ = Suite(&testTsoRequestSuite{})

type testTsoRequestSuite struct{}

func (s *testTsoRequestSuite) TestTsoRequestWait(c *C) {
	ctx, cancel := context.WithCancel(context.Background())
	req := &tsoRequest{
		done:       make(chan error, 1),
		physical:   0,
		logical:    0,
		requestCtx: context.TODO(),
		clientCtx:  ctx,
	}
	cancel()
	_, _, err := req.Wait()
	c.Assert(errors.Cause(err), Equals, context.Canceled)

	ctx, cancel = context.WithCancel(context.Background())
	req = &tsoRequest{
		done:       make(chan error, 1),
		physical:   0,
		logical:    0,
		requestCtx: ctx,
		clientCtx:  context.TODO(),
	}
	cancel()
	_, _, err = req.Wait()
	c.Assert(errors.Cause(err), Equals, context.Canceled)
}
<commit_msg>client: fix etcd proxy loop detection error in unit test (#3576)<commit_after>// Copyright 2016 TiKV Project Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package pd

import (
	"context"
	"testing"
	"time"

	. "github.com/pingcap/check"
	"github.com/pingcap/errors"
	"github.com/pingcap/kvproto/pkg/pdpb"
	"github.com/tikv/pd/pkg/testutil"
	"go.uber.org/goleak"
	"google.golang.org/grpc"
)

func Test(t *testing.T) {
	TestingT(t)
}

func TestMain(m *testing.M) {
	goleak.VerifyTestMain(m, testutil.LeakOptions...)
}

var _ = Suite(&testClientSuite{})

type testClientSuite struct{}

func (s *testClientSuite) TestTsLessEqual(c *C) {
	c.Assert(tsLessEqual(9, 9, 9, 9), IsTrue)
	c.Assert(tsLessEqual(8, 9, 9, 8), IsTrue)
	c.Assert(tsLessEqual(9, 8, 8, 9), IsFalse)
	c.Assert(tsLessEqual(9, 8, 9, 6), IsFalse)
	c.Assert(tsLessEqual(9, 6, 9, 8), IsTrue)
}

func (s *testClientSuite) TestUpdateURLs(c *C) {
	members := []*pdpb.Member{
		{Name: "pd4", ClientUrls: []string{"tmp://pd4"}},
		{Name: "pd1", ClientUrls: []string{"tmp://pd1"}},
		{Name: "pd3", ClientUrls: []string{"tmp://pd3"}},
		{Name: "pd2", ClientUrls: []string{"tmp://pd2"}},
	}
	getURLs := func(ms []*pdpb.Member) (urls []string) {
		for _, m := range ms {
			urls = append(urls, m.GetClientUrls()[0])
		}
		return
	}
	cli := &baseClient{}
	cli.updateURLs(members[1:])
	c.Assert(cli.urls, DeepEquals, getURLs([]*pdpb.Member{members[1], members[3], members[2]}))
	cli.updateURLs(members[1:])
	c.Assert(cli.urls, DeepEquals, getURLs([]*pdpb.Member{members[1], members[3], members[2]}))
	cli.updateURLs(members)
	c.Assert(cli.urls, DeepEquals, getURLs([]*pdpb.Member{members[1], members[3], members[2], members[0]}))
}

const testClientURL = "tmp://test.url:5255"

var _ = Suite(&testClientCtxSuite{})

type testClientCtxSuite struct{}

func (s *testClientCtxSuite) TestClientCtx(c *C) {
	start := time.Now()
	ctx, cancel := context.WithTimeout(context.TODO(), time.Second*3)
	defer cancel()
	_, err := NewClientWithContext(ctx, []string{testClientURL}, SecurityOption{})
	c.Assert(err, NotNil)
	c.Assert(time.Since(start), Less, time.Second*5)
}

func (s *testClientCtxSuite) TestClientWithRetry(c *C) {
	start := time.Now()
	_, err := NewClientWithContext(context.TODO(), []string{testClientURL}, SecurityOption{}, WithMaxErrorRetry(5))
	c.Assert(err, NotNil)
	c.Assert(time.Since(start), Less, time.Second*10)
}

var _ = Suite(&testClientDialOptionSuite{})

type testClientDialOptionSuite struct{}

func (s *testClientDialOptionSuite) TestGRPCDialOption(c *C) {
	start := time.Now()
	ctx, cancel := context.WithTimeout(context.TODO(), 500*time.Millisecond)
	defer cancel()
	// nolint
	cli := &baseClient{
		urls:                 []string{testClientURL},
		checkLeaderCh:        make(chan struct{}, 1),
		checkTSODispatcherCh: make(chan struct{}, 1),
		ctx:                  ctx,
		cancel:               cancel,
		security:             SecurityOption{},
		gRPCDialOptions:      []grpc.DialOption{grpc.WithBlock()},
	}

	err := cli.updateMember()
	c.Assert(err, NotNil)
	c.Assert(time.Since(start), Greater, 500*time.Millisecond)
}

var _ = Suite(&testTsoRequestSuite{})

type testTsoRequestSuite struct{}

func (s *testTsoRequestSuite) TestTsoRequestWait(c *C) {
	ctx, cancel := context.WithCancel(context.Background())
	req := &tsoRequest{
		done:       make(chan error, 1),
		physical:   0,
		logical:    0,
		requestCtx: context.TODO(),
		clientCtx:  ctx,
	}
	cancel()
	_, _, err := req.Wait()
	c.Assert(errors.Cause(err), Equals, context.Canceled)

	ctx, cancel = context.WithCancel(context.Background())
	req = &tsoRequest{
		done:       make(chan error, 1),
		physical:   0,
		logical:    0,
		requestCtx: ctx,
		clientCtx:  context.TODO(),
	}
	cancel()
	_, _, err = req.Wait()
	c.Assert(errors.Cause(err), Equals, context.Canceled)
}
<|endoftext|>


<commit_before>// Copyright 2014 SteelSeries ApS.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package implements a basic LISP interpretor for embedding in a go program for scripting.
// This file implements the tokenizer.

package golisp

import (
	"fmt"
	"unicode"
)

const (
	ILLEGAL = iota
	SYMBOL
	NUMBER
	HEXNUMBER
	FLOAT
	STRING
	QUOTE
	BACKQUOTE
	COMMA
	COMMAAT
	LPAREN
	RPAREN
	LBRACKET
	RBRACKET
	PERIOD
	TRUE
	FALSE
	COMMENT
	EOF
)

type Tokenizer struct {
	LookaheadToken int
	LookaheadLit   string
	Source         string
	Position       int
}

func NewTokenizer(src string) *Tokenizer {
	t := &Tokenizer{Source: src}
	t.ConsumeToken()
	return t
}

func (self *Tokenizer) NextToken() (token int, lit string) {
	return self.LookaheadToken, self.LookaheadLit
}

func (self *Tokenizer) isSymbolCharacter(ch rune) bool {
	return unicode.IsLetter(ch) || unicode.IsNumber(ch) || ch == '*' || ch == '-' || ch == '?' || ch == '!' || ch == '_' || ch == '>'
}

func (self *Tokenizer) readSymbol() (token int, lit string) {
	start := self.Position
	for !self.isEof() && self.isSymbolCharacter(rune(self.Source[self.Position])) {
		self.Position++
	}
	return SYMBOL, self.Source[start:self.Position]
}

func isHexChar(ch rune) bool {
	switch ch {
	case 'a', 'b', 'c', 'd', 'e', 'f', 'A', 'B', 'C', 'D', 'E', 'F':
		return true
	default:
		return false
	}
}

func (self *Tokenizer) readNumber() (token int, lit string) {
	start := self.Position
	isHex := false
	isFloat := false
	sawDecimal := false
	for !self.isEof() {
		ch := rune(self.Source[self.Position])
		if ch == '.' && !sawDecimal {
			isFloat = true
			sawDecimal = true
			self.Position++
		} else if (start == self.Position) && ch == '-' {
			self.Position++
		} else if unicode.IsNumber(ch) {
			self.Position++
		} else if (start == self.Position-1) && ch == 'x' {
			isHex = true
			self.Position++
		} else if isHex && isHexChar(ch) {
			self.Position++
		} else {
			break
		}
	}

	lit = self.Source[start:self.Position]
	if isHex {
		token = HEXNUMBER
	} else if isFloat {
		token = FLOAT
	} else {
		token = NUMBER
	}
	return
}

func (self *Tokenizer) readString() (token int, lit string) {
	buffer := make([]rune, 0, 10)
	self.Position++
	for !self.isEof() && rune(self.Source[self.Position]) != '"' {
		if rune(self.Source[self.Position]) == '\\' {
			self.Position++
		}
		buffer = append(buffer, rune(self.Source[self.Position]))
		self.Position++
	}
	if self.isEof() {
		return EOF, ""
	}
	self.Position++
	return STRING, string(buffer)
}

func (self *Tokenizer) isEof() bool {
	return self.Position >= len(self.Source)
}

func (self *Tokenizer) isAlmostEof() bool {
	return self.Position == len(self.Source)-1
}

func (self *Tokenizer) readNextToken() (token int, lit string) {
	if self.isEof() {
		return EOF, ""
	}
	for unicode.IsSpace(rune(self.Source[self.Position])) {
		self.Position++
		if self.isEof() {
			return EOF, ""
		}
	}
	currentChar := rune(self.Source[self.Position])
	var nextChar rune
	if !self.isAlmostEof() {
		nextChar = rune(self.Source[self.Position+1])
	}
	if unicode.IsLetter(currentChar) || currentChar == '_' {
		return self.readSymbol()
	} else if unicode.IsNumber(currentChar) {
		return self.readNumber()
	} else if currentChar == '-' && unicode.IsNumber(nextChar) {
		return self.readNumber()
	} else if currentChar == '"' {
		return self.readString()
	} else if currentChar == '\'' {
		self.Position++
		return QUOTE, "'"
	} else if currentChar == '`' {
		self.Position++
		return BACKQUOTE, "`"
	} else if currentChar == ',' && nextChar == '@' {
		self.Position += 2
		return COMMAAT, ",@"
	} else if currentChar == ',' {
		self.Position++
		return COMMA, ","
	} else if currentChar == '(' {
		self.Position++
		return LPAREN, "("
	} else if currentChar == ')' {
		self.Position++
		return RPAREN, ")"
	} else if currentChar == '[' {
		self.Position++
		return LBRACKET, "["
	} else if currentChar == ']' {
		self.Position++
		return RBRACKET, "]"
	} else if currentChar == '.' {
		self.Position++
		return PERIOD, "."
	} else if currentChar == '-' && nextChar == '>' {
		self.Position += 2
		return SYMBOL, "->"
	} else if currentChar == '=' && nextChar == '>' {
		self.Position += 2
		return SYMBOL, "=>"
	} else if currentChar == '+' {
		self.Position++
		return SYMBOL, "+"
	} else if currentChar == '-' {
		self.Position++
		return SYMBOL, "-"
	} else if currentChar == '*' {
		self.Position++
		return SYMBOL, "*"
	} else if currentChar == '/' {
		self.Position++
		return SYMBOL, "/"
	} else if currentChar == '%' {
		self.Position++
		return SYMBOL, "%"
	} else if currentChar == '<' && nextChar == '=' {
		self.Position += 2
		return SYMBOL, "<="
	} else if currentChar == '<' {
		self.Position++
		return SYMBOL, "<"
	} else if currentChar == '>' && nextChar == '=' {
		self.Position += 2
		return SYMBOL, ">="
	} else if currentChar == '>' {
		self.Position++
		return SYMBOL, ">"
	} else if currentChar == '=' && nextChar == '=' {
		self.Position += 2
		return SYMBOL, "=="
	} else if currentChar == '=' {
		self.Position++
		return SYMBOL, "="
	} else if currentChar == '!' && nextChar == '=' {
		self.Position += 2
		return SYMBOL, "!="
	} else if currentChar == '!' {
		self.Position++
		return SYMBOL, "!"
	} else if currentChar == '#' {
		self.Position += 2
		if nextChar == 't' {
			return TRUE, "#t"
		} else {
			return FALSE, "#f"
		}
	} else if currentChar == ';' {
		start := self.Position
		for {
			if self.isEof() {
				return COMMENT, self.Source[start:]
			} else if self.Source[self.Position] == '\n' {
				return COMMENT, self.Source[start:self.Position]
			}
			self.Position++
		}
	} else {
		return ILLEGAL, fmt.Sprintf("%c", currentChar)
	}
}

func (self *Tokenizer) ConsumeToken() {
	self.LookaheadToken, self.LookaheadLit = self.readNextToken()
	// if self.LookaheadToken == COMMENT { // skip comments
	// 	self.ConsumeToken()
	// }
}
<commit_msg>Put comment consumption back in the tokenizer<commit_after>// Copyright 2014 SteelSeries ApS.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package implements a basic LISP interpretor for embedding in a go program for scripting.
// This file implements the tokenizer.

package golisp

import (
	"fmt"
	"unicode"
)

const (
	ILLEGAL = iota
	SYMBOL
	NUMBER
	HEXNUMBER
	FLOAT
	STRING
	QUOTE
	BACKQUOTE
	COMMA
	COMMAAT
	LPAREN
	RPAREN
	LBRACKET
	RBRACKET
	PERIOD
	TRUE
	FALSE
	COMMENT
	EOF
)

type Tokenizer struct {
	LookaheadToken int
	LookaheadLit   string
	Source         string
	Position       int
}

func NewTokenizer(src string) *Tokenizer {
	t := &Tokenizer{Source: src}
	t.ConsumeToken()
	return t
}

func (self *Tokenizer) NextToken() (token int, lit string) {
	return self.LookaheadToken, self.LookaheadLit
}

func (self *Tokenizer) isSymbolCharacter(ch rune) bool {
	return unicode.IsLetter(ch) || unicode.IsNumber(ch) || ch == '*' || ch == '-' || ch == '?' || ch == '!' || ch == '_' || ch == '>'
}

func (self *Tokenizer) readSymbol() (token int, lit string) {
	start := self.Position
	for !self.isEof() && self.isSymbolCharacter(rune(self.Source[self.Position])) {
		self.Position++
	}
	return SYMBOL, self.Source[start:self.Position]
}

func isHexChar(ch rune) bool {
	switch ch {
	case 'a', 'b', 'c', 'd', 'e', 'f', 'A', 'B', 'C', 'D', 'E', 'F':
		return true
	default:
		return false
	}
}

func (self *Tokenizer) readNumber() (token int, lit string) {
	start := self.Position
	isHex := false
	isFloat := false
	sawDecimal := false
	for !self.isEof() {
		ch := rune(self.Source[self.Position])
		if ch == '.' && !sawDecimal {
			isFloat = true
			sawDecimal = true
			self.Position++
		} else if (start == self.Position) && ch == '-' {
			self.Position++
		} else if unicode.IsNumber(ch) {
			self.Position++
		} else if (start == self.Position-1) && ch == 'x' {
			isHex = true
			self.Position++
		} else if isHex && isHexChar(ch) {
			self.Position++
		} else {
			break
		}
	}

	lit = self.Source[start:self.Position]
	if isHex {
		token = HEXNUMBER
	} else if isFloat {
		token = FLOAT
	} else {
		token = NUMBER
	}
	return
}

func (self *Tokenizer) readString() (token int, lit string) {
	buffer := make([]rune, 0, 10)
	self.Position++
	for !self.isEof() && rune(self.Source[self.Position]) != '"' {
		if rune(self.Source[self.Position]) == '\\' {
			self.Position++
		}
		buffer = append(buffer, rune(self.Source[self.Position]))
		self.Position++
	}
	if self.isEof() {
		return EOF, ""
	}
	self.Position++
	return STRING, string(buffer)
}

func (self *Tokenizer) isEof() bool {
	return self.Position >= len(self.Source)
}

func (self *Tokenizer) isAlmostEof() bool {
	return self.Position == len(self.Source)-1
}

func (self *Tokenizer) readNextToken() (token int, lit string) {
	if self.isEof() {
		return EOF, ""
	}
	for unicode.IsSpace(rune(self.Source[self.Position])) {
		self.Position++
		if self.isEof() {
			return EOF, ""
		}
	}
	currentChar := rune(self.Source[self.Position])
	var nextChar rune
	if !self.isAlmostEof() {
		nextChar = rune(self.Source[self.Position+1])
	}
	if unicode.IsLetter(currentChar) || currentChar == '_' {
		return self.readSymbol()
	} else if unicode.IsNumber(currentChar) {
		return self.readNumber()
	} else if currentChar == '-' && unicode.IsNumber(nextChar) {
		return self.readNumber()
	} else if currentChar == '"' {
		return self.readString()
	} else if currentChar == '\'' {
		self.Position++
		return QUOTE, "'"
	} else if currentChar == '`' {
		self.Position++
		return BACKQUOTE, "`"
	} else if currentChar == ',' && nextChar == '@' {
		self.Position += 2
		return COMMAAT, ",@"
	} else if currentChar == ',' {
		self.Position++
		return COMMA, ","
	} else if currentChar == '(' {
		self.Position++
		return LPAREN, "("
	} else if currentChar == ')' {
		self.Position++
		return RPAREN, ")"
	} else if currentChar == '[' {
		self.Position++
		return LBRACKET, "["
	} else if currentChar == ']' {
		self.Position++
		return RBRACKET, "]"
	} else if currentChar == '.' {
		self.Position++
		return PERIOD, "."
	} else if currentChar == '-' && nextChar == '>' {
		self.Position += 2
		return SYMBOL, "->"
	} else if currentChar == '=' && nextChar == '>' {
		self.Position += 2
		return SYMBOL, "=>"
	} else if currentChar == '+' {
		self.Position++
		return SYMBOL, "+"
	} else if currentChar == '-' {
		self.Position++
		return SYMBOL, "-"
	} else if currentChar == '*' {
		self.Position++
		return SYMBOL, "*"
	} else if currentChar == '/' {
		self.Position++
		return SYMBOL, "/"
	} else if currentChar == '%' {
		self.Position++
		return SYMBOL, "%"
	} else if currentChar == '<' && nextChar == '=' {
		self.Position += 2
		return SYMBOL, "<="
	} else if currentChar == '<' {
		self.Position++
		return SYMBOL, "<"
	} else if currentChar == '>' && nextChar == '=' {
		self.Position += 2
		return SYMBOL, ">="
	} else if currentChar == '>' {
		self.Position++
		return SYMBOL, ">"
	} else if currentChar == '=' && nextChar == '=' {
		self.Position += 2
		return SYMBOL, "=="
	} else if currentChar == '=' {
		self.Position++
		return SYMBOL, "="
	} else if currentChar == '!' && nextChar == '=' {
		self.Position += 2
		return SYMBOL, "!="
	} else if currentChar == '!' {
		self.Position++
		return SYMBOL, "!"
	} else if currentChar == '#' {
		self.Position += 2
		if nextChar == 't' {
			return TRUE, "#t"
		} else {
			return FALSE, "#f"
		}
	} else if currentChar == ';' {
		start := self.Position
		for {
			if self.isEof() {
				return COMMENT, self.Source[start:]
			} else if self.Source[self.Position] == '\n' {
				return COMMENT, self.Source[start:self.Position]
			}
			self.Position++
		}
	} else {
		return ILLEGAL, fmt.Sprintf("%c", currentChar)
	}
}

func (self *Tokenizer) ConsumeToken() {
	self.LookaheadToken, self.LookaheadLit = self.readNextToken()
	if self.LookaheadToken == COMMENT { // skip comments
		self.ConsumeToken()
	}
}
<|endoftext|>


<commit_before>//Package anaconda provides structs and functions for accessing version 1.1
//of the Twitter API.
//
//Successful API queries return native Go structs that can be used immediately,
//with no need for type assertions.
//
//Authentication
//
//If you already have the access token (and secret) for your user (Twitter provides this for your own account on the developer portal), creating the client is simple:
//
//  anaconda.SetConsumerKey("your-consumer-key")
//  anaconda.SetConsumerSecret("your-consumer-secret")
//  api := anaconda.NewTwitterApi("your-access-token", "your-access-token-secret")
//
//
//Queries
//
//Executing queries on an authenticated TwitterApi struct is simple.
//
//  searchResult, _ := api.GetSearch("golang", nil)
//  for _ , tweet := range searchResult {
//      fmt.Print(tweet.Text)
//  }
//
//Certain endpoints allow separate optional parameter; if desired, these can be passed as the final parameter.
//
//  v := url.Values{}
//  v.Set("count", "30")
//  result, err := api.GetSearch("golang", v)
//
//
//Endpoints
//
//Anaconda implements most of the endpoints defined in the Twitter API documentation: https://dev.twitter.com/docs/api/1.1.
//For clarity, in most cases, the function name is simply the name of the HTTP method and the endpoint (e.g., the endpoint `GET /friendships/incoming` is provided by the function `GetFriendshipsIncoming`).
//
//In a few cases, a shortened form has been chosen to make life easier (for example, retweeting is simply the function `Retweet`)
//
//More detailed information about the behavior of each particular endpoint can be found at the official Twitter API documentation.
package anaconda

import (
	"encoding/json"
	"fmt"
	"github.com/garyburd/go-oauth/oauth"
	"io/ioutil"
	"net/http"
	"net/url"
)

var oauthClient = oauth.Client{
	TemporaryCredentialRequestURI: "http://api.twitter.com/oauth/request_token",
	ResourceOwnerAuthorizationURI: "http://api.twitter.com/oauth/authenticate",
	TokenRequestURI:               "http://api.twitter.com/oauth/access_token",
}

type TwitterApi struct {
	Credentials *oauth.Credentials
}

//NewTwitterApi takes an user-specific access token and secret and returns a TwitterApi struct for that user.
//The TwitterApi struct can be used for accessing any of the endpoints available.
func NewTwitterApi(access_token string, access_token_secret string) TwitterApi {
	return TwitterApi{&oauth.Credentials{Token: access_token, Secret: access_token_secret}}
}

//SetConsumerKey will set the application-specific consumer_key used in the initial OAuth process
//This key is listed on https://dev.twitter.com/apps/YOUR_APP_ID/show
func SetConsumerKey(consumer_key string) {
	oauthClient.Credentials.Token = consumer_key
}

//SetConsumerSecret will set the application-specific secret used in the initial OAuth process
//This secret is listed on https://dev.twitter.com/apps/YOUR_APP_ID/show
func SetConsumerSecret(consumer_secret string) {
	oauthClient.Credentials.Secret = consumer_secret
}

//AuthorizationURL generates the authorization URL for the first part of the OAuth handshake.
//Redirect the user to this URL.
//This assumes that the consumer key has already been set (using SetConsumerKey).
func AuthorizationURL(callback string) (string, error) {
	tempCred, err := oauthClient.RequestTemporaryCredentials(http.DefaultClient, callback, nil)
	if err != nil {
		return "", err
	}
	return oauthClient.AuthorizationURL(tempCred, nil), nil
}

func cleanValues(v url.Values) url.Values {
	if v == nil {
		return url.Values{}
	}
	return v
}

// apiGet issues a GET request to the Twitter API and decodes the response JSON to data.
func (c TwitterApi) apiGet(urlStr string, form url.Values, data interface{}) error {
	resp, err := oauthClient.Get(http.DefaultClient, c.Credentials, urlStr, form)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	return decodeResponse(resp, data)
}

// apiPost issues a POST request to the Twitter API and decodes the response JSON to data.
func (c TwitterApi) apiPost(urlStr string, form url.Values, data interface{}) error {
	resp, err := oauthClient.Post(http.DefaultClient, c.Credentials, urlStr, form)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	return decodeResponse(resp, data)
}

// decodeResponse decodes the JSON response from the Twitter API.
func decodeResponse(resp *http.Response, data interface{}) error {
	if resp.StatusCode != 200 {
		p, _ := ioutil.ReadAll(resp.Body)

		return fmt.Errorf("Get %s returned status %d, %s", resp.Request.URL, resp.StatusCode, p)
	}
	return json.NewDecoder(resp.Body).Decode(data)
}
<commit_msg>Add 2nd step of auth handshake. Return temporary credentials from first step for storage and verification in 2nd step.<commit_after>//Package anaconda provides structs and functions for accessing version 1.1
//of the Twitter API.
//
//Successful API queries return native Go structs that can be used immediately,
//with no need for type assertions.
//
//Authentication
//
//If you already have the access token (and secret) for your user (Twitter provides this for your own account on the developer portal), creating the client is simple:
//
//  anaconda.SetConsumerKey("your-consumer-key")
//  anaconda.SetConsumerSecret("your-consumer-secret")
//  api := anaconda.NewTwitterApi("your-access-token", "your-access-token-secret")
//
//
//Queries
//
//Executing queries on an authenticated TwitterApi struct is simple.
//
//  searchResult, _ := api.GetSearch("golang", nil)
//  for _ , tweet := range searchResult {
//      fmt.Print(tweet.Text)
//  }
//
//Certain endpoints allow separate optional parameter; if desired, these can be passed as the final parameter.
//
//  v := url.Values{}
//  v.Set("count", "30")
//  result, err := api.GetSearch("golang", v)
//
//
//Endpoints
//
//Anaconda implements most of the endpoints defined in the Twitter API documentation: https://dev.twitter.com/docs/api/1.1.
//For clarity, in most cases, the function name is simply the name of the HTTP method and the endpoint (e.g., the endpoint `GET /friendships/incoming` is provided by the function `GetFriendshipsIncoming`).
//
//In a few cases, a shortened form has been chosen to make life easier (for example, retweeting is simply the function `Retweet`)
//
//More detailed information about the behavior of each particular endpoint can be found at the official Twitter API documentation.
package anaconda

import (
	"encoding/json"
	"fmt"
	"github.com/garyburd/go-oauth/oauth"
	"io/ioutil"
	"net/http"
	"net/url"
)

var oauthClient = oauth.Client{
	TemporaryCredentialRequestURI: "http://api.twitter.com/oauth/request_token",
	ResourceOwnerAuthorizationURI: "http://api.twitter.com/oauth/authenticate",
	TokenRequestURI:               "http://api.twitter.com/oauth/access_token",
}

type TwitterApi struct {
	Credentials *oauth.Credentials
}

//NewTwitterApi takes an user-specific access token and secret and returns a TwitterApi struct for that user.
//The TwitterApi struct can be used for accessing any of the endpoints available.
func NewTwitterApi(access_token string, access_token_secret string) TwitterApi {
	return TwitterApi{&oauth.Credentials{Token: access_token, Secret: access_token_secret}}
}

//SetConsumerKey will set the application-specific consumer_key used in the initial OAuth process
//This key is listed on https://dev.twitter.com/apps/YOUR_APP_ID/show
func SetConsumerKey(consumer_key string) {
	oauthClient.Credentials.Token = consumer_key
}

//SetConsumerSecret will set the application-specific secret used in the initial OAuth process
//This secret is listed on https://dev.twitter.com/apps/YOUR_APP_ID/show
func SetConsumerSecret(consumer_secret string) {
	oauthClient.Credentials.Secret = consumer_secret
}

//AuthorizationURL generates the authorization URL for the first part of the OAuth handshake.
//Redirect the user to this URL.
//This assumes that the consumer key has already been set (using SetConsumerKey).
func AuthorizationURL(callback string) (string, *oauth.Credentials, error) {
	tempCred, err := oauthClient.RequestTemporaryCredentials(http.DefaultClient, callback, nil)
	if err != nil {
		return "", nil, err
	}
	return oauthClient.AuthorizationURL(tempCred, nil), tempCred, nil
}

func GetCredentials(tempCred *oauth.Credentials, verifier string) (*oauth.Credentials, url.Values, error) {
  return oauthClient.RequestToken(http.DefaultClient, tempCred, verifier)
}

func cleanValues(v url.Values) url.Values {
	if v == nil {
		return url.Values{}
	}
	return v
}

// apiGet issues a GET request to the Twitter API and decodes the response JSON to data.
func (c TwitterApi) apiGet(urlStr string, form url.Values, data interface{}) error {
	resp, err := oauthClient.Get(http.DefaultClient, c.Credentials, urlStr, form)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	return decodeResponse(resp, data)
}

// apiPost issues a POST request to the Twitter API and decodes the response JSON to data.
func (c TwitterApi) apiPost(urlStr string, form url.Values, data interface{}) error {
	resp, err := oauthClient.Post(http.DefaultClient, c.Credentials, urlStr, form)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	return decodeResponse(resp, data)
}

// decodeResponse decodes the JSON response from the Twitter API.
func decodeResponse(resp *http.Response, data interface{}) error {
	if resp.StatusCode != 200 {
		p, _ := ioutil.ReadAll(resp.Body)

		return fmt.Errorf("Get %s returned status %d, %s", resp.Request.URL, resp.StatusCode, p)
	}
	return json.NewDecoder(resp.Body).Decode(data)
}
<|endoftext|>


<commit_before>package mathgl

import "math"

func Rotate2D(angle float64) Matrix {
	sin, cos := math.Sin(angle), math.Cos(angle)
	return *unsafeMatrixFromSlice(ScalarSlice([]interface{}{cos, -sin,
		sin, cos}, FLOAT64), FLOAT64, 2, 2)
}

// Rotates about X-axis
// [1 0 0]
// [0 c -s]
// [0 s c ]
func Rotate3DX(angle float64) Matrix {
	sin, cos := math.Sin(angle), math.Cos(angle)
	return *unsafeMatrixFromSlice(ScalarSlice([]interface{}{1, 0, 0,
		0, cos, -sin,
		0, sin, cos}, FLOAT64), FLOAT64, 3, 3)
}

// Rotates around Y-axis
// [c 0 s]
// [0 1 0]
// [s 0 c ]
func Rotate3DY(angle float64) Matrix {
	sin, cos := math.Sin(angle), math.Cos(angle)
	return *unsafeMatrixFromSlice(ScalarSlice([]interface{}{cos, 0, sin,
		0, 1, 0,
		-sin, 0, cos}, FLOAT64), FLOAT64, 3, 3)
}

// Rotates about Z-axis
// [c -s 0]
// [s c 0]
// [0 0 1 ]
func Rotate3DZ(angle float64) Matrix {
	sin, cos := math.Sin(angle), math.Cos(angle)
	return *unsafeMatrixFromSlice(ScalarSlice([]interface{}{cos, -sin, 0,
		-sin, cos, 0,
		0, 0, 1}, FLOAT64), FLOAT64, 3, 3)
}

// Returns a 2D homogeneous (3x3) transformation matrix
func Transform2D(Tx, Ty float64) Matrix {
	return *unsafeMatrixFromSlice(ScalarSlice([]interface{}{1, 0, Tx,
		0, 1, Ty,
		0, 0, 1}, FLOAT64), FLOAT64, 3, 3)
}

// Returns a 3D homogeneous (4x4) transformation matrix
func Transform3D(Tx, Ty, Tz float64) Matrix {
	return *unsafeMatrixFromSlice(ScalarSlice([]interface{}{1, 0, 0, Tx,
		0, 1, 0, Ty,
		0, 0, 1, Tz,
		0, 0, 0, 1}, FLOAT64), FLOAT64, 4, 4)
}

func HomogRotate2D(angle float64) Matrix {
	sin, cos := math.Sin(angle), math.Cos(angle)
	return *unsafeMatrixFromSlice(ScalarSlice([]interface{}{cos, -sin, 0,
		sin, cos, 0,
		0, 0, 1}, FLOAT64), FLOAT64, 3, 3)
}

// Rotates about X-axis
// [1 0 0]
// [0 c -s]
// [0 s c ]
func HomogRotate3DX(angle float64) Matrix {
	sin, cos := math.Sin(angle), math.Cos(angle)
	return *unsafeMatrixFromSlice(ScalarSlice([]interface{}{1, 0, 0, 0,
		0, cos, -sin, 0,
		0, sin, cos, 0,
		0, 0, 0, 1}, FLOAT64), FLOAT64, 4, 4)
}

// Rotates around Y-axis
// [c 0 s]
// [0 1 0]
// [s 0 c ]
func HomogRotate3DY(angle float64) Matrix {
	sin, cos := math.Sin(angle), math.Cos(angle)
	return *unsafeMatrixFromSlice(ScalarSlice([]interface{}{cos, 0, sin, 0,
		0, 1, 0, 0,
		-sin, 0, cos, 0,
		0, 0, 0, 1}, FLOAT64), FLOAT64, 4, 4)
}

// Rotates about Z-axis
// [c -s 0]
// [s c 0]
// [0 0 1 ]
func HomogRotate3DZ(angle float64) Matrix {
	sin, cos := math.Sin(angle), math.Cos(angle)
	return *unsafeMatrixFromSlice(ScalarSlice([]interface{}{cos, -sin, 0, 0, -sin, cos, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1}, FLOAT64), FLOAT64, 4, 4)
}

// Only accepts a homogeneous 3D vector, if it's not returns the zero-type for Matrix. Vector must also be of type Float64
// It will always return a Matrix of type Float64
func HomogRotate3D(angle float64, axis Vector) Matrix {
	if axis.Size() != 4 || axis.typ != FLOAT64 {
		return Matrix{}
	}
	x, y, z := axis.dat[0].Fl64(), axis.dat[1].Fl64(), axis.dat[2].Fl64()
	s, c := math.Sin(angle), math.Cos(angle)
	k := 1 - c
	return *unsafeMatrixFromSlice(ScalarSlice([]interface{}{x*x*k + c, x*y*k - z*s, x*z*k + y*s, 0,
		x*y*k + z*s, y*y*k + c, y*z*k - x*s, 0,
		x*z*k - y*s, y*z*k + x*s, z*z*k + c, 0,
		0, 0, 0, 1}, FLOAT64), FLOAT64, 4, 4)
}

// Note: Vector type must be FLOAT64
func QuaternionRotation(angle float64, axis Vector) Quaternion {
	if axis.typ != FLOAT64 {
		return Quaternion{}
	}

	sin, cos := math.Sin(angle), math.Cos(angle)
	return Quaternion{MakeScalar(cos/float64(2), FLOAT64), axis.ScalarMul(MakeScalar(sin/float64(2), FLOAT64)), FLOAT64}
}
<commit_msg>Formatting<commit_after>package mathgl

import "math"

func Rotate2D(angle float64) Matrix {
	sin, cos := math.Sin(angle), math.Cos(angle)
	return *unsafeMatrixFromSlice(ScalarSlice([]interface{}{cos, -sin,
		sin, cos}, FLOAT64), FLOAT64, 2, 2)
}

// Rotates about X-axis
// [1 0 0]
// [0 c -s]
// [0 s c ]
func Rotate3DX(angle float64) Matrix {
	sin, cos := math.Sin(angle), math.Cos(angle)
	return *unsafeMatrixFromSlice(ScalarSlice([]interface{}{
		1, 0, 0,
		0, cos, -sin,
		0, sin, cos}, FLOAT64), FLOAT64, 3, 3)
}

// Rotates around Y-axis
// [c 0 s]
// [0 1 0]
// [s 0 c ]
func Rotate3DY(angle float64) Matrix {
	sin, cos := math.Sin(angle), math.Cos(angle)
	return *unsafeMatrixFromSlice(ScalarSlice([]interface{}{cos, 0, sin,
		0, 1, 0,
		-sin, 0, cos}, FLOAT64), FLOAT64, 3, 3)
}

// Rotates about Z-axis
// [c -s 0]
// [s c 0]
// [0 0 1 ]
func Rotate3DZ(angle float64) Matrix {
	sin, cos := math.Sin(angle), math.Cos(angle)
	return *unsafeMatrixFromSlice(ScalarSlice([]interface{}{cos, -sin, 0,
		-sin, cos, 0,
		0, 0, 1}, FLOAT64), FLOAT64, 3, 3)
}

// Returns a 2D homogeneous (3x3) transformation matrix
func Transform2D(Tx, Ty float64) Matrix {
	return *unsafeMatrixFromSlice(ScalarSlice([]interface{}{1, 0, Tx,
		0, 1, Ty,
		0, 0, 1}, FLOAT64), FLOAT64, 3, 3)
}

// Returns a 3D homogeneous (4x4) transformation matrix
func Transform3D(Tx, Ty, Tz float64) Matrix {
	return *unsafeMatrixFromSlice(ScalarSlice([]interface{}{1, 0, 0, Tx,
		0, 1, 0, Ty,
		0, 0, 1, Tz,
		0, 0, 0, 1}, FLOAT64), FLOAT64, 4, 4)
}

func HomogRotate2D(angle float64) Matrix {
	sin, cos := math.Sin(angle), math.Cos(angle)
	return *unsafeMatrixFromSlice(ScalarSlice([]interface{}{cos, -sin, 0,
		sin, cos, 0,
		0, 0, 1}, FLOAT64), FLOAT64, 3, 3)
}

// Rotates about X-axis
// [1 0 0]
// [0 c -s]
// [0 s c ]
func HomogRotate3DX(angle float64) Matrix {
	sin, cos := math.Sin(angle), math.Cos(angle)
	return *unsafeMatrixFromSlice(ScalarSlice([]interface{}{1, 0, 0, 0,
		0, cos, -sin, 0,
		0, sin, cos, 0,
		0, 0, 0, 1}, FLOAT64), FLOAT64, 4, 4)
}

// Rotates around Y-axis
// [c 0 s]
// [0 1 0]
// [s 0 c ]
func HomogRotate3DY(angle float64) Matrix {
	sin, cos := math.Sin(angle), math.Cos(angle)
	return *unsafeMatrixFromSlice(ScalarSlice([]interface{}{cos, 0, sin, 0,
		0, 1, 0, 0,
		-sin, 0, cos, 0,
		0, 0, 0, 1}, FLOAT64), FLOAT64, 4, 4)
}

// Rotates about Z-axis
// [c -s 0]
// [s c 0]
// [0 0 1 ]
func HomogRotate3DZ(angle float64) Matrix {
	sin, cos := math.Sin(angle), math.Cos(angle)
	return *unsafeMatrixFromSlice(ScalarSlice([]interface{}{cos, -sin, 0, 0, -sin, cos, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1}, FLOAT64), FLOAT64, 4, 4)
}

// Only accepts a homogeneous 3D vector, if it's not returns the zero-type for Matrix. Vector must also be of type Float64
// It will always return a Matrix of type Float64
func HomogRotate3D(angle float64, axis Vector) Matrix {
	if axis.Size() != 4 || axis.typ != FLOAT64 {
		return Matrix{}
	}
	x, y, z := axis.dat[0].Fl64(), axis.dat[1].Fl64(), axis.dat[2].Fl64()
	s, c := math.Sin(angle), math.Cos(angle)
	k := 1 - c
	return *unsafeMatrixFromSlice(ScalarSlice([]interface{}{x*x*k + c, x*y*k - z*s, x*z*k + y*s, 0,
		x*y*k + z*s, y*y*k + c, y*z*k - x*s, 0,
		x*z*k - y*s, y*z*k + x*s, z*z*k + c, 0,
		0, 0, 0, 1}, FLOAT64), FLOAT64, 4, 4)
}

// Note: Vector type must be FLOAT64
func QuaternionRotation(angle float64, axis Vector) Quaternion {
	if axis.typ != FLOAT64 {
		return Quaternion{}
	}

	sin, cos := math.Sin(angle), math.Cos(angle)
	return Quaternion{MakeScalar(cos/float64(2), FLOAT64), axis.ScalarMul(MakeScalar(sin/float64(2), FLOAT64)), FLOAT64}
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tree

import (
	"fmt"
	"rand"
	"time"
)

func init() {
	rand.Seed(time.Nanoseconds())
}

// A Tree is a binary tree with integer values.
type Tree struct {
	Left  *Tree
	Value int
	Right *Tree
}

// New returns a new, random binary tree holding the values k, 2k, ..., 10k.
func New(k int) *Tree {
	var t *Tree
	for _, v := range rand.Perm(10) {
		t = insert(t, (1+v)*k)
	}
	return t
}

func insert(t *Tree, v int) *Tree {
	if t == nil {
		return &Tree{nil, v, nil}
	}
	if v < t.Value {
		t.Left = insert(t.Left, v)
		return t
	}
	t.Right = insert(t.Right, v)
	return t
}

func (t *Tree) String() string {
	if t == nil {
		return "()"
	}
	s := ""
	if t.Left != nil {
		s += t.Left.String() + " "
	}
	s += fmt.Sprint(t.Value)
	if t.Right != nil {
		s += " " + t.Right.String()
	}
	return "(" + s + ")"
}
<commit_msg>sync tree<commit_after>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tree

import (
	"fmt"
	"rand"
)

// A Tree is a binary tree with integer values.
type Tree struct {
	Left  *Tree
	Value int
	Right *Tree
}

// New returns a new, random binary tree holding the values k, 2k, ..., 10k.
func New(k int) *Tree {
	var t *Tree
	for _, v := range rand.Perm(10) {
		t = insert(t, (1+v)*k)
	}
	return t
}

func insert(t *Tree, v int) *Tree {
	if t == nil {
		return &Tree{nil, v, nil}
	}
	if v < t.Value {
		t.Left = insert(t.Left, v)
	} else {
		t.Right = insert(t.Right, v)
	}
	return t
}

func (t *Tree) String() string {
	if t == nil {
		return "()"
	}
	s := ""
	if t.Left != nil {
		s += t.Left.String() + " "
	}
	s += fmt.Sprint(t.Value)
	if t.Right != nil {
		s += " " + t.Right.String()
	}
	return "(" + s + ")"
}
<|endoftext|>


<commit_before>// SPDX-License-Identifier: Apache-2.0
// Copyright Authors of Cilium

package cmd

import (
	"context"
	"time"

	"k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/tools/cache"

	"github.com/cilium/cilium/pkg/controller"
	"github.com/cilium/cilium/pkg/k8s"
	cilium_v2 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2"
	k8sClient "github.com/cilium/cilium/pkg/k8s/client"
	"github.com/cilium/cilium/pkg/k8s/informer"
	"github.com/cilium/cilium/pkg/k8s/utils"
	"github.com/cilium/cilium/pkg/k8s/watchers/resources"
	"github.com/cilium/cilium/pkg/kvstore/store"
	"github.com/cilium/cilium/pkg/option"
	"github.com/cilium/cilium/pkg/policy/groups"
)

var (
	// cnpStatusUpdateInterval is the amount of time between status updates
	// being sent to the K8s apiserver for a given CNP.
	cnpStatusUpdateInterval time.Duration
)

func init() {
	runtime.ErrorHandlers = []func(error){
		k8s.K8sErrorHandler,
	}
}

// enableCNPWatcher waits for the CiliumNetowrkPolicy CRD availability and then
// garbage collects stale CiliumNetowrkPolicy status field entries.
func enableCNPWatcher(clientset k8sClient.Clientset) error {
	enableCNPStatusUpdates := kvstoreEnabled() && option.Config.K8sEventHandover && !option.Config.DisableCNPStatusUpdates
	if enableCNPStatusUpdates {
		log.Info("Starting CNP Status handover from kvstore to k8s")
	}
	log.Info("Starting CNP derivative handler")

	var (
		cnpStatusMgr *k8s.CNPStatusEventHandler
	)
	cnpStore := cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc)

	if enableCNPStatusUpdates {
		cnpStatusMgr = k8s.NewCNPStatusEventHandler(clientset, cnpStore, cnpStatusUpdateInterval)
		cnpSharedStore, err := store.JoinSharedStore(store.Configuration{
			Prefix: k8s.CNPStatusesPath,
			KeyCreator: func() store.Key {
				return &k8s.CNPNSWithMeta{}
			},
			Observer: cnpStatusMgr,
		})
		if err != nil {
			return err
		}

		// It is safe to update the CNP store here given the CNP Store
		// will only be used by StartStatusHandler method which is used in the
		// cilium v2 controller below.
		cnpStatusMgr.UpdateCNPStore(cnpSharedStore)
	}

	ciliumV2Controller := informer.NewInformerWithStore(
		utils.ListerWatcherFromTyped[*cilium_v2.CiliumNetworkPolicyList](clientset.CiliumV2().CiliumNetworkPolicies("")),
		&cilium_v2.CiliumNetworkPolicy{},
		0,
		cache.ResourceEventHandlerFuncs{
			AddFunc: func(obj interface{}) {
				k8sEventMetric(resources.MetricCNP, resources.MetricCreate)
				if cnp := k8s.ObjToSlimCNP(obj); cnp != nil {
					// We need to deepcopy this structure because we are writing
					// fields.
					// See https://github.com/cilium/cilium/blob/27fee207f5422c95479422162e9ea0d2f2b6c770/pkg/policy/api/ingress.go#L112-L134
					cnpCpy := cnp.DeepCopy()

					groups.AddDerivativeCNPIfNeeded(clientset, cnpCpy.CiliumNetworkPolicy)
					if enableCNPStatusUpdates {
						cnpStatusMgr.StartStatusHandler(cnpCpy)
					}
				}
			},
			UpdateFunc: func(oldObj, newObj interface{}) {
				k8sEventMetric(resources.MetricCNP, resources.MetricUpdate)
				if oldCNP := k8s.ObjToSlimCNP(oldObj); oldCNP != nil {
					if newCNP := k8s.ObjToSlimCNP(newObj); newCNP != nil {
						if oldCNP.DeepEqual(newCNP) {
							return
						}

						// We need to deepcopy this structure because we are writing
						// fields.
						// See https://github.com/cilium/cilium/blob/27fee207f5422c95479422162e9ea0d2f2b6c770/pkg/policy/api/ingress.go#L112-L134
						newCNPCpy := newCNP.DeepCopy()
						oldCNPCpy := oldCNP.DeepCopy()

						groups.UpdateDerivativeCNPIfNeeded(clientset, newCNPCpy.CiliumNetworkPolicy, oldCNPCpy.CiliumNetworkPolicy)
					}
				}
			},
			DeleteFunc: func(obj interface{}) {
				k8sEventMetric(resources.MetricCNP, resources.MetricDelete)
				cnp := k8s.ObjToSlimCNP(obj)
				if cnp == nil {
					return
				}
				// The derivative policy will be deleted by the parent but need
				// to delete the cnp from the pooling.
				groups.DeleteDerivativeFromCache(cnp.CiliumNetworkPolicy)
				if enableCNPStatusUpdates {
					cnpStatusMgr.StopStatusHandler(cnp)
				}
			},
		},
		k8s.ConvertToCNP,
		cnpStore,
	)
	go ciliumV2Controller.Run(wait.NeverStop)

	controller.NewManager().UpdateController("cnp-to-groups",
		controller.ControllerParams{
			DoFunc: func(ctx context.Context) error {
				groups.UpdateCNPInformation(clientset)
				return nil
			},
			RunInterval: 5 * time.Minute,
		})

	return nil
}
<commit_msg>chore: fix typo in enableCNPWatcher comment<commit_after>// SPDX-License-Identifier: Apache-2.0
// Copyright Authors of Cilium

package cmd

import (
	"context"
	"time"

	"k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/tools/cache"

	"github.com/cilium/cilium/pkg/controller"
	"github.com/cilium/cilium/pkg/k8s"
	cilium_v2 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2"
	k8sClient "github.com/cilium/cilium/pkg/k8s/client"
	"github.com/cilium/cilium/pkg/k8s/informer"
	"github.com/cilium/cilium/pkg/k8s/utils"
	"github.com/cilium/cilium/pkg/k8s/watchers/resources"
	"github.com/cilium/cilium/pkg/kvstore/store"
	"github.com/cilium/cilium/pkg/option"
	"github.com/cilium/cilium/pkg/policy/groups"
)

var (
	// cnpStatusUpdateInterval is the amount of time between status updates
	// being sent to the K8s apiserver for a given CNP.
	cnpStatusUpdateInterval time.Duration
)

func init() {
	runtime.ErrorHandlers = []func(error){
		k8s.K8sErrorHandler,
	}
}

// enableCNPWatcher waits for the CiliumNetworkPolicy CRD availability and then
// garbage collects stale CiliumNetworkPolicy status field entries.
func enableCNPWatcher(clientset k8sClient.Clientset) error {
	enableCNPStatusUpdates := kvstoreEnabled() && option.Config.K8sEventHandover && !option.Config.DisableCNPStatusUpdates
	if enableCNPStatusUpdates {
		log.Info("Starting CNP Status handover from kvstore to k8s")
	}
	log.Info("Starting CNP derivative handler")

	var (
		cnpStatusMgr *k8s.CNPStatusEventHandler
	)
	cnpStore := cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc)

	if enableCNPStatusUpdates {
		cnpStatusMgr = k8s.NewCNPStatusEventHandler(clientset, cnpStore, cnpStatusUpdateInterval)
		cnpSharedStore, err := store.JoinSharedStore(store.Configuration{
			Prefix: k8s.CNPStatusesPath,
			KeyCreator: func() store.Key {
				return &k8s.CNPNSWithMeta{}
			},
			Observer: cnpStatusMgr,
		})
		if err != nil {
			return err
		}

		// It is safe to update the CNP store here given the CNP Store
		// will only be used by StartStatusHandler method which is used in the
		// cilium v2 controller below.
		cnpStatusMgr.UpdateCNPStore(cnpSharedStore)
	}

	ciliumV2Controller := informer.NewInformerWithStore(
		utils.ListerWatcherFromTyped[*cilium_v2.CiliumNetworkPolicyList](clientset.CiliumV2().CiliumNetworkPolicies("")),
		&cilium_v2.CiliumNetworkPolicy{},
		0,
		cache.ResourceEventHandlerFuncs{
			AddFunc: func(obj interface{}) {
				k8sEventMetric(resources.MetricCNP, resources.MetricCreate)
				if cnp := k8s.ObjToSlimCNP(obj); cnp != nil {
					// We need to deepcopy this structure because we are writing
					// fields.
					// See https://github.com/cilium/cilium/blob/27fee207f5422c95479422162e9ea0d2f2b6c770/pkg/policy/api/ingress.go#L112-L134
					cnpCpy := cnp.DeepCopy()

					groups.AddDerivativeCNPIfNeeded(clientset, cnpCpy.CiliumNetworkPolicy)
					if enableCNPStatusUpdates {
						cnpStatusMgr.StartStatusHandler(cnpCpy)
					}
				}
			},
			UpdateFunc: func(oldObj, newObj interface{}) {
				k8sEventMetric(resources.MetricCNP, resources.MetricUpdate)
				if oldCNP := k8s.ObjToSlimCNP(oldObj); oldCNP != nil {
					if newCNP := k8s.ObjToSlimCNP(newObj); newCNP != nil {
						if oldCNP.DeepEqual(newCNP) {
							return
						}

						// We need to deepcopy this structure because we are writing
						// fields.
						// See https://github.com/cilium/cilium/blob/27fee207f5422c95479422162e9ea0d2f2b6c770/pkg/policy/api/ingress.go#L112-L134
						newCNPCpy := newCNP.DeepCopy()
						oldCNPCpy := oldCNP.DeepCopy()

						groups.UpdateDerivativeCNPIfNeeded(clientset, newCNPCpy.CiliumNetworkPolicy, oldCNPCpy.CiliumNetworkPolicy)
					}
				}
			},
			DeleteFunc: func(obj interface{}) {
				k8sEventMetric(resources.MetricCNP, resources.MetricDelete)
				cnp := k8s.ObjToSlimCNP(obj)
				if cnp == nil {
					return
				}
				// The derivative policy will be deleted by the parent but need
				// to delete the cnp from the pooling.
				groups.DeleteDerivativeFromCache(cnp.CiliumNetworkPolicy)
				if enableCNPStatusUpdates {
					cnpStatusMgr.StopStatusHandler(cnp)
				}
			},
		},
		k8s.ConvertToCNP,
		cnpStore,
	)
	go ciliumV2Controller.Run(wait.NeverStop)

	controller.NewManager().UpdateController("cnp-to-groups",
		controller.ControllerParams{
			DoFunc: func(ctx context.Context) error {
				groups.UpdateCNPInformation(clientset)
				return nil
			},
			RunInterval: 5 * time.Minute,
		})

	return nil
}
<|endoftext|>


<commit_before>package outgoing_test

import (
	"os"
	"strconv"
	"testing"
	"time"

	"github.com/lestrrat/roccaforte/client"
	"github.com/lestrrat/roccaforte/event"
	"github.com/lestrrat/roccaforte/incoming"
	"github.com/lestrrat/roccaforte/outgoing"
	"github.com/stretchr/testify/assert"
	"golang.org/x/net/context"
)

var projectID string

func init() {
	projectID = os.Getenv("DATASTORE_PROJECT_ID")
}

func TestOutgoing(t *testing.T) {
	if projectID == "" {
		t.Skip("missing project ID. please set DATASTORE_PROJECT_ID")
		return
	}

	// Enqueue
	ctx, cancel := context.WithCancel(context.Background())

	h := incoming.NewHTTPSource()
	s := incoming.NewGDatastoreStorage(projectID)
	e := incoming.New()
	e.Storage = s
	e.AddSource(h)

	go e.Run(ctx)
	go h.Loop(ctx)
	for i := 0; i < 10; i++ {
		e.SetRule("test.notify" + strconv.Itoa(i), &incoming.Rule{})
	}

	for i := 0; i < 10; i++ {
		go func() {
			cl := client.NewHTTP("http://localhost:8080/enqueue")
			for j := 0; j < 100; j++ {
				e := event.NewCoreAttrs("test.notify" + strconv.Itoa(i))
				if !assert.NoError(t, cl.Enqueue(e), "enqueue should succeed") {
					return
				}
			}
		}()
	}

	o := outgoing.New(projectID)
	go o.Run(ctx)

	time.AfterFunc(10*time.Second, cancel)
	<-ctx.Done()
}
<commit_msg>check for errors<commit_after>package outgoing_test

import (
	"os"
	"strconv"
	"testing"
	"time"

	"github.com/lestrrat/roccaforte/client"
	"github.com/lestrrat/roccaforte/event"
	"github.com/lestrrat/roccaforte/incoming"
	"github.com/lestrrat/roccaforte/outgoing"
	"github.com/stretchr/testify/assert"
	"golang.org/x/net/context"
)

var projectID string

func init() {
	projectID = os.Getenv("DATASTORE_PROJECT_ID")
}

func TestOutgoing(t *testing.T) {
	if projectID == "" {
		t.Skip("missing project ID. please set DATASTORE_PROJECT_ID")
		return
	}

	// Enqueue
	ctx, cancel := context.WithCancel(context.Background())

	h := incoming.NewHTTPSource()
	s := incoming.NewGDatastoreStorage(projectID)
	e := incoming.New()
	e.Storage = s
	e.AddSource(h)

	go func() {
		assert.NoError(t, e.Run(ctx), "incoming engine should exit w/o errors")
	}()

	go h.Loop(ctx)
	for i := 0; i < 10; i++ {
		e.SetRule("test.notify" + strconv.Itoa(i), &incoming.Rule{})
	}

	for i := 0; i < 10; i++ {
		go func() {
			cl := client.NewHTTP("http://localhost:8080/enqueue")
			for j := 0; j < 100; j++ {
				e := event.NewCoreAttrs("test.notify" + strconv.Itoa(i))
				if !assert.NoError(t, cl.Enqueue(e), "enqueue should succeed") {
					return
				}
			}
		}()
	}

	o := outgoing.New(projectID)
	go o.Run(ctx)

	time.AfterFunc(10*time.Second, cancel)
	<-ctx.Done()
}
<|endoftext|>