<commit_before>// Copyright 2017 OpenConfigd Project.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/coreos/etcd/clientv3"
	"github.com/mitchellh/mapstructure"
	"golang.org/x/net/context"
)

type OspfNeighbor struct {
	Neighbor  string `json:"neighbor"`
	Priority  int    `json:"priority"`
	State     string `json:"state"`
	Address   string `json:"address"`
	Interface string `json:"interface"`
}

type OspfNexthop struct {
	Address   string `mapstructure:"address" json:"address"`
	Interface string `mapstructure:"interface" json:"interface"`
}

type OspfRoute struct {
	Network  string        `json:"network"`
	Nexthops []OspfNexthop `json:"next-hop"`
}

type OspfVrfStat struct {
	Vrf       string         `json:"vrf"`
	Neighbors []OspfNeighbor `json:"neighbors"`
	Routes    []OspfRoute    `json:"routes"`
}

type OspfStat struct {
	Ospf []OspfVrfStat `json:"ospf"`
}

var (
	OspfEtcdEndpoints  = []string{"http://127.0.0.1:2379"}
	OspfEtcdStatusPath = "/state/services/ospf"
)

func VrfName(vrfId int) string {
	return fmt.Sprintf("vrf%d", vrfId)
}

var ospfVrfRouteCmd = `#! /bin/bash
source /etc/bash_completion.d/cli
SHOW_MODE=json
show ip route vrf %s ospf
`

// Only used for JSON parse.
type OspfJSONRoutes struct {
	Prefix   string        `mapstructure:"prefix"`
	Nexthops []OspfNexthop `mapstructure:"nexthops"`
	Metric   int           `mapstructure:"metric"`
	Distance int           `mapstructure:"distance"`
	Type     string        `mapstructure:"type"`
}

func OspfStatusRoute(vrfName string, stat *OspfVrfStat) error {
	cmdName := "/tmp/.ospf_vrf_route_show.sh"
	os.Remove(cmdName)

	// OSPF route via ribd JSON.
	cmdStr := fmt.Sprintf(ospfVrfRouteCmd, vrfName)

	err := ioutil.WriteFile(cmdName, []byte(cmdStr), 0755)
	if err != nil {
		fmt.Println("[ospf]WriteFile err:", err)
		return err
	}

	// Execute command.
	cmd := exec.Command(cmdName)
	out, err := cmd.Output()
	os.Remove(cmdName)
	if err != nil {
		fmt.Println("OspfStatusRoute err:", err)
		return err
	}

	// Parse JSON
	var jsonIntf interface{}
	err = json.Unmarshal(out, &jsonIntf)
	if err != nil {
		fmt.Println("Unmarshal err", err)
		return err
	}

	// Convert to OspfVrfStat
	for _, ent := range jsonIntf.([]interface{}) {
		var val OspfJSONRoutes
		err = mapstructure.Decode(ent, &val)
		if err != nil {
			fmt.Println("Decode err:", err)
			return err
		}
		r := OspfRoute{
			Network:  val.Prefix,
			Nexthops: val.Nexthops,
		}
		stat.Routes = append(stat.Routes, r)
	}
	return nil
}

func OspfStatusNeighbor(vrfName string, stat *OspfVrfStat) error {
	cmd := exec.Command("vtysh", "-c", "show ip ospf neighbor")
	env := os.Environ()
	env = append(env, fmt.Sprintf("VRF=%s", vrfName))
	env = append(env, "LD_PRELOAD=/usr/bin/vrf_socket.so")
	cmd.Env = env

	out, err := cmd.Output()
	if err != nil {
		fmt.Println("OspfStatusNeighbor err:", err)
		return err
	}

	strs := strings.Split(string(out), "\n")
	for _, str := range strs {
		r := regexp.MustCompile(`^[0-9]`)
		if r.MatchString(str) {
			fields := strings.Fields(str)
			if len(fields) >= 9 {
				neigh := OspfNeighbor{}
				neigh.Neighbor = fields[0]
				neigh.Priority, _ = strconv.Atoi(fields[1])
				neigh.State = fields[2]
				neigh.Address = fields[4]
				neigh.Interface = fields[5]
				stat.Neighbors = append(stat.Neighbors, neigh)
			}
		}
	}
	return nil
}

func OspfStatusVrf(vrfName string) *OspfVrfStat {
	fmt.Println("OspfStatusVrf", vrfName)
	stat := &OspfVrfStat{
		Vrf:       vrfName,
		Neighbors: []OspfNeighbor{},
		Routes:    []OspfRoute{},
	}
	OspfStatusRoute(vrfName, stat)
	OspfStatusNeighbor(vrfName, stat)

	return stat
}

func OspfStatusUpdate() {
	stats := OspfStat{}

	OspfVrfMap = map[int]OspfArray{}
	OspfVrfMap[1] = OspfArray{}

	fmt.Println("OSPF status update")

	for vrfId, _ := range OspfVrfMap {
		s := OspfStatusVrf(VrfName(vrfId))
		stats.Ospf = append(stats.Ospf, *s)
	}

	str := "{}"
	byte, err := json.Marshal(stats)
	if err == nil {
		str = string(byte)
	}

	cfg := clientv3.Config{
		Endpoints:   OspfEtcdEndpoints,
		DialTimeout: 3 * time.Second,
	}
	conn, err := clientv3.New(cfg)
	if err != nil {
		fmt.Println("OspfStatusUpdate clientv3.New:", err)
		return
	}
	defer conn.Close()

	_, err = conn.Put(context.Background(), OspfEtcdStatusPath, str)
	if err != nil {
		fmt.Println("OpsfStatusUpdate Put:", err)
		return
	}
}
<commit_msg>Do not clear OSPF vrf config.<commit_after>// Copyright 2017 OpenConfigd Project.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/coreos/etcd/clientv3"
	"github.com/mitchellh/mapstructure"
	"golang.org/x/net/context"
)

type OspfNeighbor struct {
	Neighbor  string `json:"neighbor"`
	Priority  int    `json:"priority"`
	State     string `json:"state"`
	Address   string `json:"address"`
	Interface string `json:"interface"`
}

type OspfNexthop struct {
	Address   string `mapstructure:"address" json:"address"`
	Interface string `mapstructure:"interface" json:"interface"`
}

type OspfRoute struct {
	Network  string        `json:"network"`
	Nexthops []OspfNexthop `json:"next-hop"`
}

type OspfVrfStat struct {
	Vrf       string         `json:"vrf"`
	Neighbors []OspfNeighbor `json:"neighbors"`
	Routes    []OspfRoute    `json:"routes"`
}

type OspfStat struct {
	Ospf []OspfVrfStat `json:"ospf"`
}

var (
	OspfEtcdEndpoints  = []string{"http://127.0.0.1:2379"}
	OspfEtcdStatusPath = "/state/services/ospf"
)

func VrfName(vrfId int) string {
	return fmt.Sprintf("vrf%d", vrfId)
}

var ospfVrfRouteCmd = `#! /bin/bash
source /etc/bash_completion.d/cli
SHOW_MODE=json
show ip route vrf %s ospf
`

// Only used for JSON parse.
type OspfJSONRoutes struct {
	Prefix   string        `mapstructure:"prefix"`
	Nexthops []OspfNexthop `mapstructure:"nexthops"`
	Metric   int           `mapstructure:"metric"`
	Distance int           `mapstructure:"distance"`
	Type     string        `mapstructure:"type"`
}

func OspfStatusRoute(vrfName string, stat *OspfVrfStat) error {
	cmdName := "/tmp/.ospf_vrf_route_show.sh"
	os.Remove(cmdName)

	// OSPF route via ribd JSON.
	cmdStr := fmt.Sprintf(ospfVrfRouteCmd, vrfName)

	err := ioutil.WriteFile(cmdName, []byte(cmdStr), 0755)
	if err != nil {
		fmt.Println("[ospf]WriteFile err:", err)
		return err
	}

	// Execute command.
	cmd := exec.Command(cmdName)
	out, err := cmd.Output()
	os.Remove(cmdName)
	if err != nil {
		fmt.Println("OspfStatusRoute err:", err)
		return err
	}

	// Parse JSON
	var jsonIntf interface{}
	err = json.Unmarshal(out, &jsonIntf)
	if err != nil {
		fmt.Println("Unmarshal err", err)
		return err
	}

	// Convert to OspfVrfStat
	for _, ent := range jsonIntf.([]interface{}) {
		var val OspfJSONRoutes
		err = mapstructure.Decode(ent, &val)
		if err != nil {
			fmt.Println("Decode err:", err)
			return err
		}
		r := OspfRoute{
			Network:  val.Prefix,
			Nexthops: val.Nexthops,
		}
		stat.Routes = append(stat.Routes, r)
	}
	return nil
}

func OspfStatusNeighbor(vrfName string, stat *OspfVrfStat) error {
	cmd := exec.Command("vtysh", "-c", "show ip ospf neighbor")
	env := os.Environ()
	env = append(env, fmt.Sprintf("VRF=%s", vrfName))
	env = append(env, "LD_PRELOAD=/usr/bin/vrf_socket.so")
	cmd.Env = env

	out, err := cmd.Output()
	if err != nil {
		fmt.Println("OspfStatusNeighbor err:", err)
		return err
	}

	strs := strings.Split(string(out), "\n")
	for _, str := range strs {
		r := regexp.MustCompile(`^[0-9]`)
		if r.MatchString(str) {
			fields := strings.Fields(str)
			if len(fields) >= 9 {
				neigh := OspfNeighbor{}
				neigh.Neighbor = fields[0]
				neigh.Priority, _ = strconv.Atoi(fields[1])
				neigh.State = fields[2]
				neigh.Address = fields[4]
				neigh.Interface = fields[5]
				stat.Neighbors = append(stat.Neighbors, neigh)
			}
		}
	}
	return nil
}

func OspfStatusVrf(vrfName string) *OspfVrfStat {
	fmt.Println("OspfStatusVrf", vrfName)
	stat := &OspfVrfStat{
		Vrf:       vrfName,
		Neighbors: []OspfNeighbor{},
		Routes:    []OspfRoute{},
	}
	OspfStatusRoute(vrfName, stat)
	OspfStatusNeighbor(vrfName, stat)

	return stat
}

func OspfStatusUpdate() {
	stats := OspfStat{}

	for vrfId, _ := range OspfVrfMap {
		s := OspfStatusVrf(VrfName(vrfId))
		stats.Ospf = append(stats.Ospf, *s)
	}

	str := "{}"
	byte, err := json.Marshal(stats)
	if err == nil {
		str = string(byte)
	}

	cfg := clientv3.Config{
		Endpoints:   OspfEtcdEndpoints,
		DialTimeout: 3 * time.Second,
	}
	conn, err := clientv3.New(cfg)
	if err != nil {
		fmt.Println("OspfStatusUpdate clientv3.New:", err)
		return
	}
	defer conn.Close()

	_, err = conn.Put(context.Background(), OspfEtcdStatusPath, str)
	if err != nil {
		fmt.Println("OpsfStatusUpdate Put:", err)
		return
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kubelet

import (
	"fmt"
	"net/url"
	"strings"
	"time"

	. "k8s.io/heapster/metrics/core"

	"github.com/golang/glog"
	cadvisor "github.com/google/cadvisor/info/v1"
	kube_api "k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/client/cache"
	kube_client "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/fields"
)

const (
	infraContainerName = "POD"
	// TODO: following constants are copied from k8s, change to use them directly
	kubernetesPodNameLabel      = "io.kubernetes.pod.name"
	kubernetesPodNamespaceLabel = "io.kubernetes.pod.namespace"
	kubernetesPodUID            = "io.kubernetes.pod.uid"
	kubernetesContainerLabel    = "io.kubernetes.container.name"

	CustomMetricPrefix = "CM:"
)

type cpuVal struct {
	Val       int64
	Timestamp time.Time
}

// Kubelet-provided metrics for pod and system container.
type kubeletMetricsSource struct {
	host          Host
	kubeletClient *KubeletClient
	hostname      string
	hostId        string
	cpuLastVal    map[string]cpuVal
}

func (this *kubeletMetricsSource) String() string {
	return fmt.Sprintf("kubelet:%s:%d", this.host.IP, this.host.Port)
}

func (this *kubeletMetricsSource) decodeMetrics(c *cadvisor.ContainerInfo) (string, *MetricSet) {
	var metricSetKey string
	cMetrics := &MetricSet{
		MetricValues: map[string]MetricValue{},
		Labels:       map[string]string{},
	}

	if isNode(c) {
		metricSetKey = NodeKey(this.hostname)
		cMetrics.Labels[LabelMetricSetType.Key] = MetricSetTypeNode
	} else if isSysContainer(c) {
		cName := getSysContainerName(c)
		metricSetKey = NodeContainerKey(this.hostname, cName)
		cMetrics.Labels[LabelMetricSetType.Key] = MetricSetTypeSystemContainer
		cMetrics.Labels[LabelContainerName.Key] = cName
	} else {
		cName := c.Spec.Labels[kubernetesContainerLabel]
		if cName == infraContainerName {
			return "", nil
		}
		ns := c.Spec.Labels[kubernetesPodNamespaceLabel]
		podName := c.Spec.Labels[kubernetesPodNameLabel]

		// Support for kubernetes 1.0.*
		if ns == "" && strings.Contains(podName, "/") {
			tokens := strings.SplitN(podName, "/", 2)
			if len(tokens) == 2 {
				ns = tokens[0]
				podName = tokens[1]
			}
		}
		if cName == "" {
			if !strings.HasPrefix(c.Name, "k8s_POD") {
				cName = c.Name
			}
		}

		if cName == "" || ns == "" || podName == "" {
			glog.Errorf("Missing metadata for container %v. Got: %+v", c.Name, c.Spec.Labels)
			return "", nil
		}
		metricSetKey = PodContainerKey(ns, podName, cName)
		cMetrics.Labels[LabelMetricSetType.Key] = MetricSetTypePodContainer
		cMetrics.Labels[LabelContainerName.Key] = cName
		cMetrics.Labels[LabelPodId.Key] = c.Spec.Labels[kubernetesPodUID]
		cMetrics.Labels[LabelPodName.Key] = podName
		cMetrics.Labels[LabelNamespaceName.Key] = ns
		// Needed for backward compatibility
		cMetrics.Labels[LabelPodNamespace.Key] = ns
		cMetrics.Labels[LabelContainerBaseImage.Key] = c.Spec.Image
	}

	for _, metric := range StandardMetrics {
		if metric.HasValue(&c.Spec) {
			cMetrics.MetricValues[metric.Name] = metric.GetValue(&c.Spec, c.Stats[0])
		}
	}

	if c.Spec.HasCustomMetrics {
	metricloop:
		for _, spec := range c.Spec.CustomMetrics {
			if cmValue, ok := c.Stats[0].CustomMetrics[spec.Name]; ok && cmValue != nil && len(cmValue) >= 1 {
				newest := cmValue[0]
				for _, metricVal := range cmValue {
					if newest.Timestamp.Before(metricVal.Timestamp) {
						newest = metricVal
					}
				}
				mv := MetricValue{}
				switch spec.Type {
				case cadvisor.MetricGauge:
					mv.MetricType = MetricGauge
				case cadvisor.MetricCumulative:
					mv.MetricType = MetricCumulative
				default:
					glog.V(4).Infof("Skipping %s: unknown custom metric type: %v", spec.Name, spec.Type)
					continue metricloop
				}

				switch spec.Format {
				case cadvisor.IntType:
					mv.ValueType = ValueInt64
					mv.IntValue = newest.IntValue
				case cadvisor.FloatType:
					mv.ValueType = ValueFloat
					mv.FloatValue = float32(newest.FloatValue)
				default:
					glog.V(4).Infof("Skipping %s: unknown custom metric format", spec.Name, spec.Format)
					continue metricloop
				}

				cMetrics.MetricValues[CustomMetricPrefix+spec.Name] = mv
			}
		}
	}

	// This is temporary workaround to support cpu/usege_rate metric.
	if currentVal, ok := cMetrics.MetricValues["cpu/usage"]; ok {
		if lastVal, ok := this.cpuLastVal[metricSetKey]; ok {
			// cpu/usage values are in nanoseconds; we want to have it in millicores (that's why constant 1000 is here).
			rateVal := 1000 * (currentVal.IntValue - lastVal.Val) / (c.Stats[0].Timestamp.UnixNano() - lastVal.Timestamp.UnixNano())
			cMetrics.MetricValues["cpu/usage_rate"] = MetricValue{
				ValueType:  ValueInt64,
				MetricType: MetricGauge,
				IntValue:   rateVal,
			}
		}
		this.cpuLastVal[metricSetKey] = cpuVal{
			Val:       currentVal.IntValue,
			Timestamp: c.Stats[0].Timestamp,
		}
	}

	// common labels
	cMetrics.Labels[LabelHostname.Key] = this.hostname
	cMetrics.Labels[LabelHostID.Key] = this.hostId

	// TODO: add labels: LabelPodNamespaceUID, LabelLabels, LabelResourceID

	return metricSetKey, cMetrics
}

func (this *kubeletMetricsSource) ScrapeMetrics(start, end time.Time) *DataBatch {
	containers, err := this.kubeletClient.GetAllRawContainers(this.host, start, end)
	if err != nil {
		glog.Errorf("error while getting containers from Kubelet: %v", err)
	}
	glog.Infof("successfully obtained stats for %v containers", len(containers))

	result := &DataBatch{
		Timestamp:  end,
		MetricSets: map[string]*MetricSet{},
	}
	keys := make(map[string]bool)
	for _, c := range containers {
		name, metrics := this.decodeMetrics(&c)
		if name == "" {
			continue
		}
		result.MetricSets[name] = metrics
		keys[name] = true
	}
	// No remember data for pods that have been removed.
	for key := range this.cpuLastVal {
		if _, ok := keys[key]; !ok {
			delete(this.cpuLastVal, key)
		}
	}
	return result
}

type kubeletProvider struct {
	nodeLister    *cache.StoreToNodeLister
	reflector     *cache.Reflector
	kubeletClient *KubeletClient
	cpuLastVals   map[string]map[string]cpuVal
}

func (this *kubeletProvider) GetMetricsSources() []MetricsSource {
	sources := []MetricsSource{}
	nodes, err := this.nodeLister.List()
	if err != nil {
		glog.Errorf("error while listing nodes: %v", err)
		return sources
	}

	nodeNames := make(map[string]bool)
	for _, node := range nodes.Items {
		nodeNames[node.Name] = true
		hostname, ip, err := getNodeHostnameAndIP(&node)
		if err != nil {
			glog.Errorf("%v", err)
			continue
		}
		if _, ok := this.cpuLastVals[node.Name]; !ok {
			this.cpuLastVals[node.Name] = make(map[string]cpuVal)
		}
		sources = append(sources, &kubeletMetricsSource{
			host:          Host{IP: ip, Port: this.kubeletClient.GetPort()},
			kubeletClient: this.kubeletClient,
			hostname:      hostname,
			hostId:        node.Spec.ExternalID,
			cpuLastVal:    this.cpuLastVals[node.Name],
		})
	}

	for key := range this.cpuLastVals {
		if _, ok := nodeNames[key]; !ok {
			delete(this.cpuLastVals, key)
		}
	}

	return sources
}

func getNodeHostnameAndIP(node *kube_api.Node) (string, string, error) {
	for _, c := range node.Status.Conditions {
		if c.Type == kube_api.NodeReady && c.Status != kube_api.ConditionTrue {
			return "", "", fmt.Errorf("Node %v is not ready", node.Name)
		}
	}
	hostname, ip := node.Name, ""
	for _, addr := range node.Status.Addresses {
		if addr.Type == kube_api.NodeHostName && addr.Address != "" {
			hostname = addr.Address
		}
		if addr.Type == kube_api.NodeInternalIP && addr.Address != "" {
			ip = addr.Address
		}
	}
	if ip != "" {
		return hostname, ip, nil
	}
	return "", "", fmt.Errorf("Node %v has no valid hostname and/or IP address: %v %v", node.Name, hostname, ip)
}

func NewKubeletProvider(uri *url.URL) (MetricsSourceProvider, error) {
	// create clients
	kubeConfig, kubeletConfig, err := getKubeConfigs(uri)
	if err != nil {
		return nil, err
	}
	kubeClient := kube_client.NewOrDie(kubeConfig)
	kubeletClient, err := NewKubeletClient(kubeletConfig)
	if err != nil {
		return nil, err
	}
	// watch nodes
	lw := cache.NewListWatchFromClient(kubeClient, "nodes", kube_api.NamespaceAll, fields.Everything())
	nodeLister := &cache.StoreToNodeLister{Store: cache.NewStore(cache.MetaNamespaceKeyFunc)}
	reflector := cache.NewReflector(lw, &kube_api.Node{}, nodeLister.Store, time.Hour)
	reflector.Run()

	return &kubeletProvider{
		nodeLister:    nodeLister,
		reflector:     reflector,
		kubeletClient: kubeletClient,
		cpuLastVals:   make(map[string]map[string]cpuVal),
	}, nil
}
<commit_msg>Container name parsing for Kubernetes 1.0.*<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kubelet

import (
	"fmt"
	"net/url"
	"strings"
	"time"

	. "k8s.io/heapster/metrics/core"

	"github.com/golang/glog"
	cadvisor "github.com/google/cadvisor/info/v1"
	kube_api "k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/client/cache"
	kube_client "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/fields"
)

const (
	infraContainerName = "POD"
	// TODO: following constants are copied from k8s, change to use them directly
	kubernetesPodNameLabel      = "io.kubernetes.pod.name"
	kubernetesPodNamespaceLabel = "io.kubernetes.pod.namespace"
	kubernetesPodUID            = "io.kubernetes.pod.uid"
	kubernetesContainerLabel    = "io.kubernetes.container.name"

	CustomMetricPrefix = "CM:"
)

type cpuVal struct {
	Val       int64
	Timestamp time.Time
}

// Kubelet-provided metrics for pod and system container.
type kubeletMetricsSource struct {
	host          Host
	kubeletClient *KubeletClient
	hostname      string
	hostId        string
	cpuLastVal    map[string]cpuVal
}

func (this *kubeletMetricsSource) String() string {
	return fmt.Sprintf("kubelet:%s:%d", this.host.IP, this.host.Port)
}

func (this *kubeletMetricsSource) decodeMetrics(c *cadvisor.ContainerInfo) (string, *MetricSet) {
	var metricSetKey string
	cMetrics := &MetricSet{
		MetricValues: map[string]MetricValue{},
		Labels:       map[string]string{},
	}

	if isNode(c) {
		metricSetKey = NodeKey(this.hostname)
		cMetrics.Labels[LabelMetricSetType.Key] = MetricSetTypeNode
	} else if isSysContainer(c) {
		cName := getSysContainerName(c)
		metricSetKey = NodeContainerKey(this.hostname, cName)
		cMetrics.Labels[LabelMetricSetType.Key] = MetricSetTypeSystemContainer
		cMetrics.Labels[LabelContainerName.Key] = cName
	} else {
		cName := c.Spec.Labels[kubernetesContainerLabel]
		if cName == infraContainerName {
			return "", nil
		}
		ns := c.Spec.Labels[kubernetesPodNamespaceLabel]
		podName := c.Spec.Labels[kubernetesPodNameLabel]

		// Support for kubernetes 1.0.*
		if ns == "" && strings.Contains(podName, "/") {
			tokens := strings.SplitN(podName, "/", 2)
			if len(tokens) == 2 {
				ns = tokens[0]
				podName = tokens[1]
			}
		}
		if cName == "" {
			if !strings.HasPrefix(c.Name, "k8s_POD.") {
				// Better this than nothing. This is a temporary hack for new heapster to work
				// with Kubernetes 1.0.*.
				// TODO: fix this with POD list.
				// Parsing name like:
				// k8s_kube-ui.7f9b83f6_kube-ui-v1-bxj1w_kube-system_9abfb0bd-811f-11e5-b548-42010af00002_e6841e8d
				pos := strings.Index(c.Name, ".")
				if pos >= 0 {
					// remove first 4 chars.
					cName = c.Name[len("k8s_"):pos]
				}
			}
		}

		if cName == "" || ns == "" || podName == "" {
			glog.Errorf("Missing metadata for container %v. Got: %+v", c.Name, c.Spec.Labels)
			return "", nil
		}
		metricSetKey = PodContainerKey(ns, podName, cName)
		cMetrics.Labels[LabelMetricSetType.Key] = MetricSetTypePodContainer
		cMetrics.Labels[LabelContainerName.Key] = cName
		cMetrics.Labels[LabelPodId.Key] = c.Spec.Labels[kubernetesPodUID]
		cMetrics.Labels[LabelPodName.Key] = podName
		cMetrics.Labels[LabelNamespaceName.Key] = ns
		// Needed for backward compatibility
		cMetrics.Labels[LabelPodNamespace.Key] = ns
		cMetrics.Labels[LabelContainerBaseImage.Key] = c.Spec.Image
	}

	for _, metric := range StandardMetrics {
		if metric.HasValue(&c.Spec) {
			cMetrics.MetricValues[metric.Name] = metric.GetValue(&c.Spec, c.Stats[0])
		}
	}

	if c.Spec.HasCustomMetrics {
	metricloop:
		for _, spec := range c.Spec.CustomMetrics {
			if cmValue, ok := c.Stats[0].CustomMetrics[spec.Name]; ok && cmValue != nil && len(cmValue) >= 1 {
				newest := cmValue[0]
				for _, metricVal := range cmValue {
					if newest.Timestamp.Before(metricVal.Timestamp) {
						newest = metricVal
					}
				}
				mv := MetricValue{}
				switch spec.Type {
				case cadvisor.MetricGauge:
					mv.MetricType = MetricGauge
				case cadvisor.MetricCumulative:
					mv.MetricType = MetricCumulative
				default:
					glog.V(4).Infof("Skipping %s: unknown custom metric type: %v", spec.Name, spec.Type)
					continue metricloop
				}

				switch spec.Format {
				case cadvisor.IntType:
					mv.ValueType = ValueInt64
					mv.IntValue = newest.IntValue
				case cadvisor.FloatType:
					mv.ValueType = ValueFloat
					mv.FloatValue = float32(newest.FloatValue)
				default:
					glog.V(4).Infof("Skipping %s: unknown custom metric format", spec.Name, spec.Format)
					continue metricloop
				}

				cMetrics.MetricValues[CustomMetricPrefix+spec.Name] = mv
			}
		}
	}

	// This is temporary workaround to support cpu/usege_rate metric.
	if currentVal, ok := cMetrics.MetricValues["cpu/usage"]; ok {
		if lastVal, ok := this.cpuLastVal[metricSetKey]; ok {
			// cpu/usage values are in nanoseconds; we want to have it in millicores (that's why constant 1000 is here).
			rateVal := 1000 * (currentVal.IntValue - lastVal.Val) / (c.Stats[0].Timestamp.UnixNano() - lastVal.Timestamp.UnixNano())
			cMetrics.MetricValues["cpu/usage_rate"] = MetricValue{
				ValueType:  ValueInt64,
				MetricType: MetricGauge,
				IntValue:   rateVal,
			}
		}
		this.cpuLastVal[metricSetKey] = cpuVal{
			Val:       currentVal.IntValue,
			Timestamp: c.Stats[0].Timestamp,
		}
	}

	// common labels
	cMetrics.Labels[LabelHostname.Key] = this.hostname
	cMetrics.Labels[LabelHostID.Key] = this.hostId

	// TODO: add labels: LabelPodNamespaceUID, LabelLabels, LabelResourceID

	return metricSetKey, cMetrics
}

func (this *kubeletMetricsSource) ScrapeMetrics(start, end time.Time) *DataBatch {
	containers, err := this.kubeletClient.GetAllRawContainers(this.host, start, end)
	if err != nil {
		glog.Errorf("error while getting containers from Kubelet: %v", err)
	}
	glog.Infof("successfully obtained stats for %v containers", len(containers))

	result := &DataBatch{
		Timestamp:  end,
		MetricSets: map[string]*MetricSet{},
	}
	keys := make(map[string]bool)
	for _, c := range containers {
		name, metrics := this.decodeMetrics(&c)
		if name == "" {
			continue
		}
		result.MetricSets[name] = metrics
		keys[name] = true
	}
	// No remember data for pods that have been removed.
	for key := range this.cpuLastVal {
		if _, ok := keys[key]; !ok {
			delete(this.cpuLastVal, key)
		}
	}
	return result
}

type kubeletProvider struct {
	nodeLister    *cache.StoreToNodeLister
	reflector     *cache.Reflector
	kubeletClient *KubeletClient
	cpuLastVals   map[string]map[string]cpuVal
}

func (this *kubeletProvider) GetMetricsSources() []MetricsSource {
	sources := []MetricsSource{}
	nodes, err := this.nodeLister.List()
	if err != nil {
		glog.Errorf("error while listing nodes: %v", err)
		return sources
	}

	nodeNames := make(map[string]bool)
	for _, node := range nodes.Items {
		nodeNames[node.Name] = true
		hostname, ip, err := getNodeHostnameAndIP(&node)
		if err != nil {
			glog.Errorf("%v", err)
			continue
		}
		if _, ok := this.cpuLastVals[node.Name]; !ok {
			this.cpuLastVals[node.Name] = make(map[string]cpuVal)
		}
		sources = append(sources, &kubeletMetricsSource{
			host:          Host{IP: ip, Port: this.kubeletClient.GetPort()},
			kubeletClient: this.kubeletClient,
			hostname:      hostname,
			hostId:        node.Spec.ExternalID,
			cpuLastVal:    this.cpuLastVals[node.Name],
		})
	}

	for key := range this.cpuLastVals {
		if _, ok := nodeNames[key]; !ok {
			delete(this.cpuLastVals, key)
		}
	}

	return sources
}

func getNodeHostnameAndIP(node *kube_api.Node) (string, string, error) {
	for _, c := range node.Status.Conditions {
		if c.Type == kube_api.NodeReady && c.Status != kube_api.ConditionTrue {
			return "", "", fmt.Errorf("Node %v is not ready", node.Name)
		}
	}
	hostname, ip := node.Name, ""
	for _, addr := range node.Status.Addresses {
		if addr.Type == kube_api.NodeHostName && addr.Address != "" {
			hostname = addr.Address
		}
		if addr.Type == kube_api.NodeInternalIP && addr.Address != "" {
			ip = addr.Address
		}
	}
	if ip != "" {
		return hostname, ip, nil
	}
	return "", "", fmt.Errorf("Node %v has no valid hostname and/or IP address: %v %v", node.Name, hostname, ip)
}

func NewKubeletProvider(uri *url.URL) (MetricsSourceProvider, error) {
	// create clients
	kubeConfig, kubeletConfig, err := getKubeConfigs(uri)
	if err != nil {
		return nil, err
	}
	kubeClient := kube_client.NewOrDie(kubeConfig)
	kubeletClient, err := NewKubeletClient(kubeletConfig)
	if err != nil {
		return nil, err
	}
	// watch nodes
	lw := cache.NewListWatchFromClient(kubeClient, "nodes", kube_api.NamespaceAll, fields.Everything())
	nodeLister := &cache.StoreToNodeLister{Store: cache.NewStore(cache.MetaNamespaceKeyFunc)}
	reflector := cache.NewReflector(lw, &kube_api.Node{}, nodeLister.Store, time.Hour)
	reflector.Run()

	return &kubeletProvider{
		nodeLister:    nodeLister,
		reflector:     reflector,
		kubeletClient: kubeletClient,
		cpuLastVals:   make(map[string]map[string]cpuVal),
	}, nil
}
<|endoftext|>


<commit_before>package smtpd

import (
	"reflect"
	"strings"
	"testing"
)

func TestGetMailAddressComponents(t *testing.T) {
	name, domain := GetMailAddressComponents("")
	if name != "" || domain != "" {
		t.Fatalf("%s | %s", name, domain)
	}

	name, domain = GetMailAddressComponents("user@example.com")
	if name != "user" || domain != "example.com" {
		t.Fatalf("%s | %s", name, domain)
	}

	name, domain = GetMailAddressComponents("user@")
	if name != "user" || domain != "" {
		t.Fatalf("%s | %s", name, domain)
	}

	name, domain = GetMailAddressComponents("@example.com")
	if name != "" || domain != "example.com" {
		t.Fatalf("%s | %s", name, domain)
	}
}

func TestIsDmarcPolicyEnforcing(t *testing.T) {
	if IsDmarcPolicyEnforcing("") {
		t.Fatal("empty")
	}
	if !IsDmarcPolicyEnforcing("microsoft.com") {
		t.Fatal("microsoft")
	}
	if !IsDmarcPolicyEnforcing("google.com") {
		t.Fatal("google")
	}
	if !IsDmarcPolicyEnforcing("ifttt.com") {
		t.Fatal("ifttt")
	}
	// Test missing DMARC
	if IsDmarcPolicyEnforcing("distrowatch.com") {
		t.Fatal("distrowatch")
	}
	// Test discovery of organisational domain DMARC
	if !IsDmarcPolicyEnforcing("emails.ifttt.com") {
		t.Fatal("emails.ifttt")
	}
	if !IsDmarcPolicyEnforcing("does-not-exist.google.com") {
		t.Fatal("google")
	}
	// Test the artificial limit of how deep the improvised recursive "organisation domain" mechanism goes
	if IsDmarcPolicyEnforcing("1.2.3.4.5.6.7.8.9.microsoft.com") {
		t.Fatal("microsoft recursive over-limit")
	}
}

func TestGetFromAddressWithDmarcWorkaround(t *testing.T) {
	if addr := GetFromAddressWithDmarcWorkaround("", 123); addr != "" {
		t.Fatal(addr)
	}
	// Missing username
	if addr := GetFromAddressWithDmarcWorkaround("microsoft.com", 123); addr != "microsoft.com" {
		t.Fatal(addr)
	}
	// Missing @domain
	if addr := GetFromAddressWithDmarcWorkaround("test-user", 123); addr != "test-user" {
		t.Fatal(addr)
	}
	// DMARC-enforcing domains
	if addr := GetFromAddressWithDmarcWorkaround("user@microsoft.com", 123); addr != "user@microsoft-laitos-nodmarc-123.com" {
		t.Fatal(addr)
	}
	if addr := GetFromAddressWithDmarcWorkaround("user@emails.ifttt.com", 123); addr != "user@emails.ifttt-laitos-nodmarc-123.com" {
		t.Fatal(addr)
	}
	if addr := GetFromAddressWithDmarcWorkaround("user@123.emails.ifttt.com", 123); addr != "user@123.emails.ifttt-laitos-nodmarc-123.com" {
		t.Fatal(addr)
	}
	// Not enforcing DMARC or no DMARC information is available
	if addr := GetFromAddressWithDmarcWorkaround("user@www.b737.org.uk", 123); addr != "user@www.b737.org.uk" {
		t.Fatal(addr)
	}
	if addr := GetFromAddressWithDmarcWorkaround("user@arstechnica.com", 123); addr != "user@arstechnica.com" {
		t.Fatal(addr)
	}
	if addr := GetFromAddressWithDmarcWorkaround("user@distrowatch.com", 123); addr != "user@distrowatch.com" {
		t.Fatal(addr)
	}
}

func TestWithHeaderFromAddr(t *testing.T) {
	if b := WithHeaderFromAddr(nil, "a"); !reflect.DeepEqual(b, []byte{}) {
		t.Fatalf("%+v", b)
	}
	example := `Date: Sun, 24 May 2020 08:48:54 +0000 (UTC)
From: Webhooks via IFTTT <action@ifttt.com>
Reply-To: Do not reply <no-reply@ifttt.com>
Message-ID: <5eca34f1ec12d_2e8a6b3143706f@ip-172-31-1-54.ec2.internal.mail>`
	actual := string(WithHeaderFromAddr([]byte(example), "new@sender.com"))
	if !strings.Contains(actual, "Date: Sun, 24 May 2020 08:48:54 +0000 (UTC)\nFrom: new@sender.com\r\nReply-To: Do not reply <no-reply@ifttt.com>") {
		t.Fatal(actual)
	}
}
<commit_msg>fix a test case failure in dmarc_workaround_test.go<commit_after>package smtpd

import (
	"reflect"
	"strings"
	"testing"
)

func TestGetMailAddressComponents(t *testing.T) {
	name, domain := GetMailAddressComponents("")
	if name != "" || domain != "" {
		t.Fatalf("%s | %s", name, domain)
	}

	name, domain = GetMailAddressComponents("user@example.com")
	if name != "user" || domain != "example.com" {
		t.Fatalf("%s | %s", name, domain)
	}

	name, domain = GetMailAddressComponents("user@")
	if name != "user" || domain != "" {
		t.Fatalf("%s | %s", name, domain)
	}

	name, domain = GetMailAddressComponents("@example.com")
	if name != "" || domain != "example.com" {
		t.Fatalf("%s | %s", name, domain)
	}
}

func TestIsDmarcPolicyEnforcing(t *testing.T) {
	if IsDmarcPolicyEnforcing("") {
		t.Fatal("empty")
	}
	if !IsDmarcPolicyEnforcing("microsoft.com") {
		t.Fatal("microsoft")
	}
	if !IsDmarcPolicyEnforcing("google.com") {
		t.Fatal("google")
	}
	if !IsDmarcPolicyEnforcing("ifttt.com") {
		t.Fatal("ifttt")
	}
	// Test missing DMARC
	if IsDmarcPolicyEnforcing("distrowatch.com") {
		t.Fatal("distrowatch")
	}
	// Test discovery of organisational domain DMARC
	if !IsDmarcPolicyEnforcing("emails.ifttt.com") {
		t.Fatal("emails.ifttt")
	}
	if !IsDmarcPolicyEnforcing("does-not-exist.google.com") {
		t.Fatal("google")
	}
	// Test the artificial limit of how deep the improvised recursive "organisation domain" mechanism goes
	if IsDmarcPolicyEnforcing("1.2.3.4.5.6.7.8.9.microsoft.com") {
		t.Fatal("microsoft recursive over-limit")
	}
}

func TestGetFromAddressWithDmarcWorkaround(t *testing.T) {
	if addr := GetFromAddressWithDmarcWorkaround("", 123); addr != "" {
		t.Fatal(addr)
	}
	// Missing username
	if addr := GetFromAddressWithDmarcWorkaround("microsoft.com", 123); addr != "microsoft.com" {
		t.Fatal(addr)
	}
	// Missing @domain
	if addr := GetFromAddressWithDmarcWorkaround("test-user", 123); addr != "test-user" {
		t.Fatal(addr)
	}
	// DMARC-enforcing domains
	if addr := GetFromAddressWithDmarcWorkaround("user@microsoft.com", 123); addr != "user@microsoft-laitos-nodmarc-123.com" {
		t.Fatal(addr)
	}
	if addr := GetFromAddressWithDmarcWorkaround("user@emails.ifttt.com", 123); addr != "user@emails.ifttt-laitos-nodmarc-123.com" {
		t.Fatal(addr)
	}
	if addr := GetFromAddressWithDmarcWorkaround("user@123.emails.ifttt.com", 123); addr != "user@123.emails.ifttt-laitos-nodmarc-123.com" {
		t.Fatal(addr)
	}
	// Not enforcing DMARC or no DMARC information is available
	if addr := GetFromAddressWithDmarcWorkaround("user@www.b737.org.uk", 123); addr != "user@www.b737.org.uk" {
		t.Fatal(addr)
	}
	if addr := GetFromAddressWithDmarcWorkaround("user@distrowatch.com", 123); addr != "user@distrowatch.com" {
		t.Fatal(addr)
	}
}

func TestWithHeaderFromAddr(t *testing.T) {
	if b := WithHeaderFromAddr(nil, "a"); !reflect.DeepEqual(b, []byte{}) {
		t.Fatalf("%+v", b)
	}
	example := `Date: Sun, 24 May 2020 08:48:54 +0000 (UTC)
From: Webhooks via IFTTT <action@ifttt.com>
Reply-To: Do not reply <no-reply@ifttt.com>
Message-ID: <5eca34f1ec12d_2e8a6b3143706f@ip-172-31-1-54.ec2.internal.mail>`
	actual := string(WithHeaderFromAddr([]byte(example), "new@sender.com"))
	if !strings.Contains(actual, "Date: Sun, 24 May 2020 08:48:54 +0000 (UTC)\nFrom: new@sender.com\r\nReply-To: Do not reply <no-reply@ifttt.com>") {
		t.Fatal(actual)
	}
}
<|endoftext|>


<commit_before>package tailer

import (
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"sync"

	"github.com/golang/glog"
)

var (
	// RegexNotWatch sets the file extension to avoid watching
	RegexNotWatch = regexp.MustCompile("(?:^tailer\\.|^gobzip-|^\\..+\\.swp$|\\.gz$|\\.[0-9]$)")
)

// Tailer init the service functions
type Tailer struct {
	ch          chan bool
	waitGroup   *sync.WaitGroup
	publisher   Publisher
	matchLine   *regexp.Regexp
	numOfTail   int64
	fileLock    sync.Mutex
	visitedFile map[string]bool
}

// Make a new Tailer
func NewTailer(publishToNats bool, config Config) (*Tailer, error) {
	var err error
	t := &Tailer{
		ch:        make(chan bool),
		waitGroup: &sync.WaitGroup{},
	}
	if len(config.Match) > 0 {
		glog.Warningf("Filter line by regex: %s", config.Match)
		t.matchLine, err = regexp.Compile(config.Match)
		if err != nil {
			return nil, err
		}
	}
	if publishToNats {
		natsURL := os.Getenv("NATS_CLUSTER")
		if natsURL == "" {
			natsURL = "nats://localhost:4222"
		}
		t.publisher, err = NewNatsPublisher(natsURL)
		if err != nil {
			return nil, err
		}
	} else {
		t.publisher = &SimplePublisher{}
	}
	return t, nil
}

func (s *Tailer) Serve(watchDirs []string, fileGlob string) {
	// examine the input dir and select how many files to watch and publish
	filesToTail := []string{}
	for _, dir := range watchDirs {
		fileGlobPattern := fmt.Sprintf("%s/%s", dir, fileGlob)
		files, _ := filepath.Glob(fileGlobPattern)
		filesToTail = append(filesToTail, files...)
		glog.Warningf("Files to watch now: %v", filesToTail)
		go s.watchDir(dir)
	}

	for _, filePath := range filesToTail {
		go s.tailFile(filePath)
	}

	s.waitGroup.Wait()
}
<commit_msg>fix nil pointer<commit_after>package tailer

import (
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"sync"

	"github.com/golang/glog"
)

var (
	// RegexNotWatch sets the file extension to avoid watching
	RegexNotWatch = regexp.MustCompile("(?:^tailer\\.|^gobzip-|^\\..+\\.swp$|\\.gz$|\\.[0-9]$)")
)

// Tailer init the service functions
type Tailer struct {
	ch          chan bool
	waitGroup   *sync.WaitGroup
	publisher   Publisher
	matchLine   *regexp.Regexp
	numOfTail   int64
	fileLock    sync.Mutex
	visitedFile map[string]bool
}

// Make a new Tailer
func NewTailer(publishToNats bool, config Config) (*Tailer, error) {
	var err error
	t := &Tailer{
		ch:          make(chan bool),
		waitGroup:   &sync.WaitGroup{},
		visitedFile: map[string]bool{},
	}
	if len(config.Match) > 0 {
		glog.Warningf("Filter line by regex: %s", config.Match)
		t.matchLine, err = regexp.Compile(config.Match)
		if err != nil {
			return nil, err
		}
	}
	if publishToNats {
		natsURL := os.Getenv("NATS_CLUSTER")
		if natsURL == "" {
			natsURL = "nats://localhost:4222"
		}
		t.publisher, err = NewNatsPublisher(natsURL)
		if err != nil {
			return nil, err
		}
	} else {
		t.publisher = &SimplePublisher{}
	}
	return t, nil
}

func (s *Tailer) Serve(watchDirs []string, fileGlob string) {
	// examine the input dir and select how many files to watch and publish
	filesToTail := []string{}
	for _, dir := range watchDirs {
		fileGlobPattern := fmt.Sprintf("%s/%s", dir, fileGlob)
		files, _ := filepath.Glob(fileGlobPattern)
		filesToTail = append(filesToTail, files...)
		glog.Warningf("Files to watch now: %v", filesToTail)
		go s.watchDir(dir)
	}

	for _, filePath := range filesToTail {
		go s.tailFile(filePath)
	}

	s.waitGroup.Wait()
}
<|endoftext|>


<commit_before>package locking

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"sort"
	"testing"
	"time"

	"github.com/git-lfs/git-lfs/lfsapi"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

type LocksById []Lock

func (a LocksById) Len() int           { return len(a) }
func (a LocksById) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a LocksById) Less(i, j int) bool { return a[i].Id < a[j].Id }

func TestRefreshCache(t *testing.T) {
	var err error
	tempDir, err := ioutil.TempDir("", "testCacheLock")
	assert.Nil(t, err)

	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		assert.Equal(t, "GET", r.Method)
		assert.Equal(t, "/api/locks", r.URL.Path)

		w.Header().Set("Content-Type", "application/json")
		err = json.NewEncoder(w).Encode(lockList{
			Locks: []Lock{
				Lock{Id: "99", Path: "folder/test3.dat", Name: "Alice", Email: "alice@wonderland.com"},
				Lock{Id: "101", Path: "folder/test1.dat", Name: "Fred", Email: "fred@bloggs.com"},
				Lock{Id: "102", Path: "folder/test2.dat", Name: "Fred", Email: "fred@bloggs.com"},
				Lock{Id: "103", Path: "root.dat", Name: "Fred", Email: "fred@bloggs.com"},
				Lock{Id: "199", Path: "other/test1.dat", Name: "Charles", Email: "charles@incharge.com"},
			},
		})
		assert.Nil(t, err)
	}))

	defer func() {
		srv.Close()
	}()

	lfsclient, err := lfsapi.NewClient(nil, lfsapi.Env(map[string]string{
		"lfs.url":    srv.URL + "/api",
		"user.name":  "Fred",
		"user.email": "fred@bloggs.com",
	}))
	require.Nil(t, err)

	client, err := NewClient("", lfsclient)
	assert.Nil(t, err)
	assert.Nil(t, client.SetupCache(tempDir))

	// Should start with no cached items
	locks, err := client.SearchLocks(nil, 0, true)
	assert.Nil(t, err)
	assert.Empty(t, locks)

	// Should load from test data, just Fred's
	err = client.refreshLockCache()
	assert.Nil(t, err)

	locks, err = client.SearchLocks(nil, 0, true)
	assert.Nil(t, err)
	// Need to include zero time in structure for equal to work
	zeroTime := time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC)

	// Sort locks for stable comparison
	sort.Sort(LocksById(locks))
	assert.Equal(t, []Lock{
		Lock{Path: "folder/test1.dat", Id: "101", Name: "Fred", Email: "fred@bloggs.com", LockedAt: zeroTime},
		Lock{Path: "folder/test2.dat", Id: "102", Name: "Fred", Email: "fred@bloggs.com", LockedAt: zeroTime},
		Lock{Path: "root.dat", Id: "103", Name: "Fred", Email: "fred@bloggs.com", LockedAt: zeroTime},
	}, locks)

}
<commit_msg>locking: fix method name<commit_after>package locking

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"sort"
	"testing"
	"time"

	"github.com/git-lfs/git-lfs/lfsapi"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

type LocksById []Lock

func (a LocksById) Len() int           { return len(a) }
func (a LocksById) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a LocksById) Less(i, j int) bool { return a[i].Id < a[j].Id }

func TestRefreshCache(t *testing.T) {
	var err error
	tempDir, err := ioutil.TempDir("", "testCacheLock")
	assert.Nil(t, err)

	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		assert.Equal(t, "GET", r.Method)
		assert.Equal(t, "/api/locks", r.URL.Path)

		w.Header().Set("Content-Type", "application/json")
		err = json.NewEncoder(w).Encode(lockList{
			Locks: []Lock{
				Lock{Id: "99", Path: "folder/test3.dat", Name: "Alice", Email: "alice@wonderland.com"},
				Lock{Id: "101", Path: "folder/test1.dat", Name: "Fred", Email: "fred@bloggs.com"},
				Lock{Id: "102", Path: "folder/test2.dat", Name: "Fred", Email: "fred@bloggs.com"},
				Lock{Id: "103", Path: "root.dat", Name: "Fred", Email: "fred@bloggs.com"},
				Lock{Id: "199", Path: "other/test1.dat", Name: "Charles", Email: "charles@incharge.com"},
			},
		})
		assert.Nil(t, err)
	}))

	defer func() {
		srv.Close()
	}()

	lfsclient, err := lfsapi.NewClient(nil, lfsapi.Env(map[string]string{
		"lfs.url":    srv.URL + "/api",
		"user.name":  "Fred",
		"user.email": "fred@bloggs.com",
	}))
	require.Nil(t, err)

	client, err := NewClient("", lfsclient)
	assert.Nil(t, err)
	assert.Nil(t, client.SetupFileCache(tempDir))

	// Should start with no cached items
	locks, err := client.SearchLocks(nil, 0, true)
	assert.Nil(t, err)
	assert.Empty(t, locks)

	// Should load from test data, just Fred's
	err = client.refreshLockCache()
	assert.Nil(t, err)

	locks, err = client.SearchLocks(nil, 0, true)
	assert.Nil(t, err)
	// Need to include zero time in structure for equal to work
	zeroTime := time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC)

	// Sort locks for stable comparison
	sort.Sort(LocksById(locks))
	assert.Equal(t, []Lock{
		Lock{Path: "folder/test1.dat", Id: "101", Name: "Fred", Email: "fred@bloggs.com", LockedAt: zeroTime},
		Lock{Path: "folder/test2.dat", Id: "102", Name: "Fred", Email: "fred@bloggs.com", LockedAt: zeroTime},
		Lock{Path: "root.dat", Id: "103", Name: "Fred", Email: "fred@bloggs.com", LockedAt: zeroTime},
	}, locks)

}
<|endoftext|>


<commit_before>package goperrin

import "fmt"

func perrin() func() int {
	a, b, c := 3, 0, 2

	return func() int {
		a, b, c = b, c, a+b
		return a
	}
}

func perrin_max(max int) func() int {
	a, b, c := 3, 0, 2

	return func() int {
		if a >= max {
			return a
		}

		a, b, c = b, c, a+b
		return a
	}
}

func perrin_reset(max int) func() int {
	a, b, c := 3, 0, 2

	return func() int {
		if a >= max {
			a, b, c = 3, 0, 2
		}

		a, b, c = b, c, a+b
		return a
	}
}
<commit_msg>Remove un-nneded import<commit_after>package goperrin

func perrin() func() int {
	a, b, c := 3, 0, 2

	return func() int {
		a, b, c = b, c, a+b
		return a
	}
}

func perrin_max(max int) func() int {
	a, b, c := 3, 0, 2

	return func() int {
		if a >= max {
			return a
		}

		a, b, c = b, c, a+b
		return a
	}
}

func perrin_reset(max int) func() int {
	a, b, c := 3, 0, 2

	return func() int {
		if a >= max {
			a, b, c = 3, 0, 2
		}

		a, b, c = b, c, a+b
		return a
	}
}
<|endoftext|>


<commit_before>package cluster

import (
	"sync"
	"time"

	"github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/endpoints"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	log "github.com/lxc/lxd/shared/log15"
	"github.com/lxc/lxd/shared/logger"
)

var listeners = map[string]*lxd.EventListener{}
var listenersLock sync.Mutex

// EventsUpdateListeners refreshes the cluster event listener connections.
func EventsUpdateListeners(endpoints *endpoints.Endpoints, cluster *db.Cluster, serverCert func() *shared.CertInfo, members map[int64]APIHeartbeatMember, f func(int64, api.Event)) {
	// If no heartbeat members provided, populate from global database.
	if members == nil {
		var dbMembers []db.NodeInfo
		var offlineThreshold time.Duration

		err := cluster.Transaction(func(tx *db.ClusterTx) error {
			var err error

			dbMembers, err = tx.GetNodes()
			if err != nil {
				return err
			}

			offlineThreshold, err = tx.GetNodeOfflineThreshold()
			if err != nil {
				return err
			}

			return nil
		})
		if err != nil {
			logger.Warn("Failed to get current cluster members", log.Ctx{"err": err})
			return
		}

		members = make(map[int64]APIHeartbeatMember, len(dbMembers))
		for _, dbMember := range dbMembers {
			members[dbMember.ID] = APIHeartbeatMember{
				ID:            dbMember.ID,
				Name:          dbMember.Name,
				Address:       dbMember.Address,
				LastHeartbeat: dbMember.Heartbeat,
				Online:        !dbMember.IsOffline(offlineThreshold),
			}
		}
	}

	networkAddress := endpoints.NetworkAddress()

	keepListeners := make(map[string]struct{})
	wg := sync.WaitGroup{}
	for _, member := range members {
		// Don't bother trying to connect to ourselves or offline members.
		if member.Address == networkAddress || !member.Online {
			continue
		}

		listenersLock.Lock()
		listener, ok := listeners[member.Address]

		// If the member already has a listener associated to it, check that the listener is still active.
		// If it is, just move on to next member, but if not then we'll try to connect again.
		if ok {
			if listeners[member.Address].IsActive() {
				keepListeners[member.Address] = struct{}{} // Add to current listeners list.
				listenersLock.Unlock()
				continue
			}

			listener.Disconnect()
			delete(listeners, member.Address)
			logger.Info("Removed inactive member event listener", log.Ctx{"local": networkAddress, "remote": member.Address})
		}
		listenersLock.Unlock()

		keepListeners[member.Address] = struct{}{} // Add to current listeners list.

		// Connect to remote concurrently and add to active listeners if successful.
		wg.Add(1)
		go func(m APIHeartbeatMember) {
			defer wg.Done()
			listener, err := eventsConnect(m.Address, endpoints.NetworkCert(), serverCert())
			if err != nil {
				logger.Warn("Failed adding member event listener", log.Ctx{"local": networkAddress, "remote": m.Address, "err": err})
				return
			}

			listener.AddHandler(nil, func(event api.Event) { f(m.ID, event) })

			listenersLock.Lock()
			listeners[m.Address] = listener
			logger.Info("Added member event listener", log.Ctx{"local": networkAddress, "remote": m.Address})
			listenersLock.Unlock()
		}(member)
	}

	wg.Wait()

	// Disconnect and delete any out of date listeners.
	listenersLock.Lock()
	for address, listener := range listeners {
		if _, found := keepListeners[address]; !found {
			listener.Disconnect()
			delete(listeners, address)
			logger.Info("Removed old member event listener", log.Ctx{"local": networkAddress, "remote": address})
		}
	}
	listenersLock.Unlock()
}

// Establish a client connection to get events from the given node.
func eventsConnect(address string, networkCert *shared.CertInfo, serverCert *shared.CertInfo) (*lxd.EventListener, error) {
	client, err := Connect(address, networkCert, serverCert, nil, true)
	if err != nil {
		return nil, err
	}

	return client.GetEventsAllProjects()
}
<commit_msg>lxd/cluster/events: Adds EventListenerWait to allow waiting for a event listener to connect<commit_after>package cluster

import (
	"context"
	"sync"
	"time"

	"github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/endpoints"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	log "github.com/lxc/lxd/shared/log15"
	"github.com/lxc/lxd/shared/logger"
)

var listeners = map[string]*lxd.EventListener{}
var listenersNotify = map[string]chan struct{}{}
var listenersLock sync.Mutex

// EventListenerWait waits for there to be listener connected to the specified address.
func EventListenerWait(ctx context.Context, address string) (*lxd.EventListener, error) {
	// Check if there is already a listener.
	listenersLock.Lock()
	listener, found := listeners[address]
	if found && listener.IsActive() {
		listenersLock.Unlock()
		return listener, nil
	}

	// If not then create a notify channel if doesn't exist already.
	listenerNotify, found := listenersNotify[address]
	if !found {
		listenerNotify = make(chan struct{})
		listenersNotify[address] = listenerNotify
	}
	listenersLock.Unlock()

	// Wait for the notify channel to be closed (indicating a new listener has been connected), and return it.
	select {
	case <-listenerNotify:
		listenersLock.Lock()
		defer listenersLock.Unlock()
		return listeners[address], nil
	case <-ctx.Done():
		return nil, ctx.Err()
	}
}

// EventsUpdateListeners refreshes the cluster event listener connections.
func EventsUpdateListeners(endpoints *endpoints.Endpoints, cluster *db.Cluster, serverCert func() *shared.CertInfo, members map[int64]APIHeartbeatMember, f func(int64, api.Event)) {
	// If no heartbeat members provided, populate from global database.
	if members == nil {
		var dbMembers []db.NodeInfo
		var offlineThreshold time.Duration

		err := cluster.Transaction(func(tx *db.ClusterTx) error {
			var err error

			dbMembers, err = tx.GetNodes()
			if err != nil {
				return err
			}

			offlineThreshold, err = tx.GetNodeOfflineThreshold()
			if err != nil {
				return err
			}

			return nil
		})
		if err != nil {
			logger.Warn("Failed to get current cluster members", log.Ctx{"err": err})
			return
		}

		members = make(map[int64]APIHeartbeatMember, len(dbMembers))
		for _, dbMember := range dbMembers {
			members[dbMember.ID] = APIHeartbeatMember{
				ID:            dbMember.ID,
				Name:          dbMember.Name,
				Address:       dbMember.Address,
				LastHeartbeat: dbMember.Heartbeat,
				Online:        !dbMember.IsOffline(offlineThreshold),
			}
		}
	}

	networkAddress := endpoints.NetworkAddress()

	keepListeners := make(map[string]struct{})
	wg := sync.WaitGroup{}
	for _, member := range members {
		// Don't bother trying to connect to ourselves or offline members.
		if member.Address == networkAddress || !member.Online {
			continue
		}

		listenersLock.Lock()
		listener, ok := listeners[member.Address]

		// If the member already has a listener associated to it, check that the listener is still active.
		// If it is, just move on to next member, but if not then we'll try to connect again.
		if ok {
			if listeners[member.Address].IsActive() {
				keepListeners[member.Address] = struct{}{} // Add to current listeners list.
				listenersLock.Unlock()
				continue
			}

			// Disconnect and delete listener, but don't delete any listenersNotify entry as there
			// might be something waiting for a future connection.
			listener.Disconnect()
			delete(listeners, member.Address)
			logger.Info("Removed inactive member event listener", log.Ctx{"local": networkAddress, "remote": member.Address})
		}
		listenersLock.Unlock()

		keepListeners[member.Address] = struct{}{} // Add to current listeners list.

		// Connect to remote concurrently and add to active listeners if successful.
		wg.Add(1)
		go func(m APIHeartbeatMember) {
			defer wg.Done()
			listener, err := eventsConnect(m.Address, endpoints.NetworkCert(), serverCert())
			if err != nil {
				logger.Warn("Failed adding member event listener", log.Ctx{"local": networkAddress, "remote": m.Address, "err": err})
				return
			}

			listener.AddHandler(nil, func(event api.Event) { f(m.ID, event) })

			listenersLock.Lock()
			listeners[m.Address] = listener

			// If there is something waiting to be notify, then notify them and delete the notifier.
			listenerNotify, found := listenersNotify[m.Address]
			if found {
				close(listenerNotify)
				delete(listenersNotify, m.Address)
			}

			logger.Info("Added member event listener", log.Ctx{"local": networkAddress, "remote": m.Address})
			listenersLock.Unlock()
		}(member)
	}

	wg.Wait()

	// Disconnect and delete any out of date listeners and their notifiers.
	listenersLock.Lock()
	for address, listener := range listeners {
		if _, found := keepListeners[address]; !found {
			listener.Disconnect()
			delete(listeners, address)
			delete(listenersNotify, address)
			logger.Info("Removed old member event listener", log.Ctx{"local": networkAddress, "remote": address})
		}
	}
	listenersLock.Unlock()
}

// Establish a client connection to get events from the given node.
func eventsConnect(address string, networkCert *shared.CertInfo, serverCert *shared.CertInfo) (*lxd.EventListener, error) {
	client, err := Connect(address, networkCert, serverCert, nil, true)
	if err != nil {
		return nil, err
	}

	return client.GetEventsAllProjects()
}
<|endoftext|>


<commit_before>// Copyright 2017 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package integrations

import (
	"net/http"
	"testing"

	"github.com/PuerkitoBio/goquery"
	"github.com/stretchr/testify/assert"
)

func resultFilenames(t testing.TB, doc *HTMLDoc) []string {
	resultsSelection := doc.doc.Find(".repository.search")
	assert.EqualValues(t, 1, resultsSelection.Length(),
		"Invalid template (repo search template has changed?)")
	filenameSelections := resultsSelection.Find(".repo-search-result").Find(".header").Find("span.file")
	result := make([]string, filenameSelections.Length())
	filenameSelections.Each(func(i int, selection *goquery.Selection) {
		result[i] = selection.Text()
	})
	return result
}

func TestSearchRepo(t *testing.T) {
	prepareTestEnv(t)

	req := NewRequestf(t, "GET", "/user2/repo1/search?q=Description&page=1")
	resp := MakeRequest(t, req, http.StatusOK)

	filenames := resultFilenames(t, NewHTMLParser(t, resp.Body))
	assert.EqualValues(t, []string{"README.md"}, filenames)
}
<commit_msg>Fix TestSearchRepo by waiting till indexing is done (#7004)<commit_after>// Copyright 2017 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package integrations

import (
	"log"
	"net/http"
	"testing"
	"time"

	"code.gitea.io/gitea/models"

	"github.com/PuerkitoBio/goquery"
	"github.com/stretchr/testify/assert"
)

func resultFilenames(t testing.TB, doc *HTMLDoc) []string {
	resultsSelection := doc.doc.Find(".repository.search")
	assert.EqualValues(t, 1, resultsSelection.Length(),
		"Invalid template (repo search template has changed?)")
	filenameSelections := resultsSelection.Find(".repo-search-result").Find(".header").Find("span.file")
	result := make([]string, filenameSelections.Length())
	filenameSelections.Each(func(i int, selection *goquery.Selection) {
		result[i] = selection.Text()
	})
	return result
}

func TestSearchRepo(t *testing.T) {
	prepareTestEnv(t)

	repo, err := models.GetRepositoryByOwnerAndName("user2", "repo1")
	assert.NoError(t, err)

	models.UpdateRepoIndexer(repo)

	log.Printf("Waiting for indexing\n")

	i := 0
	for i < 60 {
		if repo.IndexerStatus != nil && len(repo.IndexerStatus.CommitSha) != 0 {
			break
		}
		time.Sleep(1 * time.Second)
		i++
	}
	if i < 60 {
		log.Printf("Indexing took: %ds\n", i)
	} else {
		log.Printf("Waited the limit: %ds for indexing, continuing\n", i)
	}

	req := NewRequestf(t, "GET", "/user2/repo1/search?q=Description&page=1")
	resp := MakeRequest(t, req, http.StatusOK)

	filenames := resultFilenames(t, NewHTMLParser(t, resp.Body))
	assert.EqualValues(t, []string{"README.md"}, filenames)
}
<|endoftext|>


<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package export

import (
	"context"
	"fmt"
	"io"

	"golang.org/x/tools/internal/telemetry"
)

// LogWriter returns an observer that logs events to the supplied writer.
// If onlyErrors is true it does not log any event that did not have an
// associated error.
// It ignores all telemetry other than log events.
func LogWriter(w io.Writer, onlyErrors bool) Exporter {
	return &logWriter{writer: w, onlyErrors: onlyErrors}
}

type logWriter struct {
	writer     io.Writer
	onlyErrors bool
}

func (w *logWriter) StartSpan(context.Context, *telemetry.Span)  {}
func (w *logWriter) FinishSpan(context.Context, *telemetry.Span) {}
func (w *logWriter) Log(ctx context.Context, event telemetry.Event) {
	if event.Error == nil {
		// we only log errors by default
		return
	}
	fmt.Fprintf(w.writer, "%v\n", event)
}
func (w *logWriter) Metric(context.Context, telemetry.MetricData) {}
func (w *logWriter) Flush()                                       {}
<commit_msg>internal/telemetry: obey the onlyErrors flag in the log writer<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package export

import (
	"context"
	"fmt"
	"io"

	"golang.org/x/tools/internal/telemetry"
)

// LogWriter returns an observer that logs events to the supplied writer.
// If onlyErrors is true it does not log any event that did not have an
// associated error.
// It ignores all telemetry other than log events.
func LogWriter(w io.Writer, onlyErrors bool) Exporter {
	return &logWriter{writer: w, onlyErrors: onlyErrors}
}

type logWriter struct {
	writer     io.Writer
	onlyErrors bool
}

func (w *logWriter) StartSpan(context.Context, *telemetry.Span)  {}
func (w *logWriter) FinishSpan(context.Context, *telemetry.Span) {}
func (w *logWriter) Log(ctx context.Context, event telemetry.Event) {
	if w.onlyErrors && event.Error == nil {
		return
	}
	fmt.Fprintf(w.writer, "%v\n", event)
}
func (w *logWriter) Metric(context.Context, telemetry.MetricData) {}
func (w *logWriter) Flush()                                       {}
<|endoftext|>


<commit_before>// Copyright © 2017 Toomore Chiang
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package cmd

import (
	"encoding/csv"
	"fmt"
	"log"
	"os"
	"time"

	"github.com/spf13/cobra"
)

var (
	dryRun *bool
)

type user struct {
	email  string
	groups string
	fname  string
	lname  string
}

func readCSV(path string) []user {
	file, err := os.Open(path)
	if err != nil {
		log.Fatal(err)
	}
	data, err := csv.NewReader(file).ReadAll()
	if err != nil {
		log.Fatal(err)
	}
	result := make([]user, len(data[1:]))
	for i, v := range data[0] {
		switch v {
		case "email":
			for di, dv := range data[1:] {
				result[di].email = dv[i]
			}
		case "groups":
			for di, dv := range data[1:] {
				result[di].groups = dv[i]
			}
		case "f_name":
			for di, dv := range data[1:] {
				result[di].fname = dv[i]
			}
		case "l_name":
			for di, dv := range data[1:] {
				result[di].lname = dv[i]
			}
		}
	}
	return result
}

func insertInto(data []user) {
	stmt, err := conn.Prepare(`INSERT INTO user(email,groups,f_name,l_name)
	                           VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE f_name=?, l_name=?`)
	if err != nil {
		log.Fatal(err)
	}
	for _, v := range data {
		if result, err := stmt.Exec(v.email, v.groups, v.fname, v.lname, v.fname, v.lname); err == nil {
			insertID, _ := result.LastInsertId()
			rowAff, _ := result.RowsAffected()
			log.Println("LastInsertId", insertID, "RowsAffected", rowAff)
		} else {
			log.Println("[Err]", err)
		}
	}
}

func readUser(group string) {
	rows, err := conn.Query(`SELECT email,groups,created FROM user where groups=?`, group)
	defer rows.Close()
	if err != nil {
		log.Fatal(">>>>>", err)
	}
	var email string
	var groups string
	var created time.Time
	for rows.Next() {
		if err := rows.Scan(&email, &groups, &created); err != nil {
			log.Println(err)
		}
		log.Println(email, groups, created)
	}
}

var userCmd = &cobra.Command{
	Use:   "user",
	Short: "user info",
	Long:  `Import user from csv`,
	Run: func(cmd *cobra.Command, args []string) {
		cmd.Help()
	},
}

var importCmd = &cobra.Command{
	Use:   "import [csv path ...]",
	Short: "Import user from csv",
	Long:  "Import user data from csv file",
	Run: func(cmd *cobra.Command, args []string) {
		for n, path := range args {
			log.Printf(">>> Read csv[%d]: `%s`", n, path)
			if *dryRun {
				log.Println(">>> Dry Run data")
				for i, v := range readCSV(path) {
					fmt.Printf("%d %+v\n", i, v)
				}
			} else {
				insertInto(readCSV(path))
			}
		}
	},
}

var showCmd = &cobra.Command{
	Use:   "show [groups ...]",
	Short: "Show users",
	Long:  "Show all/group users",
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) == 0 {
			cmd.Help()
		} else {
			for _, g := range args {
				fmt.Printf("----- %s -----\n", g)
				readUser(g)
			}
		}
	},
}

func init() {
	dryRun = importCmd.Flags().BoolP("dryRun", "d", false, "Dry run read csv data")

	RootCmd.AddCommand(userCmd)
	userCmd.AddCommand(importCmd, showCmd)
}
<commit_msg>Using tabwriter for user show<commit_after>// Copyright © 2017 Toomore Chiang
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package cmd

import (
	"encoding/csv"
	"fmt"
	"log"
	"os"
	"text/tabwriter"
	"time"

	"github.com/spf13/cobra"
)

var (
	dryRun *bool
)

type user struct {
	email  string
	groups string
	fname  string
	lname  string
}

func readCSV(path string) []user {
	file, err := os.Open(path)
	if err != nil {
		log.Fatal(err)
	}
	data, err := csv.NewReader(file).ReadAll()
	if err != nil {
		log.Fatal(err)
	}
	result := make([]user, len(data[1:]))
	for i, v := range data[0] {
		switch v {
		case "email":
			for di, dv := range data[1:] {
				result[di].email = dv[i]
			}
		case "groups":
			for di, dv := range data[1:] {
				result[di].groups = dv[i]
			}
		case "f_name":
			for di, dv := range data[1:] {
				result[di].fname = dv[i]
			}
		case "l_name":
			for di, dv := range data[1:] {
				result[di].lname = dv[i]
			}
		}
	}
	return result
}

func insertInto(data []user) {
	stmt, err := conn.Prepare(`INSERT INTO user(email,groups,f_name,l_name)
	                           VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE f_name=?, l_name=?`)
	if err != nil {
		log.Fatal(err)
	}
	for _, v := range data {
		if result, err := stmt.Exec(v.email, v.groups, v.fname, v.lname, v.fname, v.lname); err == nil {
			insertID, _ := result.LastInsertId()
			rowAff, _ := result.RowsAffected()
			log.Println("LastInsertId", insertID, "RowsAffected", rowAff)
		} else {
			log.Println("[Err]", err)
		}
	}
}

func readUser(group string) {
	rows, err := conn.Query(`SELECT id,email,f_name,l_name,created FROM user WHERE groups=?`, group)
	defer rows.Close()
	if err != nil {
		log.Fatal(err)
	}
	var (
		id      string
		email   string
		fname   string
		lname   string
		created time.Time
	)
	w := tabwriter.NewWriter(os.Stdout, 0, 0, 0, ' ', tabwriter.AlignRight|tabwriter.Debug)
	fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", "id", "email", "fname", "lname", "created")
	for rows.Next() {
		if err := rows.Scan(&id, &email, &fname, &lname, &created); err != nil {
			log.Println(err)
		} else {
			fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", id, email, fname, lname, created)
		}
	}
	fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", "id", "email", "fname", "lname", "created")
	w.Flush()
}

var userCmd = &cobra.Command{
	Use:   "user",
	Short: "user info",
	Long:  `Import user from csv`,
	Run: func(cmd *cobra.Command, args []string) {
		cmd.Help()
	},
}

var importCmd = &cobra.Command{
	Use:   "import [csv path ...]",
	Short: "Import user from csv",
	Long:  "Import user data from csv file",
	Run: func(cmd *cobra.Command, args []string) {
		for n, path := range args {
			log.Printf(">>> Read csv[%d]: `%s`", n, path)
			if *dryRun {
				log.Println(">>> Dry Run data")
				for i, v := range readCSV(path) {
					fmt.Printf("%d %+v\n", i, v)
				}
			} else {
				insertInto(readCSV(path))
			}
		}
	},
}

var showCmd = &cobra.Command{
	Use:   "show [groups ...]",
	Short: "Show users",
	Long:  "Show all/group users",
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) == 0 {
			cmd.Help()
		} else {
			for _, g := range args {
				fmt.Printf("----- %s -----\n", g)
				readUser(g)
			}
		}
	},
}

func init() {
	dryRun = importCmd.Flags().BoolP("dryRun", "d", false, "Dry run read csv data")

	RootCmd.AddCommand(userCmd)
	userCmd.AddCommand(importCmd, showCmd)
}
<|endoftext|>


<commit_before>// Package basex generates alpha id (alphanumeric id) for big integers.  This
// is particularly useful for shortening URLs.
package basex

import (
	"math/big"
	"strconv"
)

var (
	dictionary = []byte{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'}
)

// Encode converts the big integer to alpha id (an alphanumeric id with mixed cases)
func Encode(val string) string {
	var result []byte
	var index int
	var strVal string

	base := big.NewInt(int64(len(dictionary)))
	a := big.NewInt(0)
	b := big.NewInt(0)
	c := big.NewInt(0)
	d := big.NewInt(0)

	exponent := 1

	remaining := big.NewInt(0)
	remaining.SetString(val, 10)

	for remaining.Cmp(big.NewInt(0)) != 0 {

		a.Exp(base, big.NewInt(int64(exponent)), nil) //16^1 = 16
		b = b.Mod(remaining, a)                      //119 % 16 = 7 | 112 % 256 = 112
		c = c.Exp(base, big.NewInt(int64(exponent-1)), nil)
		d = d.Div(b, c)

		//if d > dictionary.length, we have a problem. but BigInteger doesnt have
		//a greater than method :-(  hope for the best. theoretically, d is always
		//an index of the dictionary!
		strVal = d.String()
		index, _ = strconv.Atoi(strVal)
		result = append(result, dictionary[index])
		remaining = remaining.Sub(remaining, b) //119 - 7 = 112 | 112 - 112 = 0
		exponent = exponent + 1
	}

	//need to reverse it, since the start of the list contains the least significant values
	return string(reverse(result))
}

// Decode converts the alpha id to big integer
func Decode(s string) string {
	//reverse it, coz its already reversed!
	chars2 := reverse([]byte(s))

	//for efficiency, make a map
	var dictMap map[byte]*big.Int
	dictMap = make(map[byte]*big.Int)

	j := 0
	for _, val := range dictionary {
		dictMap[val] = big.NewInt(int64(j))
		j = j + 1
	}

	bi := big.NewInt(0)
	base := big.NewInt(int64(len(dictionary)))

	exponent := 0
	a := big.NewInt(0)
	b := big.NewInt(0)
	intermed := big.NewInt(0)

	for _, c := range chars2 {
		a = dictMap[c]
		intermed = intermed.Exp(base, big.NewInt(int64(exponent)), nil)
		b = b.Mul(intermed, a)
		bi = bi.Add(bi, b)
		exponent = exponent + 1
	}
	return bi.String()
}

func reverse(bs []byte) []byte {
	for i, j := 0, len(bs)-1; i < j; i, j = i+1, j-1 {
		bs[i], bs[j] = bs[j], bs[i]
	}
	return bs
}
<commit_msg>grind<commit_after>// Package basex generates alpha id (alphanumeric id) for big integers.  This
// is particularly useful for shortening URLs.
package basex

import (
	"math/big"
	"strconv"
)

var (
	dictionary = []byte{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'}
)

// Encode converts the big integer to alpha id (an alphanumeric id with mixed cases)
func Encode(val string) string {
	var result []byte
	var index int
	var strVal string

	base := big.NewInt(int64(len(dictionary)))
	a := big.NewInt(0)
	b := big.NewInt(0)
	c := big.NewInt(0)
	d := big.NewInt(0)

	exponent := 1

	remaining := big.NewInt(0)
	remaining.SetString(val, 10)

	for remaining.Cmp(big.NewInt(0)) != 0 {
		a.Exp(base, big.NewInt(int64(exponent)), nil) //16^1 = 16
		b = b.Mod(remaining, a)                       //119 % 16 = 7 | 112 % 256 = 112
		c = c.Exp(base, big.NewInt(int64(exponent-1)), nil)
		d = d.Div(b, c)

		//if d > dictionary.length, we have a problem. but BigInteger doesnt have
		//a greater than method :-(  hope for the best. theoretically, d is always
		//an index of the dictionary!
		strVal = d.String()
		index, _ = strconv.Atoi(strVal)
		result = append(result, dictionary[index])
		remaining = remaining.Sub(remaining, b) //119 - 7 = 112 | 112 - 112 = 0
		exponent = exponent + 1
	}

	//need to reverse it, since the start of the list contains the least significant values
	return string(reverse(result))
}

// Decode converts the alpha id to big integer
func Decode(s string) string {
	//reverse it, coz its already reversed!
	chars2 := reverse([]byte(s))

	//for efficiency, make a map
	dictMap := make(map[byte]*big.Int)

	j := 0
	for _, val := range dictionary {
		dictMap[val] = big.NewInt(int64(j))
		j = j + 1
	}

	bi := big.NewInt(0)
	base := big.NewInt(int64(len(dictionary)))

	exponent := 0
	a := big.NewInt(0)
	b := big.NewInt(0)
	intermed := big.NewInt(0)

	for _, c := range chars2 {
		a = dictMap[c]
		intermed = intermed.Exp(base, big.NewInt(int64(exponent)), nil)
		b = b.Mul(intermed, a)
		bi = bi.Add(bi, b)
		exponent = exponent + 1
	}
	return bi.String()
}

func reverse(bs []byte) []byte {
	for i, j := 0, len(bs)-1; i < j; i, j = i+1, j-1 {
		bs[i], bs[j] = bs[j], bs[i]
	}
	return bs
}
<|endoftext|>


<commit_before>/*
Copyright IBM Corp. All Rights Reserved.

SPDX-License-Identifier: Apache-2.0
*/

package chaincode_test

import (
	"github.com/hyperledger/fabric/core/chaincode"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"
)

var _ = Describe("ActiveTransactions", func() {
	var activeTx *chaincode.ActiveTransactions

	BeforeEach(func() {
		activeTx = chaincode.NewActiveTransactions()
	})

	It("tracks active transactions", func() {
		// Add unique transactions
		ok := activeTx.Add("channel-id", "tx-id")
		Expect(ok).To(BeTrue(), "a new transaction should return true")
		ok = activeTx.Add("channel-id", "tx-id-2")
		Expect(ok).To(BeTrue(), "adding a different transaction id should return true")
		ok = activeTx.Add("channel-id-2", "tx-id")
		Expect(ok).To(BeTrue(), "adding a different channel-id should return true")

		// Attempt to add a transaction that already exists
		ok = activeTx.Add("channel-id", "tx-id")
		Expect(ok).To(BeFalse(), "attempting to an existing transaction should return false")

		// Remove existing and make sure the ID can be reused
		activeTx.Remove("channel-id", "tx-id")
		ok = activeTx.Add("channel-id", "tx-id")
		Expect(ok).To(BeTrue(), "using a an id that has been removed should return true")
	})

	DescribeTable("NewTxKey",
		func(channelID, txID, expected string) {
			result := chaincode.NewTxKey(channelID, txID)
			Expect(result).To(Equal(expected))
		},
		Entry("empty channel and tx", "", "", ""),
		Entry("empty channel", "", "tx-1", "tx-1"),
		Entry("empty tx", "chan-1", "tx-1", "chan-1tx-1"),
		Entry("channel and tx", "chan-1", "tx-1", "chan-1tx-1"),
	)
})
<commit_msg>[FAB-16128] fix test sending an empty tx<commit_after>/*
Copyright IBM Corp. All Rights Reserved.

SPDX-License-Identifier: Apache-2.0
*/

package chaincode_test

import (
	"github.com/hyperledger/fabric/core/chaincode"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"
)

var _ = Describe("ActiveTransactions", func() {
	var activeTx *chaincode.ActiveTransactions

	BeforeEach(func() {
		activeTx = chaincode.NewActiveTransactions()
	})

	It("tracks active transactions", func() {
		// Add unique transactions
		ok := activeTx.Add("channel-id", "tx-id")
		Expect(ok).To(BeTrue(), "a new transaction should return true")
		ok = activeTx.Add("channel-id", "tx-id-2")
		Expect(ok).To(BeTrue(), "adding a different transaction id should return true")
		ok = activeTx.Add("channel-id-2", "tx-id")
		Expect(ok).To(BeTrue(), "adding a different channel-id should return true")

		// Attempt to add a transaction that already exists
		ok = activeTx.Add("channel-id", "tx-id")
		Expect(ok).To(BeFalse(), "attempting to an existing transaction should return false")

		// Remove existing and make sure the ID can be reused
		activeTx.Remove("channel-id", "tx-id")
		ok = activeTx.Add("channel-id", "tx-id")
		Expect(ok).To(BeTrue(), "using a an id that has been removed should return true")
	})

	DescribeTable("NewTxKey",
		func(channelID, txID, expected string) {
			result := chaincode.NewTxKey(channelID, txID)
			Expect(result).To(Equal(expected))
		},
		Entry("empty channel and tx", "", "", ""),
		Entry("empty channel", "", "tx-1", "tx-1"),
		Entry("empty tx", "chan-1", "", "chan-1"),
		Entry("channel and tx", "chan-1", "tx-1", "chan-1tx-1"),
	)
})
<|endoftext|>


<commit_before>package whookie

import (
	"encoding/json"
	"fmt"
	"math"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/go-redis/redis"
)

var (
	client *http.Client
)

func init() {
	client = &http.Client{}
}

// Batch represents a group of events that occurred in a given timeperiod for a domain
type Batch struct {
	DomainId  string
	Timestamp string
	StartedAt time.Time
	Events    []Event
}

func (b Batch) BlankClone() Batch {
	return Batch{
		DomainId:  b.DomainId,
		Timestamp: b.Timestamp,
		Events:    []Event{},
	}
}

func (b Batch) Key() string {
	return b.DomainId + ":" + b.Timestamp
}

func (batch Batch) Filtered(sub Sub) (Batch, bool) {
	out := batch.BlankClone()

	// Create a hash lookup of valid events.
	eventMap := map[string]bool{}
	for _, e := range sub.Events {
		eventMap[e] = true
	}

	// Loops through the supplied batch and add valid events.
	for _, event := range batch.Events {
		_, match := eventMap[event.Type]
		if match {
			out.Events = append(out.Events, event)
		}
	}
	return out, len(out.Events) > 0
}

// BatchFinder is responsible for returning batches ready to process.
type BatchFinder interface {
	ReadyBatchKeys() ([]Batch, error)
	Batch(key string) (Batch, error)
}

func newBatchFinder() RedisBatchFinder {
	return RedisBatchFinder{expiry: time.Minute}
}

type RedisBatchFinder struct {
	expiry time.Duration
}

// ReadyBatches returns an batches over a minute old from Redis.
func (f RedisBatchFinder) ReadyBatchKeys() ([]Batch, error) {
	batches := []Batch{}
	members, err := Redis.SMembers("webhooks:batches:current").Result()
	if err != nil {
		return nil, err
	}

	for _, key := range members {
		batch, err := f.Batch(key)
		if err != nil {
			return batches, err
		}
		// Skip if the time of this batch plus 1 min, is now or later
		if time.Now().Before(batch.StartedAt.Add(f.expiry)) {
			continue
		}

		batches = append(batches, batch)
	}
	return batches, nil
}

func (f RedisBatchFinder) Batch(key string) (Batch, error) {
	batch := Batch{}
	parts := strings.Split(key, ":")
	if len(parts) != 2 {
		return batch, fmt.Errorf("Invalid batch key format: %s", key)
	}

	timestamp, err := strconv.ParseInt(parts[1], 0, 0)
	if err != nil {
		return batch, fmt.Errorf(
			"Could not convert timestamp '%s' to int: %s",
			parts[1],
			err.Error(),
		)
	}

	// Get events for this batch
	rawEvents, err := Redis.LRange(
		fmt.Sprintf("webhooks:batches:%s:%s", parts[0], parts[1]),
		0,
		-1,
	).Result()
	if err != nil {
		return batch, err
	}

	batch.Events = []Event{}
	for _, rawEvent := range rawEvents {
		event := Event{}
		err := json.Unmarshal([]byte(rawEvent), &event)
		if err != nil {
			logger.Errorf(
				"Could not unmarshal event data: %s\nData: %s",
				err.Error(),
				string(rawEvent),
			)
			continue
		}

		batch.Events = append(batch.Events, event)
	}

	if len(batch.Events) == 0 && len(rawEvents) > 0 {
		return batch, fmt.Errorf("Batch contains no events")
	}

	batch.StartedAt = time.Unix(timestamp, 0)
	batch.DomainId = parts[0]
	batch.Timestamp = parts[1]
	return batch, nil
}

// BatchProcessor is responsible for the processing of batches to their subs.
type BatchProcessor interface {
	ProcessBatch(Batch) error
}

func newBatchProcessor() RedisBatchProcessor {
	return RedisBatchProcessor{}
}

type RedisBatchProcessor struct {
}

// ProcessBatch gets any subs from redis, and sends the batch to the sub.
func (p RedisBatchProcessor) ProcessBatch(batch Batch) error {

	subs, err := subFinder.Subs(batch.DomainId)
	if err != nil {
		return err
	}

	subsLen := len(subs)
	if subsLen == 0 {
		logger.Info("Found no subscriptions for ", batch.Key())
	} else {
		logger.Infof("Found %d subscriptions for %s", subsLen, batch.Key())
	}

	tx := Redis.Multi()
	defer tx.Close()

	for _, sub := range subs {
		err := p.PersistBatch(tx, batch, sub)
		if err != nil {
			logger.Errorf("Error processing batch: %s", err.Error())
			err = tx.Discard()
			return err
		}
	}

	_, err = tx.Del(fmt.Sprintf("webhooks:batches:%s", batch.Key())).Result()
	if err != nil {
		logger.Errorf("Error removing batch events: %s", err.Error())
		err = tx.Discard()
		return err
	}

	_, err = tx.SRem("webhooks:batches:current", batch.Key()).Result()
	if err != nil {
		logger.Errorf("Error removing batch from current: %s", err.Error())
		err = tx.Discard()
		return err
	}

	_, err = tx.Exec(func() error { return nil })
	if err != nil {
		return err
	}
	return nil
}

func (p RedisBatchProcessor) PersistBatch(tx *redis.Multi, batch Batch, sub Sub) error {
	// New batch with events filtered for this sub
	filtered, send := batch.Filtered(sub)
	if !send {
		logger.Infof(
			"Not sending batch to %s, no of the %d events are valid",
			sub.URL,
			len(batch.Events),
		)
		return nil
	}

	payload, err := json.Marshal(filtered)
	if err != nil {
		return err
	}

	rootKey := fmt.Sprintf("webhooks:sends:%s", batch.DomainId)
	sendKey := batch.Timestamp + ":" + sub.URL

	_, err = tx.HSet(rootKey+":batch", sendKey, string(payload)).Result()
	if err != nil {
		return err
	}

	lenStr := fmt.Sprintf("%d", len(filtered.Events))
	_, err = tx.HSet(rootKey+":count", sendKey, lenStr).Result()
	if err != nil {
		return err
	}

	_, err = tx.SAdd("webhooks:sends:ready", batch.Key()+":"+sub.URL).Result()
	if err != nil {
		return err
	}
	logger.Infof("Added send with %d events", len(filtered.Events))

	return nil
}

// BatchSender is responsible for sending a batch to a sub.
type BatchSender interface {
	Send(Batch, Sub)
	Retry(Batch, Sub)
	Fail(Batch, Sub, error)
}

func newBatchSender() HttpBatchSender {
	return HttpBatchSender{}
}

type HttpBatchSender struct{}

// Send will submit the batch to the given sub according to the sub configuration.
// Anything which is considered an error at the client end shall return false, nil
// Anything which is an error at the server end shall return true, err
func (s HttpBatchSender) Send(batch Batch, sub Sub) {
	logger.Infof("Sending batch with %d events to %s", len(batch.Events), sub.URL)

	key := fmt.Sprintf("%s:%s", batch.Timestamp, sub.URL)
	rootKey := fmt.Sprintf("webhooks:batches:%s", batch.DomainId)

	// Set the state to sending
	_, err := Redis.HSet(
		fmt.Sprintf("%s:status", rootKey),
		key,
		"sending",
	).Result()
	if err != nil {
		s.Fail(batch, sub, err)
		return
	}

	body, err := json.Marshal(batch.Events)
	if err != nil {
		s.Fail(batch, sub, err)
		return
	}

	// Create a new post request to the sub url, with the event payload.
	req, err := http.NewRequest("POST", sub.URL, strings.NewReader(string(body)))
	if err != nil {
		s.Fail(batch, sub, err)
		return
	}

	// Add headers
	for key, value := range sub.Headers {
		req.Header.Add(key, value)
	}

	resp, err := client.Do(req)
	if err != nil {
		s.Fail(batch, sub, err)
		return
	}

	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		logger.Noticef("Expected 2xx response, received %d", resp.StatusCode)
		s.Retry(batch, sub)
		return
	}

	// Set the state to sent
	_, err = Redis.HSet(
		fmt.Sprintf("%s:status", rootKey),
		key,
		"sent",
	).Result()
	if err != nil {
		s.Fail(batch, sub, err)
		return
	}

	logger.Infof("Successfully sent %s to %s", batch.Key(), sub.URL)
}

func (s HttpBatchSender) Fail(batch Batch, sub Sub, err error) {
	logger.Errorf("Failed to correctly %s:%s: %s", batch.Key(), sub.URL, err.Error())
	logger.Error("Message in incorrect state")
	// TODO(revert to previous state)
}

// Fail is responsible for placing the batch sub combo into failed state
// and setting retry count, time etc.
func (s HttpBatchSender) Retry(batch Batch, sub Sub) {
	logger.Noticef("Failed to send %s to %s ", batch.Key(), sub.URL)

	key := fmt.Sprintf("%s:%s", batch.Timestamp, sub.URL)
	rootKey := fmt.Sprintf("webhooks:batches:%s", batch.DomainId)

	// Set the state to failed
	_, err := Redis.HSet(
		fmt.Sprintf("%s:status", rootKey),
		key,
		"resend",
	).Result()
	if err != nil {
		logger.Error(err.Error())
		return
	}

	// Update the retry count
	count, err := Redis.Incr(
		fmt.Sprintf("%s:%s:retries", rootKey, sub.URL),
	).Result()
	if err != nil {
		logger.Error(err.Error())
		return
	}

	duration := backoffDuration(int(count))
	logger.Noticef("Retrying in %s", duration)

	// Set the next send time
	resendTime := time.Now().Add(duration)
	z := redis.Z{float64(resendTime.Unix()), batch.DomainId + ":" + key}
	_, err = Redis.ZAdd("webhooks:sends:retry", z).Result()
	if err != nil {
		logger.Error(err.Error())
		return
	}

}

func backoffDuration(retryCount int) time.Duration {
	seconds := math.Pow(float64(retryCount), 4) + 15
	return time.Second * time.Duration(seconds)
}
<commit_msg>Add marshalling for batch<commit_after>package whookie

import (
	"encoding/json"
	"fmt"
	"math"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/go-redis/redis"
)

var (
	client *http.Client
)

func init() {
	client = &http.Client{}
}

// Batch represents a group of events that occurred in a given timeperiod for a domain
type Batch struct {
	DomainId  string    `json:"domain_id"`
	Timestamp string    `json:"-"`
	StartedAt time.Time `json:"-"`
	Events    []Event   `json:"events"`
}

func (b Batch) BlankClone() Batch {
	return Batch{
		DomainId:  b.DomainId,
		Timestamp: b.Timestamp,
		Events:    []Event{},
	}
}

func (b Batch) Key() string {
	return b.DomainId + ":" + b.Timestamp
}

func (batch Batch) Filtered(sub Sub) (Batch, bool) {
	out := batch.BlankClone()

	// Create a hash lookup of valid events.
	eventMap := map[string]bool{}
	for _, e := range sub.Events {
		eventMap[e] = true
	}

	// Loops through the supplied batch and add valid events.
	for _, event := range batch.Events {
		_, match := eventMap[event.Type]
		if match {
			out.Events = append(out.Events, event)
		}
	}
	return out, len(out.Events) > 0
}

// BatchFinder is responsible for returning batches ready to process.
type BatchFinder interface {
	ReadyBatchKeys() ([]Batch, error)
	Batch(key string) (Batch, error)
}

func newBatchFinder() RedisBatchFinder {
	return RedisBatchFinder{expiry: time.Minute}
}

type RedisBatchFinder struct {
	expiry time.Duration
}

// ReadyBatches returns an batches over a minute old from Redis.
func (f RedisBatchFinder) ReadyBatchKeys() ([]Batch, error) {
	batches := []Batch{}
	members, err := Redis.SMembers("webhooks:batches:current").Result()
	if err != nil {
		return nil, err
	}

	for _, key := range members {
		batch, err := f.Batch(key)
		if err != nil {
			return batches, err
		}
		// Skip if the time of this batch plus 1 min, is now or later
		if time.Now().Before(batch.StartedAt.Add(f.expiry)) {
			continue
		}

		batches = append(batches, batch)
	}
	return batches, nil
}

func (f RedisBatchFinder) Batch(key string) (Batch, error) {
	batch := Batch{}
	parts := strings.Split(key, ":")
	if len(parts) != 2 {
		return batch, fmt.Errorf("Invalid batch key format: %s", key)
	}

	timestamp, err := strconv.ParseInt(parts[1], 0, 0)
	if err != nil {
		return batch, fmt.Errorf(
			"Could not convert timestamp '%s' to int: %s",
			parts[1],
			err.Error(),
		)
	}

	// Get events for this batch
	rawEvents, err := Redis.LRange(
		fmt.Sprintf("webhooks:batches:%s:%s", parts[0], parts[1]),
		0,
		-1,
	).Result()
	if err != nil {
		return batch, err
	}

	batch.Events = []Event{}
	for _, rawEvent := range rawEvents {
		event := Event{}
		err := json.Unmarshal([]byte(rawEvent), &event)
		if err != nil {
			logger.Errorf(
				"Could not unmarshal event data: %s\nData: %s",
				err.Error(),
				string(rawEvent),
			)
			continue
		}

		batch.Events = append(batch.Events, event)
	}

	if len(batch.Events) == 0 && len(rawEvents) > 0 {
		return batch, fmt.Errorf("Batch contains no events")
	}

	batch.StartedAt = time.Unix(timestamp, 0)
	batch.DomainId = parts[0]
	batch.Timestamp = parts[1]
	return batch, nil
}

// BatchProcessor is responsible for the processing of batches to their subs.
type BatchProcessor interface {
	ProcessBatch(Batch) error
}

func newBatchProcessor() RedisBatchProcessor {
	return RedisBatchProcessor{}
}

type RedisBatchProcessor struct {
}

// ProcessBatch gets any subs from redis, and sends the batch to the sub.
func (p RedisBatchProcessor) ProcessBatch(batch Batch) error {

	subs, err := subFinder.Subs(batch.DomainId)
	if err != nil {
		return err
	}

	subsLen := len(subs)
	if subsLen == 0 {
		logger.Info("Found no subscriptions for ", batch.Key())
	} else {
		logger.Infof("Found %d subscriptions for %s", subsLen, batch.Key())
	}

	tx := Redis.Multi()
	defer tx.Close()

	for _, sub := range subs {
		err := p.PersistBatch(tx, batch, sub)
		if err != nil {
			logger.Errorf("Error processing batch: %s", err.Error())
			err = tx.Discard()
			return err
		}
	}

	_, err = tx.Del(fmt.Sprintf("webhooks:batches:%s", batch.Key())).Result()
	if err != nil {
		logger.Errorf("Error removing batch events: %s", err.Error())
		err = tx.Discard()
		return err
	}

	_, err = tx.SRem("webhooks:batches:current", batch.Key()).Result()
	if err != nil {
		logger.Errorf("Error removing batch from current: %s", err.Error())
		err = tx.Discard()
		return err
	}

	_, err = tx.Exec(func() error { return nil })
	if err != nil {
		return err
	}
	return nil
}

func (p RedisBatchProcessor) PersistBatch(tx *redis.Multi, batch Batch, sub Sub) error {
	// New batch with events filtered for this sub
	filtered, send := batch.Filtered(sub)
	if !send {
		logger.Infof(
			"Not sending batch to %s, no of the %d events are valid",
			sub.URL,
			len(batch.Events),
		)
		return nil
	}

	payload, err := json.Marshal(filtered)
	if err != nil {
		return err
	}

	rootKey := fmt.Sprintf("webhooks:sends:%s", batch.DomainId)
	sendKey := batch.Timestamp + ":" + sub.URL

	_, err = tx.HSet(rootKey+":batch", sendKey, string(payload)).Result()
	if err != nil {
		return err
	}

	lenStr := fmt.Sprintf("%d", len(filtered.Events))
	_, err = tx.HSet(rootKey+":count", sendKey, lenStr).Result()
	if err != nil {
		return err
	}

	_, err = tx.SAdd("webhooks:sends:ready", batch.Key()+":"+sub.URL).Result()
	if err != nil {
		return err
	}
	logger.Infof("Added send with %d events", len(filtered.Events))

	return nil
}

// BatchSender is responsible for sending a batch to a sub.
type BatchSender interface {
	Send(Batch, Sub)
	Retry(Batch, Sub)
	Fail(Batch, Sub, error)
}

func newBatchSender() HttpBatchSender {
	return HttpBatchSender{}
}

type HttpBatchSender struct{}

// Send will submit the batch to the given sub according to the sub configuration.
// Anything which is considered an error at the client end shall return false, nil
// Anything which is an error at the server end shall return true, err
func (s HttpBatchSender) Send(batch Batch, sub Sub) {
	logger.Infof("Sending batch with %d events to %s", len(batch.Events), sub.URL)

	key := fmt.Sprintf("%s:%s", batch.Timestamp, sub.URL)
	rootKey := fmt.Sprintf("webhooks:batches:%s", batch.DomainId)

	// Set the state to sending
	_, err := Redis.HSet(
		fmt.Sprintf("%s:status", rootKey),
		key,
		"sending",
	).Result()
	if err != nil {
		s.Fail(batch, sub, err)
		return
	}

	body, err := json.Marshal(batch.Events)
	if err != nil {
		s.Fail(batch, sub, err)
		return
	}

	// Create a new post request to the sub url, with the event payload.
	req, err := http.NewRequest("POST", sub.URL, strings.NewReader(string(body)))
	if err != nil {
		s.Fail(batch, sub, err)
		return
	}

	// Add headers
	for key, value := range sub.Headers {
		req.Header.Add(key, value)
	}

	resp, err := client.Do(req)
	if err != nil {
		s.Fail(batch, sub, err)
		return
	}

	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		logger.Noticef("Expected 2xx response, received %d", resp.StatusCode)
		s.Retry(batch, sub)
		return
	}

	// Set the state to sent
	_, err = Redis.HSet(
		fmt.Sprintf("%s:status", rootKey),
		key,
		"sent",
	).Result()
	if err != nil {
		s.Fail(batch, sub, err)
		return
	}

	logger.Infof("Successfully sent %s to %s", batch.Key(), sub.URL)
}

func (s HttpBatchSender) Fail(batch Batch, sub Sub, err error) {
	logger.Errorf("Failed to correctly %s:%s: %s", batch.Key(), sub.URL, err.Error())
	logger.Error("Message in incorrect state")
	// TODO(revert to previous state)
}

// Fail is responsible for placing the batch sub combo into failed state
// and setting retry count, time etc.
func (s HttpBatchSender) Retry(batch Batch, sub Sub) {
	logger.Noticef("Failed to send %s to %s ", batch.Key(), sub.URL)

	key := fmt.Sprintf("%s:%s", batch.Timestamp, sub.URL)
	rootKey := fmt.Sprintf("webhooks:batches:%s", batch.DomainId)

	// Set the state to failed
	_, err := Redis.HSet(
		fmt.Sprintf("%s:status", rootKey),
		key,
		"resend",
	).Result()
	if err != nil {
		logger.Error(err.Error())
		return
	}

	// Update the retry count
	count, err := Redis.Incr(
		fmt.Sprintf("%s:%s:retries", rootKey, sub.URL),
	).Result()
	if err != nil {
		logger.Error(err.Error())
		return
	}

	duration := backoffDuration(int(count))
	logger.Noticef("Retrying in %s", duration)

	// Set the next send time
	resendTime := time.Now().Add(duration)
	z := redis.Z{float64(resendTime.Unix()), batch.DomainId + ":" + key}
	_, err = Redis.ZAdd("webhooks:sends:retry", z).Result()
	if err != nil {
		logger.Error(err.Error())
		return
	}

}

func backoffDuration(retryCount int) time.Duration {
	seconds := math.Pow(float64(retryCount), 4) + 15
	return time.Second * time.Duration(seconds)
}
<|endoftext|>