<commit_before>// Copyright 2019 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package client_test

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io/ioutil"
	"math/rand"
	"sync"
	"time"

	"testing"
	"text/template"

	"istio.io/proxy/test/envoye2e/driver"
	"istio.io/proxy/test/envoye2e/env"
)

const metadataExchangeIstioConfigFilter = `
- name: envoy.filters.network.metadata_exchange
  config:
    protocol: istio2
- name: envoy.filters.network.wasm
  config:
    config:
      root_id: "stats_inbound"
      vm_config:
        runtime: envoy.wasm.runtime.null
        code:
          local: { inline_string: "envoy.wasm.stats" }
      configuration: |
        { "debug": "false", max_peer_cache_size: 20, field_separator: ";.;", tcp_reporting_duration: "0.00000001s" }
`

var (
	statsConfig        = driver.LoadTestData("testdata/bootstrap/stats.yaml.tmpl")
	clientNodeMetadata = driver.LoadTestData("testdata/client_node_metadata.json.tmpl")
	serverNodeMetadata = driver.LoadTestData("testdata/server_node_metadata.json.tmpl")
)

const metadataExchangeIstioUpstreamConfigFilterChain = `
filters:
- name: envoy.filters.network.upstream.metadata_exchange
  typed_config:
    "@type": type.googleapis.com/envoy.tcp.metadataexchange.config.MetadataExchange
    protocol: istio2
`

const metadataExchangeIstioClientFilter = `
- name: envoy.filters.network.wasm
  config:
    config:
      root_id: "stats_outbound"
      vm_config:
        runtime: envoy.wasm.runtime.null
        code:
          local: { inline_string: "envoy.wasm.stats" }
      configuration: |
        { "debug": "false", max_peer_cache_size: 20, field_separator: ";.;", tcp_reporting_duration: "0.00000001s" }
`

const tlsContext = `
tls_context:
  common_tls_context:
    alpn_protocols:
    - istio2
    tls_certificates:
    - certificate_chain: { filename: "testdata/certs/cert-chain.pem" }
      private_key: { filename: "testdata/certs/key.pem" }
    validation_context:
      trusted_ca: { filename: "testdata/certs/root-cert.pem" }
  require_client_certificate: true
`

const clusterTLSContext = `
tls_context:
  common_tls_context:
    alpn_protocols:
    - istio2
    tls_certificates:
    - certificate_chain: { filename: "testdata/certs/cert-chain.pem" }
      private_key: { filename: "testdata/certs/key.pem" }
    validation_context:
      trusted_ca: { filename: "testdata/certs/root-cert.pem" }
`

const serverTLSContext = `
tls_context:
  common_tls_context:
    alpn_protocols:
    - istio3
    tls_certificates:
    - certificate_chain: { filename: "testdata/certs/cert-chain.pem" }
      private_key: { filename: "testdata/certs/key.pem" }
    validation_context:
      trusted_ca: { filename: "testdata/certs/root-cert.pem" }
  require_client_certificate: true
`

const serverClusterTLSContext = `
tls_context:
  common_tls_context:
    alpn_protocols:
    - istio3
    tls_certificates:
    - certificate_chain: { filename: "testdata/certs/cert-chain.pem" }
      private_key: { filename: "testdata/certs/key.pem" }
    validation_context:
      trusted_ca: { filename: "testdata/certs/root-cert.pem" }
`

// Stats in Client Envoy proxy.
var expectedClientStats = map[string]int{
	"cluster.client.metadata_exchange.alpn_protocol_found":      5,
	"cluster.client.metadata_exchange.alpn_protocol_not_found":  0,
	"cluster.client.metadata_exchange.initial_header_not_found": 0,
	"cluster.client.metadata_exchange.header_not_found":         0,
	"cluster.client.metadata_exchange.metadata_added":           5,
}

// Stats in Server Envoy proxy.
var expectedPrometheusServerLabels = map[string]string{
	"reporter":        "destination",
	"source_app":      "productpage",
	"destination_app": "ratings",
}
var expectedPrometheusServerStats = map[string]env.Stat{
	"istio_tcp_connections_opened_total": {Value: 5, Labels: expectedPrometheusServerLabels},
	"istio_tcp_connections_closed_total": {Value: 5, Labels: expectedPrometheusServerLabels},
	"istio_tcp_received_bytes_total":     {Value: 35, Labels: expectedPrometheusServerLabels},
	"istio_tcp_sent_bytes_total":         {Value: 65, Labels: expectedPrometheusServerLabels},
}

// Stats in Server Envoy proxy Fail Case.
var expectedPrometheusServerLabelsFailCase = map[string]string{
	"reporter":        "destination",
	"source_app":      "unknown",
	"destination_app": "ratings",
}
var expectedPrometheusServerStatsFailCase = map[string]env.Stat{
	"istio_tcp_connections_opened_total": {Value: 5, Labels: expectedPrometheusServerLabelsFailCase},
	"istio_tcp_connections_closed_total": {Value: 5, Labels: expectedPrometheusServerLabelsFailCase},
	"istio_tcp_received_bytes_total":     {Value: 35, Labels: expectedPrometheusServerLabelsFailCase},
	"istio_tcp_sent_bytes_total":         {Value: 65, Labels: expectedPrometheusServerLabelsFailCase},
}

// Stats in Client Envoy proxy.
var expectedPrometheusClientLabels = map[string]string{
	"reporter":        "source",
	"source_app":      "productpage",
	"destination_app": "ratings",
}
var expectedPrometheusClientStats = map[string]env.Stat{
	"istio_tcp_connections_opened_total": {Value: 5, Labels: expectedPrometheusClientLabels},
	"istio_tcp_connections_closed_total": {Value: 5, Labels: expectedPrometheusClientLabels},
	"istio_tcp_received_bytes_total":     {Value: 35, Labels: expectedPrometheusClientLabels},
	"istio_tcp_sent_bytes_total":         {Value: 65, Labels: expectedPrometheusClientLabels},
}

// Stats in Server Envoy proxy.
var expectedServerStats = map[string]int{
	"metadata_exchange.alpn_protocol_found":      5,
	"metadata_exchange.alpn_protocol_not_found":  0,
	"metadata_exchange.initial_header_not_found": 0,
	"metadata_exchange.header_not_found":         0,
	"metadata_exchange.metadata_added":           5,
}

var expectedServerStatsFailCase = map[string]int{
	"metadata_exchange.alpn_protocol_found":      0,
	"metadata_exchange.alpn_protocol_not_found":  5,
	"metadata_exchange.initial_header_not_found": 0,
	"metadata_exchange.header_not_found":         0,
	"metadata_exchange.metadata_added":           0,
}

func TestTCPMetadataExchange(t *testing.T) {
	s := env.NewClientServerEnvoyTestSetup(env.TCPMetadataExchangeTest, t)
	s.Dir = driver.BazelWorkspace()
	s.SetStartHTTPBackend(false)
	s.SetStartTCPBackend(true)
	s.SetTLSContext(tlsContext)
	s.SetClusterTLSContext(clusterTLSContext)
	s.SetServerTLSContext(tlsContext)
	s.SetServerClusterTLSContext(clusterTLSContext)
	s.SetFiltersBeforeEnvoyRouterInProxyToServer(metadataExchangeIstioConfigFilter)
	s.SetUpstreamFiltersInClient(metadataExchangeIstioUpstreamConfigFilterChain)
	s.SetFiltersBeforeEnvoyRouterInAppToClient(metadataExchangeIstioClientFilter)
	s.SetEnableTLS(true)
	s.SetClientNodeMetadata(clientNodeMetadata)
	s.SetServerNodeMetadata(serverNodeMetadata)
	s.SetExtraConfig(statsConfig)
	s.ClientEnvoyTemplate = env.GetTCPClientEnvoyConfTmp()
	s.ServerEnvoyTemplate = env.GetTCPServerEnvoyConfTmp()
	s.SetCopyYamlFiles(true)
	if err := s.SetUpClientServerEnvoy(); err != nil {
		t.Fatalf("Failed to setup te1	st: %v", err)
	}
	defer s.TearDownClientServerEnvoy()

	SendRequests(t, s)

	s.VerifyEnvoyStats(getParsedExpectedStats(expectedClientStats, t, s), s.Ports().ClientAdminPort)
	s.VerifyEnvoyStats(getParsedExpectedStats(expectedServerStats, t, s), s.Ports().ServerAdminPort)

	time.Sleep(time.Second * 5)
	s.VerifyPrometheusStats(expectedPrometheusServerStats, s.Ports().ServerAdminPort)
	s.VerifyPrometheusStats(expectedPrometheusClientStats, s.Ports().ClientAdminPort)
}

func TestTCPMetadataExchangeNoClientFilter(t *testing.T) {
	s := env.NewClientServerEnvoyTestSetup(env.TCPMetadataExchangeFailTest, t)
	s.Dir = driver.BazelWorkspace()
	s.SetStartHTTPBackend(false)
	s.SetStartTCPBackend(true)
	// Client send istio2  alpn in tls context.
	s.SetTLSContext(tlsContext)
	s.SetClusterTLSContext(clusterTLSContext)
	// Server accepts istio3 alpn in tls context.
	s.SetServerTLSContext(serverTLSContext)
	s.SetServerClusterTLSContext(serverClusterTLSContext)
	// Only setting mxc filter in server and stats filter in client and filter.
	// Mxc  upstream filter in  server is not set.
	s.SetFiltersBeforeEnvoyRouterInProxyToServer(metadataExchangeIstioConfigFilter)
	s.SetFiltersBeforeEnvoyRouterInAppToClient(metadataExchangeIstioClientFilter)
	s.SetEnableTLS(true)
	s.SetClientNodeMetadata(clientNodeMetadata)
	s.SetServerNodeMetadata(serverNodeMetadata)
	s.SetExtraConfig(statsConfig)
	s.ClientEnvoyTemplate = env.GetTCPClientEnvoyConfTmp()
	s.ServerEnvoyTemplate = env.GetTCPServerEnvoyConfTmp()
	if err := s.SetUpClientServerEnvoy(); err != nil {
		t.Fatalf("Failed to setup te1	st: %v", err)
	}
	defer s.TearDownClientServerEnvoy()

	SendRequests(t, s)
	s.VerifyEnvoyStats(getParsedExpectedStats(expectedServerStatsFailCase, t, s), s.Ports().ServerAdminPort)

	time.Sleep(time.Second * 5)
	s.VerifyPrometheusStats(expectedPrometheusServerStatsFailCase, s.Ports().ServerAdminPort)
}

func SendRequests(t *testing.T, s *env.TestSetup) {
	certPool := x509.NewCertPool()
	bs, err := ioutil.ReadFile(driver.TestPath("testdata/certs/cert-chain.pem"))
	if err != nil {
		t.Fatalf("failed to read client ca cert: %s", err)
	}
	ok := certPool.AppendCertsFromPEM(bs)
	if !ok {
		t.Fatal("failed to append client certs")
	}

	certificate, err := tls.LoadX509KeyPair(driver.TestPath("testdata/certs/cert-chain.pem"),
		driver.TestPath("testdata/certs/key.pem"))
	if err != nil {
		t.Fatal("failed to get certificate")
	}
	config := &tls.Config{Certificates: []tls.Certificate{certificate}, ServerName: "localhost", NextProtos: []string{"istio3"}, RootCAs: certPool}
	rand.Seed(time.Now().UTC().UnixNano())

	var wg sync.WaitGroup

	response := make(chan error, 5)
	wg.Add(5)
	go sendRequest(response, config, s.Ports().AppToClientProxyPort, &wg)
	go sendRequest(response, config, s.Ports().AppToClientProxyPort, &wg)
	go sendRequest(response, config, s.Ports().AppToClientProxyPort, &wg)
	go sendRequest(response, config, s.Ports().AppToClientProxyPort, &wg)
	go sendRequest(response, config, s.Ports().AppToClientProxyPort, &wg)
	wg.Wait()
	close(response)

	for err := range response {
		if err != nil {
			t.Fatal(err)
		}
	}
}

func sendRequest(response chan<- error, config *tls.Config, port uint16, wg *sync.WaitGroup) {
	defer wg.Done()
	conn, err := tls.Dial("tcp", fmt.Sprintf("localhost:%d", port), config)
	if err != nil {
		response <- err
	}

	conn.Write([]byte("world \n"))
	reply := make([]byte, 256)
	n, err := conn.Read(reply)
	if err != nil {
		response <- err
	}

	if fmt.Sprintf("%s", reply[:n]) != "hello world \n" {
		response <- fmt.Errorf("verification Failed. Expected: hello world. Got: %v", fmt.Sprintf("%s", reply[:n]))
	}

	_ = conn.Close()

	response <- nil
}

func getParsedExpectedStats(expectedStats map[string]int, t *testing.T, s *env.TestSetup) map[string]int {
	parsedExpectedStats := make(map[string]int)
	for key, value := range expectedStats {
		tmpl, err := template.New("parse_state").Parse(key)
		if err != nil {
			t.Errorf("failed to parse config template: %v", err)
		}

		var tpl bytes.Buffer
		err = tmpl.Execute(&tpl, s)
		if err != nil {
			t.Errorf("failed to execute config template: %v", err)
		}
		parsedExpectedStats[tpl.String()] = value
	}

	return parsedExpectedStats
}
<commit_msg>Add a test for TCP metrics for a long running TCP connection (#2753)<commit_after>// Copyright 2019 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package client_test

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io/ioutil"
	"math/rand"
	"sync"
	"time"

	"testing"
	"text/template"

	"istio.io/proxy/test/envoye2e/driver"
	"istio.io/proxy/test/envoye2e/env"
)

const metadataExchangeIstioConfigFilter = `
- name: envoy.filters.network.metadata_exchange
  config:
    protocol: istio2
- name: envoy.filters.network.wasm
  config:
    config:
      root_id: "stats_inbound"
      vm_config:
        runtime: envoy.wasm.runtime.null
        code:
          local: { inline_string: "envoy.wasm.stats" }
      configuration: |
        { "debug": "false", max_peer_cache_size: 20, field_separator: ";.;", tcp_reporting_duration: "1s" }
`

var (
	statsConfig        = driver.LoadTestData("testdata/bootstrap/stats.yaml.tmpl")
	clientNodeMetadata = driver.LoadTestData("testdata/client_node_metadata.json.tmpl")
	serverNodeMetadata = driver.LoadTestData("testdata/server_node_metadata.json.tmpl")
)

const metadataExchangeIstioUpstreamConfigFilterChain = `
filters:
- name: envoy.filters.network.upstream.metadata_exchange
  typed_config:
    "@type": type.googleapis.com/envoy.tcp.metadataexchange.config.MetadataExchange
    protocol: istio2
`

const metadataExchangeIstioClientFilter = `
- name: envoy.filters.network.wasm
  config:
    config:
      root_id: "stats_outbound"
      vm_config:
        runtime: envoy.wasm.runtime.null
        code:
          local: { inline_string: "envoy.wasm.stats" }
      configuration: |
        { "debug": "false", max_peer_cache_size: 20, field_separator: ";.;", tcp_reporting_duration: "1s" }
`

const tlsContext = `
tls_context:
  common_tls_context:
    alpn_protocols:
    - istio2
    tls_certificates:
    - certificate_chain: { filename: "testdata/certs/cert-chain.pem" }
      private_key: { filename: "testdata/certs/key.pem" }
    validation_context:
      trusted_ca: { filename: "testdata/certs/root-cert.pem" }
  require_client_certificate: true
`

const clusterTLSContext = `
tls_context:
  common_tls_context:
    alpn_protocols:
    - istio2
    tls_certificates:
    - certificate_chain: { filename: "testdata/certs/cert-chain.pem" }
      private_key: { filename: "testdata/certs/key.pem" }
    validation_context:
      trusted_ca: { filename: "testdata/certs/root-cert.pem" }
`

const serverTLSContext = `
tls_context:
  common_tls_context:
    alpn_protocols:
    - istio3
    tls_certificates:
    - certificate_chain: { filename: "testdata/certs/cert-chain.pem" }
      private_key: { filename: "testdata/certs/key.pem" }
    validation_context:
      trusted_ca: { filename: "testdata/certs/root-cert.pem" }
  require_client_certificate: true
`

const serverClusterTLSContext = `
tls_context:
  common_tls_context:
    alpn_protocols:
    - istio3
    tls_certificates:
    - certificate_chain: { filename: "testdata/certs/cert-chain.pem" }
      private_key: { filename: "testdata/certs/key.pem" }
    validation_context:
      trusted_ca: { filename: "testdata/certs/root-cert.pem" }
`

// Stats in Client Envoy proxy.
var expectedClientStats = map[string]int{
	"cluster.client.metadata_exchange.alpn_protocol_found":      5,
	"cluster.client.metadata_exchange.alpn_protocol_not_found":  0,
	"cluster.client.metadata_exchange.initial_header_not_found": 0,
	"cluster.client.metadata_exchange.header_not_found":         0,
	"cluster.client.metadata_exchange.metadata_added":           5,
}

// Stats in Server Envoy proxy.
var expectedPrometheusServerLabels = map[string]string{
	"reporter":        "destination",
	"source_app":      "productpage",
	"destination_app": "ratings",
}
var expectedPrometheusServerStats = map[string]env.Stat{
	"istio_tcp_connections_opened_total": {Value: 5, Labels: expectedPrometheusServerLabels},
	"istio_tcp_connections_closed_total": {Value: 5, Labels: expectedPrometheusServerLabels},
	"istio_tcp_received_bytes_total":     {Value: 35, Labels: expectedPrometheusServerLabels},
	"istio_tcp_sent_bytes_total":         {Value: 65, Labels: expectedPrometheusServerLabels},
}

// Stats in Server Envoy proxy Fail Case.
var expectedPrometheusServerLabelsFailCase = map[string]string{
	"reporter":        "destination",
	"source_app":      "unknown",
	"destination_app": "ratings",
}
var expectedPrometheusServerStatsFailCase = map[string]env.Stat{
	"istio_tcp_connections_opened_total": {Value: 5, Labels: expectedPrometheusServerLabelsFailCase},
	"istio_tcp_connections_closed_total": {Value: 5, Labels: expectedPrometheusServerLabelsFailCase},
	"istio_tcp_received_bytes_total":     {Value: 35, Labels: expectedPrometheusServerLabelsFailCase},
	"istio_tcp_sent_bytes_total":         {Value: 65, Labels: expectedPrometheusServerLabelsFailCase},
}

// Stats in Client Envoy proxy.
var expectedPrometheusClientLabels = map[string]string{
	"reporter":        "source",
	"source_app":      "productpage",
	"destination_app": "ratings",
}
var expectedPrometheusClientStats = map[string]env.Stat{
	"istio_tcp_connections_opened_total": {Value: 5, Labels: expectedPrometheusClientLabels},
	"istio_tcp_connections_closed_total": {Value: 5, Labels: expectedPrometheusClientLabels},
	"istio_tcp_received_bytes_total":     {Value: 35, Labels: expectedPrometheusClientLabels},
	"istio_tcp_sent_bytes_total":         {Value: 65, Labels: expectedPrometheusClientLabels},
}

// Stats in Server Envoy proxy.
var expectedServerStats = map[string]int{
	"metadata_exchange.alpn_protocol_found":      5,
	"metadata_exchange.alpn_protocol_not_found":  0,
	"metadata_exchange.initial_header_not_found": 0,
	"metadata_exchange.header_not_found":         0,
	"metadata_exchange.metadata_added":           5,
}

var expectedServerStatsFailCase = map[string]int{
	"metadata_exchange.alpn_protocol_found":      0,
	"metadata_exchange.alpn_protocol_not_found":  5,
	"metadata_exchange.initial_header_not_found": 0,
	"metadata_exchange.header_not_found":         0,
	"metadata_exchange.metadata_added":           0,
}

func setupMXC(t *testing.T) *env.TestSetup {
	s := env.NewClientServerEnvoyTestSetup(env.TCPMetadataExchangeTest, t)
	s.Dir = driver.BazelWorkspace()
	s.SetStartHTTPBackend(false)
	s.SetStartTCPBackend(true)
	s.SetTLSContext(tlsContext)
	s.SetClusterTLSContext(clusterTLSContext)
	s.SetServerTLSContext(tlsContext)
	s.SetServerClusterTLSContext(clusterTLSContext)
	s.SetFiltersBeforeEnvoyRouterInProxyToServer(metadataExchangeIstioConfigFilter)
	s.SetUpstreamFiltersInClient(metadataExchangeIstioUpstreamConfigFilterChain)
	s.SetFiltersBeforeEnvoyRouterInAppToClient(metadataExchangeIstioClientFilter)
	s.SetEnableTLS(true)
	s.SetClientNodeMetadata(clientNodeMetadata)
	s.SetServerNodeMetadata(serverNodeMetadata)
	s.SetExtraConfig(statsConfig)
	s.ClientEnvoyTemplate = env.GetTCPClientEnvoyConfTmp()
	s.ServerEnvoyTemplate = env.GetTCPServerEnvoyConfTmp()
	s.SetCopyYamlFiles(true)
	if err := s.SetUpClientServerEnvoy(); err != nil {
		t.Fatalf("Failed to setup te1	st: %v", err)
	}
	return s
}

func setupCerts(t *testing.T) *tls.Config {
	certPool := x509.NewCertPool()
	bs, err := ioutil.ReadFile(driver.TestPath("testdata/certs/cert-chain.pem"))
	if err != nil {
		t.Fatalf("failed to read client ca cert: %s", err)
	}
	ok := certPool.AppendCertsFromPEM(bs)
	if !ok {
		t.Fatal("failed to append client certs")
		return nil
	}

	certificate, err := tls.LoadX509KeyPair(driver.TestPath("testdata/certs/cert-chain.pem"),
		driver.TestPath("testdata/certs/key.pem"))
	if err != nil {
		t.Fatal("failed to get certificate")
		return nil
	}
	config := &tls.Config{Certificates: []tls.Certificate{certificate}, ServerName: "localhost", NextProtos: []string{"istio3"}, RootCAs: certPool}
	rand.Seed(time.Now().UTC().UnixNano())
	return config
}

func TestTCPMetadataExchange(t *testing.T) {
	s := setupMXC(t)
	defer s.TearDownClientServerEnvoy()

	SendRequests(t, s)

	s.VerifyEnvoyStats(getParsedExpectedStats(expectedClientStats, t, s), s.Ports().ClientAdminPort)
	s.VerifyEnvoyStats(getParsedExpectedStats(expectedServerStats, t, s), s.Ports().ServerAdminPort)

	time.Sleep(time.Second * 5)
	s.VerifyPrometheusStats(expectedPrometheusServerStats, s.Ports().ServerAdminPort)
	s.VerifyPrometheusStats(expectedPrometheusClientStats, s.Ports().ClientAdminPort)
}

func TestTCPMetadataExchangeOnTicker(t *testing.T) {
	s := setupMXC(t)
	defer s.TearDownClientServerEnvoy()

	config := setupCerts(t)

	conn, err := tls.Dial("tcp", fmt.Sprintf("localhost:%d", s.Ports().AppToClientProxyPort), config)
	if err != nil {
		t.Fatalf("Failed err: %v", err)
	}

	labels := map[string]string{
		"source_app":      "productpage",
		"destination_app": "ratings",
	}

	for i := 0; i < 10; i++ {
		conn.Write([]byte("world \n"))
		reply := make([]byte, 256)
		n, err := conn.Read(reply)
		if err != nil {
			t.Fatalf("Failed err: %v", err)
		}

		if fmt.Sprintf("%s", reply[:n]) != "hello world \n" {
			t.Fatalf("verification Failed. Expected: hello world. Got: %v", fmt.Sprintf("%s", reply[:n]))
		}

		if i == 8 {
			stats := map[string]env.Stat{
				"istio_tcp_connections_opened_total": {Value: 1, Labels: labels},
			}
			s.VerifyPrometheusStats(stats, s.Ports().ServerAdminPort)
			s.VerifyPrometheusStats(stats, s.Ports().ClientAdminPort)
		}

		time.Sleep(time.Second * 1)
	}

	_ = conn.Close()

	time.Sleep(time.Second * 2)

	stats := map[string]env.Stat{
		"istio_tcp_connections_opened_total": {Value: 1, Labels: labels},
		"istio_tcp_connections_closed_total": {Value: 1, Labels: labels},
	}
	s.VerifyPrometheusStats(stats, s.Ports().ServerAdminPort)
	s.VerifyPrometheusStats(stats, s.Ports().ClientAdminPort)
}

func TestTCPMetadataExchangeNoClientFilter(t *testing.T) {
	s := env.NewClientServerEnvoyTestSetup(env.TCPMetadataExchangeFailTest, t)
	s.Dir = driver.BazelWorkspace()
	s.SetStartHTTPBackend(false)
	s.SetStartTCPBackend(true)
	// Client send istio2  alpn in tls context.
	s.SetTLSContext(tlsContext)
	s.SetClusterTLSContext(clusterTLSContext)
	// Server accepts istio3 alpn in tls context.
	s.SetServerTLSContext(serverTLSContext)
	s.SetServerClusterTLSContext(serverClusterTLSContext)
	// Only setting mxc filter in server and stats filter in client and filter.
	// Mxc  upstream filter in  server is not set.
	s.SetFiltersBeforeEnvoyRouterInProxyToServer(metadataExchangeIstioConfigFilter)
	s.SetFiltersBeforeEnvoyRouterInAppToClient(metadataExchangeIstioClientFilter)
	s.SetEnableTLS(true)
	s.SetClientNodeMetadata(clientNodeMetadata)
	s.SetServerNodeMetadata(serverNodeMetadata)
	s.SetExtraConfig(statsConfig)
	s.ClientEnvoyTemplate = env.GetTCPClientEnvoyConfTmp()
	s.ServerEnvoyTemplate = env.GetTCPServerEnvoyConfTmp()
	if err := s.SetUpClientServerEnvoy(); err != nil {
		t.Fatalf("Failed to setup te1	st: %v", err)
	}
	defer s.TearDownClientServerEnvoy()

	SendRequests(t, s)
	s.VerifyEnvoyStats(getParsedExpectedStats(expectedServerStatsFailCase, t, s), s.Ports().ServerAdminPort)

	time.Sleep(time.Second * 5)
	s.VerifyPrometheusStats(expectedPrometheusServerStatsFailCase, s.Ports().ServerAdminPort)
}

func SendRequests(t *testing.T, s *env.TestSetup) {
	config := setupCerts(t)

	var wg sync.WaitGroup

	response := make(chan error, 5)
	wg.Add(5)
	go sendRequest(response, config, s.Ports().AppToClientProxyPort, &wg)
	go sendRequest(response, config, s.Ports().AppToClientProxyPort, &wg)
	go sendRequest(response, config, s.Ports().AppToClientProxyPort, &wg)
	go sendRequest(response, config, s.Ports().AppToClientProxyPort, &wg)
	go sendRequest(response, config, s.Ports().AppToClientProxyPort, &wg)
	wg.Wait()
	close(response)

	for err := range response {
		if err != nil {
			t.Fatal(err)
		}
	}
}

func sendRequest(response chan<- error, config *tls.Config, port uint16, wg *sync.WaitGroup) {
	defer wg.Done()
	conn, err := tls.Dial("tcp", fmt.Sprintf("localhost:%d", port), config)
	if err != nil {
		response <- err
	}

	conn.Write([]byte("world \n"))
	reply := make([]byte, 256)
	n, err := conn.Read(reply)
	if err != nil {
		response <- err
	}

	if fmt.Sprintf("%s", reply[:n]) != "hello world \n" {
		response <- fmt.Errorf("verification Failed. Expected: hello world. Got: %v", fmt.Sprintf("%s", reply[:n]))
	}

	_ = conn.Close()

	response <- nil
}

func getParsedExpectedStats(expectedStats map[string]int, t *testing.T, s *env.TestSetup) map[string]int {
	parsedExpectedStats := make(map[string]int)
	for key, value := range expectedStats {
		tmpl, err := template.New("parse_state").Parse(key)
		if err != nil {
			t.Errorf("failed to parse config template: %v", err)
		}

		var tpl bytes.Buffer
		err = tmpl.Execute(&tpl, s)
		if err != nil {
			t.Errorf("failed to execute config template: %v", err)
		}
		parsedExpectedStats[tpl.String()] = value
	}

	return parsedExpectedStats
}
<|endoftext|>


<commit_before>package veneur

import (
	"bytes"
	"compress/zlib"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
)

// Flush takes the slices of metrics, combines then and marshals them to json
// for posting to Datadog.
func (s *Server) Flush(interval time.Duration, metricLimit int) {
	// number of ddmetrics generated when a histogram flushes
	histogramSize := 2 + len(s.HistogramPercentiles)
	if s.HistogramCounter {
		histogramSize++
	}

	// allocating this long array to count up the sizes is cheaper than appending
	// the []DDMetrics together one at a time
	tempMetrics := make([]WorkerMetrics, 0, len(s.Workers))
	totalLength := 0
	for i, w := range s.Workers {
		s.logger.WithField("worker", i).Debug("Flushing")
		wm := w.Flush()
		tempMetrics = append(tempMetrics, wm)
		totalLength += len(wm.counters) + len(wm.gauges) + len(wm.sets) + ((len(wm.timers) + len(wm.histograms)) * histogramSize)
	}

	finalMetrics := make([]DDMetric, 0, totalLength)
	for _, wm := range tempMetrics {
		for _, c := range wm.counters {
			finalMetrics = append(finalMetrics, c.Flush(interval)...)
		}
		for _, g := range wm.gauges {
			finalMetrics = append(finalMetrics, g.Flush()...)
		}
		for _, h := range wm.histograms {
			finalMetrics = append(finalMetrics, h.Flush(interval, s.HistogramPercentiles, s.HistogramCounter)...)
		}
		for _, s := range wm.sets {
			finalMetrics = append(finalMetrics, s.Flush()...)
		}
		for _, t := range wm.timers {
			finalMetrics = append(finalMetrics, t.Flush(interval, s.HistogramPercentiles, s.HistogramCounter)...)
		}
	}
	for i := range finalMetrics {
		finalMetrics[i].Hostname = s.Hostname
	}

	s.statsd.Gauge("flush.post_metrics_total", float64(len(finalMetrics)), nil, 1.0)
	// Check to see if we have anything to do
	if len(finalMetrics) == 0 {
		s.logger.Info("Nothing to flush, skipping.")
		return
	}

	// break the metrics into chunks of approximately equal size, such that
	// each chunk is less than the limit
	// we compute the chunks using rounding-up integer division
	workers := ((len(finalMetrics) - 1) / metricLimit) + 1
	chunkSize := ((len(finalMetrics) - 1) / workers) + 1
	s.logger.WithField("workers", workers).Debug("Worker count chosen")
	s.logger.WithField("chunkSize", chunkSize).Debug("Chunk size chosen")
	var wg sync.WaitGroup
	flushStart := time.Now()
	for i := 0; i < workers; i++ {
		chunk := finalMetrics[i*chunkSize:]
		if i < workers-1 {
			// trim to chunk size unless this is the last one
			chunk = chunk[:chunkSize]
		}
		wg.Add(1)
		go s.flushPart(chunk, &wg)
	}
	wg.Wait()
	s.statsd.TimeInMilliseconds("flush.total_duration_ns", float64(time.Now().Sub(flushStart).Nanoseconds()), nil, 1.0)

	s.statsd.Count("flush.error_total", 0, nil, 0.1) // make sure this metric is not sparse
	s.logger.WithField("metrics", len(finalMetrics)).Info("Completed flush to Datadog")
}

func (s *Server) flushPart(metricSlice []DDMetric, wg *sync.WaitGroup) {
	defer wg.Done()

	cstart := time.Now()
	var reqBody bytes.Buffer
	compressor := zlib.NewWriter(&reqBody)
	encoder := json.NewEncoder(compressor)
	err := encoder.Encode(map[string][]DDMetric{
		"series": metricSlice,
	})
	if err != nil {
		s.statsd.Count("flush.error_total", int64(len(metricSlice)), []string{"cause:json"}, 1.0)
		s.logger.WithError(err).Error("Error rendering JSON request body")
		return
	}
	// make sure to flush remaining compressed bytes to the buffer
	compressor.Close()
	s.statsd.TimeInMilliseconds(
		"flush.part_duration_ns",
		float64(time.Now().Sub(cstart).Nanoseconds()),
		[]string{"part:marshal"},
		1.0,
	)
	// Len reports the unread length, so we have to record this before it's POSTed
	bodyLength := reqBody.Len()
	s.statsd.Histogram("flush.content_length_bytes", float64(bodyLength), nil, 1.0)

	req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/series?api_key=%s", s.DDHostname, s.DDAPIKey), &reqBody)
	if err != nil {
		s.statsd.Count("flush.error_total", int64(len(metricSlice)), []string{"cause:construct"}, 1.0)
		s.logger.WithError(err).Error("Error constructing POST request")
		return
	}
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("Content-Encoding", "deflate")

	fstart := time.Now()
	resp, err := http.DefaultClient.Do(req) // TODO: add configurable http client to server struct
	if err != nil {
		s.statsd.Count("flush.error_total", int64(len(metricSlice)), []string{"cause:io"}, 1.0)
		s.logger.WithError(err).Error("Error writing POST request")
		return
	}
	s.statsd.TimeInMilliseconds(
		"flush.part_duration_ns",
		float64(time.Now().Sub(fstart).Nanoseconds()),
		[]string{"part:post"},
		1.0,
	)
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		// don't bail out if this errors, we'll just log the body as empty
		s.logger.WithError(err).Error("Error reading response body")
	}
	resultFields := logrus.Fields{
		"status":           resp.Status,
		"request_headers":  req.Header,
		"response_headers": resp.Header,
		"request_length":   bodyLength,
		"response":         string(body),
		"total_metrics":    len(metricSlice),
	}

	if resp.StatusCode != http.StatusAccepted {
		s.statsd.Count("flush.error_total", int64(len(metricSlice)), []string{fmt.Sprintf("cause:%d", resp.StatusCode)}, 1.0)
		s.logger.WithFields(resultFields).Error("Error POSTing")
		return
	}

	s.logger.WithFields(resultFields).Debug("POSTing JSON")
}
<commit_msg>Remove sampling on error_total<commit_after>package veneur

import (
	"bytes"
	"compress/zlib"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
)

// Flush takes the slices of metrics, combines then and marshals them to json
// for posting to Datadog.
func (s *Server) Flush(interval time.Duration, metricLimit int) {
	// number of ddmetrics generated when a histogram flushes
	histogramSize := 2 + len(s.HistogramPercentiles)
	if s.HistogramCounter {
		histogramSize++
	}

	// allocating this long array to count up the sizes is cheaper than appending
	// the []DDMetrics together one at a time
	tempMetrics := make([]WorkerMetrics, 0, len(s.Workers))
	totalLength := 0
	for i, w := range s.Workers {
		s.logger.WithField("worker", i).Debug("Flushing")
		wm := w.Flush()
		tempMetrics = append(tempMetrics, wm)
		totalLength += len(wm.counters) + len(wm.gauges) + len(wm.sets) + ((len(wm.timers) + len(wm.histograms)) * histogramSize)
	}

	finalMetrics := make([]DDMetric, 0, totalLength)
	for _, wm := range tempMetrics {
		for _, c := range wm.counters {
			finalMetrics = append(finalMetrics, c.Flush(interval)...)
		}
		for _, g := range wm.gauges {
			finalMetrics = append(finalMetrics, g.Flush()...)
		}
		for _, h := range wm.histograms {
			finalMetrics = append(finalMetrics, h.Flush(interval, s.HistogramPercentiles, s.HistogramCounter)...)
		}
		for _, s := range wm.sets {
			finalMetrics = append(finalMetrics, s.Flush()...)
		}
		for _, t := range wm.timers {
			finalMetrics = append(finalMetrics, t.Flush(interval, s.HistogramPercentiles, s.HistogramCounter)...)
		}
	}
	for i := range finalMetrics {
		finalMetrics[i].Hostname = s.Hostname
	}

	s.statsd.Gauge("flush.post_metrics_total", float64(len(finalMetrics)), nil, 1.0)
	// Check to see if we have anything to do
	if len(finalMetrics) == 0 {
		s.logger.Info("Nothing to flush, skipping.")
		return
	}

	// break the metrics into chunks of approximately equal size, such that
	// each chunk is less than the limit
	// we compute the chunks using rounding-up integer division
	workers := ((len(finalMetrics) - 1) / metricLimit) + 1
	chunkSize := ((len(finalMetrics) - 1) / workers) + 1
	s.logger.WithField("workers", workers).Debug("Worker count chosen")
	s.logger.WithField("chunkSize", chunkSize).Debug("Chunk size chosen")
	var wg sync.WaitGroup
	flushStart := time.Now()
	for i := 0; i < workers; i++ {
		chunk := finalMetrics[i*chunkSize:]
		if i < workers-1 {
			// trim to chunk size unless this is the last one
			chunk = chunk[:chunkSize]
		}
		wg.Add(1)
		go s.flushPart(chunk, &wg)
	}
	wg.Wait()
	s.statsd.TimeInMilliseconds("flush.total_duration_ns", float64(time.Now().Sub(flushStart).Nanoseconds()), nil, 1.0)

	s.statsd.Count("flush.error_total", 0, nil, 1.0) // make sure this metric is not sparse
	s.logger.WithField("metrics", len(finalMetrics)).Info("Completed flush to Datadog")
}

func (s *Server) flushPart(metricSlice []DDMetric, wg *sync.WaitGroup) {
	defer wg.Done()

	cstart := time.Now()
	var reqBody bytes.Buffer
	compressor := zlib.NewWriter(&reqBody)
	encoder := json.NewEncoder(compressor)
	err := encoder.Encode(map[string][]DDMetric{
		"series": metricSlice,
	})
	if err != nil {
		s.statsd.Count("flush.error_total", int64(len(metricSlice)), []string{"cause:json"}, 1.0)
		s.logger.WithError(err).Error("Error rendering JSON request body")
		return
	}
	// make sure to flush remaining compressed bytes to the buffer
	compressor.Close()
	s.statsd.TimeInMilliseconds(
		"flush.part_duration_ns",
		float64(time.Now().Sub(cstart).Nanoseconds()),
		[]string{"part:marshal"},
		1.0,
	)
	// Len reports the unread length, so we have to record this before it's POSTed
	bodyLength := reqBody.Len()
	s.statsd.Histogram("flush.content_length_bytes", float64(bodyLength), nil, 1.0)

	req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/series?api_key=%s", s.DDHostname, s.DDAPIKey), &reqBody)
	if err != nil {
		s.statsd.Count("flush.error_total", int64(len(metricSlice)), []string{"cause:construct"}, 1.0)
		s.logger.WithError(err).Error("Error constructing POST request")
		return
	}
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("Content-Encoding", "deflate")

	fstart := time.Now()
	resp, err := http.DefaultClient.Do(req) // TODO: add configurable http client to server struct
	if err != nil {
		s.statsd.Count("flush.error_total", int64(len(metricSlice)), []string{"cause:io"}, 1.0)
		s.logger.WithError(err).Error("Error writing POST request")
		return
	}
	s.statsd.TimeInMilliseconds(
		"flush.part_duration_ns",
		float64(time.Now().Sub(fstart).Nanoseconds()),
		[]string{"part:post"},
		1.0,
	)
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		// don't bail out if this errors, we'll just log the body as empty
		s.logger.WithError(err).Error("Error reading response body")
	}
	resultFields := logrus.Fields{
		"status":           resp.Status,
		"request_headers":  req.Header,
		"response_headers": resp.Header,
		"request_length":   bodyLength,
		"response":         string(body),
		"total_metrics":    len(metricSlice),
	}

	if resp.StatusCode != http.StatusAccepted {
		s.statsd.Count("flush.error_total", int64(len(metricSlice)), []string{fmt.Sprintf("cause:%d", resp.StatusCode)}, 1.0)
		s.logger.WithFields(resultFields).Error("Error POSTing")
		return
	}

	s.logger.WithFields(resultFields).Debug("POSTing JSON")
}
<|endoftext|>


<commit_before>package formatter

import (
	"github.com/BluePecker/JwtAuth/dialog/client/formatter/context"
	"github.com/BluePecker/JwtAuth/dialog/server/parameter/jwt/response"
	"bytes"
	"strconv"
)

const (
	AddrHeader   = "CLIENT ADDR"
	TLLHeader    = "TOKEN TLL"
	DeviceHeader = "DEVICE"
	TokenHeader  = "TOKEN"

	QuietFormat    = "{{.Token}}"
	JwtTableFormat = "table {{.Addr}}\t{{.Tll}}\t{{.Device}}\t{{.Token}}"
)

type (
	JsonWebToken struct {
		context.BaseSubjectContext
		truncate bool
		jwt      response.JsonWebToken
	}

	JsonWebTokenContext struct {
		context.Context
		JsonWebTokens []response.JsonWebToken
	}
)

func (ctx JsonWebTokenContext) Write() {
	switch ctx.Template {
	case context.RawKey:
		if ctx.Quiet {
			ctx.Template = `Singed: {{.Singed}}`
		} else {
			ctx.Template = `Client Addr: {{.Addr}}\nToken TTL: {{.Tll}}\nDevice: {{.Device}}\nToken: {{.Token}}\n`
		}
	case context.TableKey:
		if ctx.Quiet {
			ctx.Template = QuietFormat
		} else {
			ctx.Template = JwtTableFormat
		}
	}

	ctx.Buffer = bytes.NewBufferString("")
	ctx.PreFormat()

	tpl, err := ctx.Parser()
	if err != nil {
		return
	}

	for _, jwt := range ctx.JsonWebTokens {
		jwtCtx := &JsonWebToken{
			truncate: ctx.Truncate,
			jwt:      jwt,
		}
		err = ctx.ContextFormat(tpl, jwtCtx)
		if err != nil {
			return
		}
	}

	ctx.FormFormat(tpl, &JsonWebToken{})
}

func (j *JsonWebToken) Addr() string {
	j.AddHeader(AddrHeader)
	return j.jwt.Addr
}

func (j *JsonWebToken) Tll() string {
	j.AddHeader(TLLHeader)
	return strconv.FormatFloat(j.jwt.TTL, 'f', -1, 64)
}

func (j *JsonWebToken) Device() string {
	j.AddHeader(DeviceHeader)
	return j.jwt.Device
}

func (j *JsonWebToken) Token() string {
	j.AddHeader(TokenHeader)
	return j.jwt.Singed
}
<commit_msg>debug<commit_after>package formatter

import (
	"github.com/BluePecker/JwtAuth/dialog/client/formatter/context"
	"github.com/BluePecker/JwtAuth/dialog/server/parameter/jwt/response"
	"bytes"
	"strconv"
)

const (
	AddrHeader   = "CLIENT ADDR"
	TLLHeader    = "TOKEN TLL"
	DeviceHeader = "DEVICE"
	TokenHeader  = "TOKEN"

	QuietFormat    = "{{.Token}}"
	JwtTableFormat = "table {{.Token}}\t{{.Addr}}\t{{.Tll}}\t{{.Device}}"
)

type (
	JsonWebToken struct {
		context.BaseSubjectContext
		truncate bool
		jwt      response.JsonWebToken
	}

	JsonWebTokenContext struct {
		context.Context
		JsonWebTokens []response.JsonWebToken
	}
)

func (ctx JsonWebTokenContext) Write() {
	switch ctx.Template {
	case context.RawKey:
		if ctx.Quiet {
			ctx.Template = `Token: {{.Token}}`
		} else {
			ctx.Template = `Token: {{.Token}}\nClient Addr: {{.Addr}}\nToken TTL: {{.Tll}}\nDevice: {{.Device}}\n`
		}
	case context.TableKey:
		if ctx.Quiet {
			ctx.Template = QuietFormat
		} else {
			ctx.Template = JwtTableFormat
		}
	}

	ctx.Buffer = bytes.NewBufferString("")
	ctx.PreFormat()

	tpl, err := ctx.Parser()
	if err != nil {
		return
	}

	for _, jwt := range ctx.JsonWebTokens {
		jwtCtx := &JsonWebToken{
			truncate: ctx.Truncate,
			jwt:      jwt,
		}
		err = ctx.ContextFormat(tpl, jwtCtx)
		if err != nil {
			return
		}
	}

	ctx.FormFormat(tpl, &JsonWebToken{})
}

func (j *JsonWebToken) Addr() string {
	j.AddHeader(AddrHeader)
	return j.jwt.Addr
}

func (j *JsonWebToken) Tll() string {
	j.AddHeader(TLLHeader)
	return strconv.FormatFloat(j.jwt.TTL, 'f', -1, 64)
}

func (j *JsonWebToken) Device() string {
	j.AddHeader(DeviceHeader)
	return j.jwt.Device
}

func (j *JsonWebToken) Token() string {
	j.AddHeader(TokenHeader)
	return j.jwt.Singed
}
<|endoftext|>


<commit_before>// Copyright 2013-2019 Aerospike, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package aerospike

// Priority of operations on database server.
type Priority int

const (

	// DEFAULT determines that the server defines the priority.
	DEFAULT Priority = iota

	// LOW determines that the server should run the operation in a background thread.
	LOW

	// MEDIUM determines that the server should run the operation at medium priority.
	MEDIUM

	// HIGH determines that the server should run the operation at the highest priority.
	HIGH
)
<commit_msg>Added clarifying comment regarding Priority only being applicable to Scans<commit_after>// Copyright 2013-2019 Aerospike, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package aerospike

// Priority of operations on database server.
// Currently, this only affects Scan operations.
type Priority int

const (

	// DEFAULT determines that the server defines the priority.
	DEFAULT Priority = iota

	// LOW determines that the server should run the operation in a background thread.
	LOW

	// MEDIUM determines that the server should run the operation at medium priority.
	MEDIUM

	// HIGH determines that the server should run the operation at the highest priority.
	HIGH
)
<|endoftext|>


<commit_before>package grip

import "github.com/coreos/go-systemd/journal"

func (self *Journaler) SetDefaultLevel(level interface{}) {
	self.defaultLevel = convertPriority(level, self.defaultLevel)
}
func SetDefaultLevel(level interface{}) {
	std.SetDefaultLevel(level)
}

func (self *Journaler) SetThreshold(level int) {
	self.thresholdLevel = convertPriority(level, self.thresholdLevel)
}
func SetThreshold(level int) {
	std.SetThreshold(level)
}

func (self *Journaler) GetThresholdLevel() int {
	return int(self.thresholdLevel)
}
func GetThresholdLevel() int {
	return int(std.thresholdLevel)
}

func (self *Journaler) GetThresholdLevelString() string {
	switch {
	case self.thresholdLevel == 0:
		return "emergency"
	case self.thresholdLevel == 1:
		return "alert"
	case self.thresholdLevel == 2:
		return "critical"
	case self.thresholdLevel == 3:
		return "error"
	case self.thresholdLevel == 4:
		return "warning"
	case self.thresholdLevel == 5:
		return "notice"
	case self.thresholdLevel == 6:
		return "info"
	case self.thresholdLevel == 7:
		return "debug"
	default:
		return ""
	}
}
func GetThresholdLevelString() string {
	return int(std.GetThresholdLevelString())
}

func convertPriority(priority interface{}, fallback journal.Priority) journal.Priority {
	switch p := priority.(type) {
	case string:
		return convertPriorityString(p, fallback)
	case int:
		return convertPriorityInt(p, fallback)
	default:
		return fallback
	}
}

func convertPriorityInt(priority int, fallback journal.Priority) journal.Priority {
	p := fallback

	switch {
	case priority == 0:
		p = journal.PriEmerg
	case priority == 1:
		p = journal.PriAlert
	case priority == 2:
		p = journal.PriCrit
	case priority == 3:
		p = journal.PriErr
	case priority == 4:
		p = journal.PriWarning
	case priority == 5:
		p = journal.PriNotice
	case priority == 6:
		p = journal.PriInfo
	case priority == 7:
		p = journal.PriDebug
	}

	return p
}

func convertPriorityString(priority string, fallback journal.Priority) journal.Priority {
	p := fallback

	switch {
	case priority == "emergency":
		p = journal.PriEmerg
	case priority == "alert":
		p = journal.PriAlert
	case priority == "critical":
		p = journal.PriCrit
	case priority == "error":
		p = journal.PriErr
	case priority == "warning":
		p = journal.PriWarning
	case priority == "notice":
		p = journal.PriNotice
	case priority == "info":
		p = journal.PriInfo
	case priority == "debig":
		p = journal.PriDebug
	}

	return p
}
<commit_msg>fixing type<commit_after>package grip

import "github.com/coreos/go-systemd/journal"

func (self *Journaler) SetDefaultLevel(level interface{}) {
	self.defaultLevel = convertPriority(level, self.defaultLevel)
}
func SetDefaultLevel(level interface{}) {
	std.SetDefaultLevel(level)
}

func (self *Journaler) SetThreshold(level interface{}) {
	self.thresholdLevel = convertPriority(level, self.thresholdLevel)
}
func SetThreshold(level interface{}) {
	std.SetThreshold(level)
}

func (self *Journaler) GetThresholdLevel() int {
	return int(self.thresholdLevel)
}
func GetThresholdLevel() int {
	return int(std.thresholdLevel)
}

func (self *Journaler) GetThresholdLevelString() string {
	switch {
	case self.thresholdLevel == 0:
		return "emergency"
	case self.thresholdLevel == 1:
		return "alert"
	case self.thresholdLevel == 2:
		return "critical"
	case self.thresholdLevel == 3:
		return "error"
	case self.thresholdLevel == 4:
		return "warning"
	case self.thresholdLevel == 5:
		return "notice"
	case self.thresholdLevel == 6:
		return "info"
	case self.thresholdLevel == 7:
		return "debug"
	default:
		return ""
	}
}
func GetThresholdLevelString() string {
	return std.GetThresholdLevelString()
}

func convertPriority(priority interface{}, fallback journal.Priority) journal.Priority {
	switch p := priority.(type) {
	case string:
		return convertPriorityString(p, fallback)
	case int:
		return convertPriorityInt(p, fallback)
	default:
		return fallback
	}
}

func convertPriorityInt(priority int, fallback journal.Priority) journal.Priority {
	p := fallback

	switch {
	case priority == 0:
		p = journal.PriEmerg
	case priority == 1:
		p = journal.PriAlert
	case priority == 2:
		p = journal.PriCrit
	case priority == 3:
		p = journal.PriErr
	case priority == 4:
		p = journal.PriWarning
	case priority == 5:
		p = journal.PriNotice
	case priority == 6:
		p = journal.PriInfo
	case priority == 7:
		p = journal.PriDebug
	}

	return p
}

func convertPriorityString(priority string, fallback journal.Priority) journal.Priority {
	p := fallback

	switch {
	case priority == "emergency":
		p = journal.PriEmerg
	case priority == "alert":
		p = journal.PriAlert
	case priority == "critical":
		p = journal.PriCrit
	case priority == "error":
		p = journal.PriErr
	case priority == "warning":
		p = journal.PriWarning
	case priority == "notice":
		p = journal.PriNotice
	case priority == "info":
		p = journal.PriInfo
	case priority == "debig":
		p = journal.PriDebug
	}

	return p
}
<|endoftext|>


<commit_before>package config

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/astaxie/beego/config"
	"github.com/astaxie/beego/utils"
	"github.com/ysqi/atop/common/log2"
	"github.com/ysqi/atop/common/util"
)

var (
	// AppCfg is the instance of Config, store the config information from file
	AppCfg *myAppConfig
	// AppPath is the absolute path to the app
	AppPath string
	// AppCfgPath is the path to the config files
	AppCfgPath string
)

func init() {
	var err error
	if AppPath, err = filepath.Abs(filepath.Dir(os.Args[0])); err != nil {
		panic(err)
	}
	if p := os.Getenv("AppConfigPath"); p != "" && utils.FileExists(p) {
		AppCfgPath = p
	} else {
		workPath, err := os.Getwd()
		if err != nil {
			panic(err)
		}
		AppCfgPath = filepath.Join(workPath, "conf", "app.conf")
	}
	if !utils.FileExists(AppCfgPath) {
		AppCfgPath = filepath.Join(AppPath, "conf", "app.conf")
		if !utils.FileExists(AppCfgPath) {
			if path, err := util.SearchFile(filepath.Join("conf", "app.conf")); err == nil {
				AppCfgPath = path
			} else {
				AppCfgPath = ""
				AppCfg = &myAppConfig{
					innerConfig: config.NewFakeConfig(),
					RunMode:     "dev",
				}
				return
			}
		}
	}
	if err = parseConfig(AppCfgPath); err != nil {
		panic(err)
	}
}

// now only support ini, next will support json.
func parseConfig(appConfigPath string) (err error) {
	log2.Infof("Load Config From %q", AppCfgPath)
	AppCfg, err = newAppConfig("ini", appConfigPath)
	if err != nil {
		return err
	}
	return assignConfig(AppCfg)
}

func assignConfig(ac config.Configer) error {
	// set the run mode first
	if envRunMode := os.Getenv(ac.DefaultString("AppName", "MyAPP") + "_RunMode"); envRunMode != "" {
		AppCfg.RunMode = envRunMode
	} else if runMode := ac.String("RunMode"); runMode != "" {
		AppCfg.RunMode = runMode
	}
	AppCfg.AppName = ac.String("AppName")

	logOuputs := map[string]string{"console": ""}
	if lo := ac.String("LogOutputs"); lo != "" {
		los := strings.Split(lo, ";")
		for _, v := range los {
			if logType2Config := strings.SplitN(v, ",", 2); len(logType2Config) == 2 {
				logOuputs[logType2Config[0]] = logType2Config[1]
			} else {
				continue
			}
		}
	}
	//init log
	logger := log2.GetLogger()
	logger.Reset()
	for adaptor, config := range logOuputs {
		err := logger.SetLogger(adaptor, config)
		if err != nil {
			fmt.Fprintln(os.Stderr, fmt.Sprintf("%s with the config %q got err:%s", adaptor, config, err.Error()))
		}
	}
	logger.EnableFuncCallDepth(false)
	return nil
}

type myAppConfig struct {
	innerConfig config.Configer
	RunMode     string
	AppName     string
}

func newAppConfig(appConfigProvider, appConfigPath string) (*myAppConfig, error) {
	ac, err := config.NewConfig(appConfigProvider, appConfigPath)
	if err != nil {
		return nil, err
	}
	return &myAppConfig{innerConfig: ac, RunMode: "dev"}, nil
}

func (b *myAppConfig) Set(key, val string) error {
	if err := b.innerConfig.Set(b.RunMode+"::"+key, val); err != nil {
		return err
	}
	return b.innerConfig.Set(key, val)
}

func (b *myAppConfig) String(key string) string {
	if v := b.innerConfig.String(b.RunMode + "::" + key); v != "" {
		return v
	}
	return b.innerConfig.String(key)
}

func (b *myAppConfig) Strings(key string) []string {
	if v := b.innerConfig.Strings(b.RunMode + "::" + key); len(v) > 0 {
		return v
	}
	return b.innerConfig.Strings(key)
}

func (b *myAppConfig) Int(key string) (int, error) {
	if v, err := b.innerConfig.Int(b.RunMode + "::" + key); err == nil {
		return v, nil
	}
	return b.innerConfig.Int(key)
}

func (b *myAppConfig) Int64(key string) (int64, error) {
	if v, err := b.innerConfig.Int64(b.RunMode + "::" + key); err == nil {
		return v, nil
	}
	return b.innerConfig.Int64(key)
}

func (b *myAppConfig) Bool(key string) (bool, error) {
	if v, err := b.innerConfig.Bool(b.RunMode + "::" + key); err == nil {
		return v, nil
	}
	return b.innerConfig.Bool(key)
}

func (b *myAppConfig) Float(key string) (float64, error) {
	if v, err := b.innerConfig.Float(b.RunMode + "::" + key); err == nil {
		return v, nil
	}
	return b.innerConfig.Float(key)
}

func (b *myAppConfig) DefaultString(key string, defaultVal string) string {
	if v := b.String(key); v != "" {
		return v
	}
	return defaultVal
}

func (b *myAppConfig) DefaultStrings(key string, defaultVal []string) []string {
	if v := b.Strings(key); len(v) != 0 {
		return v
	}
	return defaultVal
}

func (b *myAppConfig) DefaultInt(key string, defaultVal int) int {
	if v, err := b.Int(key); err == nil {
		return v
	}
	return defaultVal
}

func (b *myAppConfig) DefaultInt64(key string, defaultVal int64) int64 {
	if v, err := b.Int64(key); err == nil {
		return v
	}
	return defaultVal
}

func (b *myAppConfig) DefaultBool(key string, defaultVal bool) bool {
	if v, err := b.Bool(key); err == nil {
		return v
	}
	return defaultVal
}

func (b *myAppConfig) DefaultFloat(key string, defaultVal float64) float64 {
	if v, err := b.Float(key); err == nil {
		return v
	}
	return defaultVal
}

func (b *myAppConfig) DIY(key string) (interface{}, error) {
	return b.innerConfig.DIY(key)
}

func (b *myAppConfig) GetSection(section string) (map[string]string, error) {
	return b.innerConfig.GetSection(section)
}

func (b *myAppConfig) SaveConfigFile(filename string) error {
	return b.innerConfig.SaveConfigFile(filename)
}
<commit_msg>识别是否是Test模式<commit_after>package config

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/astaxie/beego/config"
	"github.com/astaxie/beego/utils"
	"github.com/ysqi/atop/common/log2"
	"github.com/ysqi/atop/common/util"
)

var (
	// AppCfg is the instance of Config, store the config information from file
	AppCfg *myAppConfig
	// AppPath is the absolute path to the app
	AppPath string
	// AppCfgPath is the path to the config files
	AppCfgPath string
)

func init() {
	cfgName := "app.conf"
	//检查是否是test环境下
	if flag.Lookup("test.v") != nil {
		cfgName = "app.test.conf"
	}

	var err error
	if AppPath, err = filepath.Abs(filepath.Dir(os.Args[0])); err != nil {
		panic(err)
	}
	if p := os.Getenv("AppConfigPath"); p != "" && utils.FileExists(p) {
		AppCfgPath = p
	} else {
		workPath, err := os.Getwd()
		if err != nil {
			panic(err)
		}
		AppCfgPath = filepath.Join(workPath, "conf", cfgName)
	}
	if !utils.FileExists(AppCfgPath) {
		AppCfgPath = filepath.Join(AppPath, "conf", cfgName)
		if !utils.FileExists(AppCfgPath) {
			if path, err := util.SearchFile(filepath.Join("conf", cfgName)); err == nil {
				AppCfgPath = path
			} else {
				AppCfgPath = ""
				AppCfg = &myAppConfig{
					innerConfig: config.NewFakeConfig(),
					RunMode:     "dev",
				}
				return
			}
		}
	}
	if err = parseConfig(AppCfgPath); err != nil {
		panic(err)
	}
}

// now only support ini, next will support json.
func parseConfig(appConfigPath string) (err error) {
	log2.Infof("Load Config From %q", AppCfgPath)
	AppCfg, err = newAppConfig("ini", appConfigPath)
	if err != nil {
		return err
	}
	return assignConfig(AppCfg)
}

func assignConfig(ac config.Configer) error {
	// set the run mode first
	if envRunMode := os.Getenv(ac.DefaultString("AppName", "MyAPP") + "_RunMode"); envRunMode != "" {
		AppCfg.RunMode = envRunMode
	} else if runMode := ac.String("RunMode"); runMode != "" {
		AppCfg.RunMode = runMode
	}
	AppCfg.AppName = ac.String("AppName")

	logOuputs := map[string]string{"console": ""}
	if lo := ac.String("LogOutputs"); lo != "" {
		los := strings.Split(lo, ";")
		for _, v := range los {
			if logType2Config := strings.SplitN(v, ",", 2); len(logType2Config) == 2 {
				logOuputs[logType2Config[0]] = logType2Config[1]
			} else {
				continue
			}
		}
	}
	//init log
	logger := log2.GetLogger()
	logger.Reset()
	for adaptor, config := range logOuputs {
		err := logger.SetLogger(adaptor, config)
		if err != nil {
			fmt.Fprintln(os.Stderr, fmt.Sprintf("%s with the config %q got err:%s", adaptor, config, err.Error()))
		}
	}
	logger.EnableFuncCallDepth(false)
	return nil
}

type myAppConfig struct {
	innerConfig config.Configer
	RunMode     string
	AppName     string
}

func newAppConfig(appConfigProvider, appConfigPath string) (*myAppConfig, error) {
	ac, err := config.NewConfig(appConfigProvider, appConfigPath)
	if err != nil {
		return nil, err
	}
	return &myAppConfig{innerConfig: ac, RunMode: "dev"}, nil
}

func (b *myAppConfig) Set(key, val string) error {
	if err := b.innerConfig.Set(b.RunMode+"::"+key, val); err != nil {
		return err
	}
	return b.innerConfig.Set(key, val)
}

func (b *myAppConfig) String(key string) string {
	if v := b.innerConfig.String(b.RunMode + "::" + key); v != "" {
		return v
	}
	return b.innerConfig.String(key)
}

func (b *myAppConfig) Strings(key string) []string {
	if v := b.innerConfig.Strings(b.RunMode + "::" + key); len(v) > 0 {
		return v
	}
	return b.innerConfig.Strings(key)
}

func (b *myAppConfig) Int(key string) (int, error) {
	if v, err := b.innerConfig.Int(b.RunMode + "::" + key); err == nil {
		return v, nil
	}
	return b.innerConfig.Int(key)
}

func (b *myAppConfig) Int64(key string) (int64, error) {
	if v, err := b.innerConfig.Int64(b.RunMode + "::" + key); err == nil {
		return v, nil
	}
	return b.innerConfig.Int64(key)
}

func (b *myAppConfig) Bool(key string) (bool, error) {
	if v, err := b.innerConfig.Bool(b.RunMode + "::" + key); err == nil {
		return v, nil
	}
	return b.innerConfig.Bool(key)
}

func (b *myAppConfig) Float(key string) (float64, error) {
	if v, err := b.innerConfig.Float(b.RunMode + "::" + key); err == nil {
		return v, nil
	}
	return b.innerConfig.Float(key)
}

func (b *myAppConfig) DefaultString(key string, defaultVal string) string {
	if v := b.String(key); v != "" {
		return v
	}
	return defaultVal
}

func (b *myAppConfig) DefaultStrings(key string, defaultVal []string) []string {
	if v := b.Strings(key); len(v) != 0 {
		return v
	}
	return defaultVal
}

func (b *myAppConfig) DefaultInt(key string, defaultVal int) int {
	if v, err := b.Int(key); err == nil {
		return v
	}
	return defaultVal
}

func (b *myAppConfig) DefaultInt64(key string, defaultVal int64) int64 {
	if v, err := b.Int64(key); err == nil {
		return v
	}
	return defaultVal
}

func (b *myAppConfig) DefaultBool(key string, defaultVal bool) bool {
	if v, err := b.Bool(key); err == nil {
		return v
	}
	return defaultVal
}

func (b *myAppConfig) DefaultFloat(key string, defaultVal float64) float64 {
	if v, err := b.Float(key); err == nil {
		return v
	}
	return defaultVal
}

func (b *myAppConfig) DIY(key string) (interface{}, error) {
	return b.innerConfig.DIY(key)
}

func (b *myAppConfig) GetSection(section string) (map[string]string, error) {
	return b.innerConfig.GetSection(section)
}

func (b *myAppConfig) SaveConfigFile(filename string) error {
	return b.innerConfig.SaveConfigFile(filename)
}
<|endoftext|>


<commit_before>package veneur

import (
	"fmt"
	"time"

	"github.com/VividCortex/gohistogram"
	"github.com/willf/bloom"
)

// DDMetric is a data structure that represents the JSON that Datadog
// wants when posting to the API
type DDMetric struct {
	Name       string        `json:"metric"`
	Value      [1][2]float64 `json:"points"`
	Tags       []string      `json:"tags,omitempty"`
	MetricType string        `json:"type"`
	Hostname   string        `json:"host"`
	Interval   int32         `json:"interval,omitempty"`
}

// Counter is an accumulator
type Counter struct {
	name  string
	tags  []string
	value int64
}

// Sample adds a sample to the counter.
func (c *Counter) Sample(sample float64, sampleRate float32) {
	c.value += int64(sample) * int64(1/sampleRate)
}

// Flush takes the current state of the counter, generates a
// DDMetric then clears it.
func (c *Counter) Flush(interval time.Duration) []DDMetric {
	rate := float64(c.value) / interval.Seconds()
	c.value = 0
	return []DDMetric{{
		Name:       c.name,
		Value:      [1][2]float64{{float64(time.Now().Unix()), rate}},
		Tags:       c.tags,
		MetricType: "rate",
		Interval:   int32(interval.Seconds()),
	}}
}

// NewCounter generates and returns a new Counter.
func NewCounter(name string, tags []string) *Counter {
	return &Counter{name: name, tags: tags}
}

// Gauge retains whatever the last value was.
type Gauge struct {
	name  string
	tags  []string
	value float64
}

// Sample takes on whatever value is passed in as a sample.
func (g *Gauge) Sample(sample float64, sampleRate float32) {
	g.value = sample
}

// Flush takes the current state of the gauge, generates a
// DDMetric then clears it.
func (g *Gauge) Flush() []DDMetric {
	v := g.value
	g.value = 0
	return []DDMetric{{
		Name:       g.name,
		Value:      [1][2]float64{{float64(time.Now().Unix()), float64(v)}},
		Tags:       g.tags,
		MetricType: "gauge",
	}}
}

// NewGauge genearaaaa who am I kidding just getting rid of the warning.
func NewGauge(name string, tags []string) *Gauge {
	return &Gauge{name: name, tags: tags}
}

// Set is a list of unique values seen.
type Set struct {
	name   string
	tags   []string
	value  float64
	filter *bloom.BloomFilter
}

// Sample checks if the supplied value has is already in the filter. If not, it increments
// the counter!
func (s *Set) Sample(sample string, sampleRate float32) {
	if !s.filter.TestAndAddString(sample) {
		s.value++
	}
}

// NewSet generates a new Set and returns it
func NewSet(name string, tags []string, setSize uint, accuracy float64) *Set {
	return &Set{
		name:  name,
		tags:  tags,
		value: 0,
		// TODO We could likely set this based on the set size at last flush to dynamically adjust the storage?
		filter: bloom.NewWithEstimates(setSize, accuracy),
	}
}

// Flush takes the current state of the set, generates a
// DDMetric then clears it.
func (s *Set) Flush() []DDMetric {
	v := s.value
	s.value = 0
	return []DDMetric{{
		Name:       s.name,
		Value:      [1][2]float64{{float64(time.Now().Unix()), float64(v)}},
		Tags:       s.tags,
		MetricType: "gauge",
	}}
}

// Histo is a collection of values that generates max, min, count, and
// percentiles over time.
type Histo struct {
	name        string
	tags        []string
	count       int64
	max         float64
	min         float64
	value       gohistogram.NumericHistogram
	percentiles []float64
	counter     bool
}

// Sample adds the supplied value to the histogram.
func (h *Histo) Sample(sample float64, sampleRate float32) {
	if h.count == 0 {
		h.max = sample
		h.min = sample
	} else {
		if h.max < sample {
			h.max = sample
		}
		if h.min > sample {
			h.min = sample
		}
	}
	h.count += 1 * int64(1/sampleRate)
	h.value.Add(sample)
}

// NewHist generates a new Histo and returns it.
func NewHist(name string, tags []string, percentiles []float64, counter bool) *Histo {
	// For gohistogram, the following is from the docs:
	// There is no "optimal" bin count, but somewhere between 20 and 80 bins should be sufficient. (50!)
	return &Histo{
		name:        name,
		tags:        tags,
		count:       0,
		value:       *gohistogram.NewHistogram(50),
		percentiles: percentiles,
		counter:     counter,
	}
}

// Flush generates DDMetrics for the current state of the
// Histo.
func (h *Histo) Flush(interval time.Duration) []DDMetric {
	now := float64(time.Now().Unix())
	rate := float64(h.count) / interval.Seconds()
	metrics := []DDMetric{
		{
			Name:       fmt.Sprintf("%s.max", h.name),
			Value:      [1][2]float64{{now, h.max}},
			Tags:       h.tags,
			MetricType: "gauge",
		},
		{
			Name:       fmt.Sprintf("%s.min", h.name),
			Value:      [1][2]float64{{now, h.min}},
			Tags:       h.tags,
			MetricType: "gauge",
		},
	}
	if h.counter {
		metrics = append(metrics, DDMetric{
			Name:       fmt.Sprintf("%s.count", h.name),
			Value:      [1][2]float64{{now, rate}},
			Tags:       h.tags,
			MetricType: "rate",
			Interval:   int32(interval.Seconds()),
		})
	}

	for i, p := range h.percentiles {
		metrics = append(
			metrics,
			// TODO Fix to allow for p999, etc
			DDMetric{
				Name:       fmt.Sprintf("%s.%dpercentile", h.name, int(h.percentiles[i]*100)),
				Value:      [1][2]float64{{now, h.value.Quantile(p)}},
				Tags:       h.tags,
				MetricType: "gauge",
			},
		)
	}

	return metrics
}
<commit_msg>Standardize behavior of sampler flushes<commit_after>package veneur

import (
	"fmt"
	"time"

	"github.com/VividCortex/gohistogram"
	"github.com/willf/bloom"
)

// DDMetric is a data structure that represents the JSON that Datadog
// wants when posting to the API
type DDMetric struct {
	Name       string        `json:"metric"`
	Value      [1][2]float64 `json:"points"`
	Tags       []string      `json:"tags,omitempty"`
	MetricType string        `json:"type"`
	Hostname   string        `json:"host"`
	Interval   int32         `json:"interval,omitempty"`
}

// Counter is an accumulator
type Counter struct {
	name  string
	tags  []string
	value int64
}

// Sample adds a sample to the counter.
func (c *Counter) Sample(sample float64, sampleRate float32) {
	c.value += int64(sample) * int64(1/sampleRate)
}

// Flush generates a DDMetric from the current state of this Counter.
func (c *Counter) Flush(interval time.Duration) []DDMetric {
	return []DDMetric{{
		Name:       c.name,
		Value:      [1][2]float64{{float64(time.Now().Unix()), float64(c.value) / interval.Seconds()}},
		Tags:       c.tags,
		MetricType: "rate",
		Interval:   int32(interval.Seconds()),
	}}
}

// NewCounter generates and returns a new Counter.
func NewCounter(name string, tags []string) *Counter {
	return &Counter{name: name, tags: tags}
}

// Gauge retains whatever the last value was.
type Gauge struct {
	name  string
	tags  []string
	value float64
}

// Sample takes on whatever value is passed in as a sample.
func (g *Gauge) Sample(sample float64, sampleRate float32) {
	g.value = sample
}

// Flush generates a DDMetric from the current state of this gauge.
func (g *Gauge) Flush() []DDMetric {
	return []DDMetric{{
		Name:       g.name,
		Value:      [1][2]float64{{float64(time.Now().Unix()), float64(g.value)}},
		Tags:       g.tags,
		MetricType: "gauge",
	}}
}

// NewGauge genearaaaa who am I kidding just getting rid of the warning.
func NewGauge(name string, tags []string) *Gauge {
	return &Gauge{name: name, tags: tags}
}

// Set is a list of unique values seen.
type Set struct {
	name   string
	tags   []string
	value  float64
	filter *bloom.BloomFilter
}

// Sample checks if the supplied value has is already in the filter. If not, it increments
// the counter!
func (s *Set) Sample(sample string, sampleRate float32) {
	if !s.filter.TestAndAddString(sample) {
		s.value++
	}
}

// NewSet generates a new Set and returns it
func NewSet(name string, tags []string, setSize uint, accuracy float64) *Set {
	return &Set{
		name:  name,
		tags:  tags,
		value: 0,
		// TODO We could likely set this based on the set size at last flush to dynamically adjust the storage?
		filter: bloom.NewWithEstimates(setSize, accuracy),
	}
}

// Flush generates a DDMetric for the state of this Set.
func (s *Set) Flush() []DDMetric {
	return []DDMetric{{
		Name:       s.name,
		Value:      [1][2]float64{{float64(time.Now().Unix()), float64(s.value)}},
		Tags:       s.tags,
		MetricType: "gauge",
	}}
}

// Histo is a collection of values that generates max, min, count, and
// percentiles over time.
type Histo struct {
	name        string
	tags        []string
	count       int64
	max         float64
	min         float64
	value       gohistogram.NumericHistogram
	percentiles []float64
	counter     bool
}

// Sample adds the supplied value to the histogram.
func (h *Histo) Sample(sample float64, sampleRate float32) {
	if h.count == 0 {
		h.max = sample
		h.min = sample
	} else {
		if h.max < sample {
			h.max = sample
		}
		if h.min > sample {
			h.min = sample
		}
	}
	h.count += 1 * int64(1/sampleRate)
	h.value.Add(sample)
}

// NewHist generates a new Histo and returns it.
func NewHist(name string, tags []string, percentiles []float64, counter bool) *Histo {
	// For gohistogram, the following is from the docs:
	// There is no "optimal" bin count, but somewhere between 20 and 80 bins should be sufficient. (50!)
	return &Histo{
		name:        name,
		tags:        tags,
		count:       0,
		value:       *gohistogram.NewHistogram(50),
		percentiles: percentiles,
		counter:     counter,
	}
}

// Flush generates DDMetrics for the current state of the Histo.
func (h *Histo) Flush(interval time.Duration) []DDMetric {
	now := float64(time.Now().Unix())
	rate := float64(h.count) / interval.Seconds()
	metrics := make([]DDMetric, 0, 3+len(h.percentiles))

	metrics = append(metrics,
		DDMetric{
			Name:       fmt.Sprintf("%s.max", h.name),
			Value:      [1][2]float64{{now, h.max}},
			Tags:       h.tags,
			MetricType: "gauge",
		},
		DDMetric{
			Name:       fmt.Sprintf("%s.min", h.name),
			Value:      [1][2]float64{{now, h.min}},
			Tags:       h.tags,
			MetricType: "gauge",
		},
	)
	if h.counter {
		metrics = append(metrics, DDMetric{
			Name:       fmt.Sprintf("%s.count", h.name),
			Value:      [1][2]float64{{now, rate}},
			Tags:       h.tags,
			MetricType: "rate",
			Interval:   int32(interval.Seconds()),
		})
	}

	for i, p := range h.percentiles {
		metrics = append(
			metrics,
			// TODO Fix to allow for p999, etc
			DDMetric{
				Name:       fmt.Sprintf("%s.%dpercentile", h.name, int(h.percentiles[i]*100)),
				Value:      [1][2]float64{{now, h.value.Quantile(p)}},
				Tags:       h.tags,
				MetricType: "gauge",
			},
		)
	}

	return metrics
}
<|endoftext|>


<commit_before>package stack

// This file is Copyright 2014 Manu Martinez-Almeida.  All rights reserved.
// Use of this source code is governed by a MIT style license.
//
// Modifications by Kevin Stenerson for Reflexion Health Inc. Copyright 2015

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"regexp"
	"runtime"

	"github.com/reflexionhealth/vanilla/httpserver"
)

var (
	// stackFilters is a list of Regexps to filter out lines in the callstack
	stackFilters = []*regexp.Regexp{
		regexp.MustCompile(`internal/server/context`),
		regexp.MustCompile(`internal/server/server`),
		regexp.MustCompile(`net/http/server`),
		regexp.MustCompile(`go/src/runtime`),
	}
)

var (
	dunno     = []byte("???")
	centerDot = []byte("·")
	dot       = []byte(".")
	slash     = []byte("/")
)

// Recover is a middlerware that recovers from any panics and writes a 500 if there was one.
// Logs to the specified writter buffer. If nil is provided, it will still recover, but won't log.
// Example: os.Stdout, a file opened in write mode, a socket...
func Recover(c *httpserver.Context) {
	// Use "defer" so we can capture a panic
	defer func() {
		if err := recover(); err != nil {
			stack := stack(4)
			Logger.LogResponse(c, "Panic", err)
			Logger.Logf(c, "%s\n", stack)

			if !c.Response.Rendered() {
				c.Response.Header().Set("Content-Type", "application/json; charset=utf-8")
				c.Response.Header().Set(HeaderRequestErrors, "[\"Something went wrong\"]")
				c.Response.JSON(500, "{\"errors\":[\"Something went wrong\"]}")
			} else {
				Logger.Logf(c, "\n  Panic occured after write: error not included in response\n")
			}
		}
	}()

	// Use PerfomRequest instead of ContinueRequest so that any errors that may
	// occur are wrapped by our defer and recover.
	c.PerformRequest()
}

// stack returns a nicely formated stack frame, skipping "skip" frames
func stack(skip int) []byte {
	buf := new(bytes.Buffer) // the returned data
	// As we loop, we open files and read them. These variables record the currently
	// loaded file.
	var lines [][]byte
	var lastFile string
	for i := skip; ; i++ { // Skip the expected number of frames
		pc, file, line, ok := runtime.Caller(i)
		if !ok {
			break
		}

		shouldSkip := false
		for _, exp := range stackFilters {
			if exp.MatchString(file) {
				shouldSkip = true
			}
		}

		// Skip filtered lines, but only if they aren't in the top two lines of the callstack
		if shouldSkip && i > (skip+1) {
			continue
		}

		// Print this much at least.  If we can't find the source, it won't show.
		fmt.Fprintf(buf, "\n  %s:%d (0x%x)\n", file, line, pc)
		if file != lastFile {
			data, err := ioutil.ReadFile(file)
			if err != nil {
				continue
			}
			lines = bytes.Split(data, []byte{'\n'})
			lastFile = file
		}
		fmt.Fprintf(buf, "\t%s: %s\n", function(pc), source(lines, line))
	}
	return buf.Bytes()
}

// source returns a space-trimmed slice of the n'th line.
func source(lines [][]byte, n int) []byte {
	n-- // in stack trace, lines are 1-indexed but our array is 0-indexed
	if n < 0 || n >= len(lines) {
		return dunno
	}
	return bytes.TrimSpace(lines[n])
}

// function returns, if possible, the name of the function containing the PC.
func function(pc uintptr) []byte {
	fn := runtime.FuncForPC(pc)
	if fn == nil {
		return dunno
	}
	name := []byte(fn.Name())

	// The name includes the path name to the package, which is unnecessary
	// since the file name is already included.  Plus, it has center dots.
	//
	// That is, we see:
	//
	//	runtime/debug.*T·ptrmethod
	//
	// and want:
	//
	//	*T.ptrmethod
	//
	// Also the package path might contains dot (e.g. code.google.com/...),
	// so first eliminate the path prefix
	if lastslash := bytes.LastIndex(name, slash); lastslash >= 0 {
		name = name[lastslash+1:]
	}
	if period := bytes.Index(name, dot); period >= 0 {
		name = name[period+1:]
	}
	name = bytes.Replace(name, centerDot, dot, -1)
	return name
}
<commit_msg>task: make the Recover stacktrace shorted and more useful<commit_after>package stack

// This file is Copyright 2014 Manu Martinez-Almeida.  All rights reserved.
// Use of this source code is governed by a MIT style license.
//
// Modifications by Kevin Stenerson for Reflexion Health Inc. Copyright 2015

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"regexp"
	"runtime"
	"strings"

	"github.com/reflexionhealth/vanilla/httpserver"
)

var (
	// lastLineRegexp is the stack line for "recover", we don't need to show anything after it
	lastLineRegexp = regexp.MustCompile(`vanilla/httpserver/stack/recover.go`)
	// stackFilters is a list of Regexps to filter out lines in the callstack
	stackFilters = []*regexp.Regexp{
		regexp.MustCompile(`vanilla/httpserver/context`),
		regexp.MustCompile(`vanilla/httpserver/server`),
		regexp.MustCompile(`net/http/server`),
		regexp.MustCompile(`go/src/runtime`),
	}
)

var (
	dunno     = []byte("???")
	centerDot = []byte("·")
	dot       = []byte(".")
	slash     = []byte("/")
)

// Recover is a middlerware that recovers from any panics and writes a 500 if there was one.
// Logs to the specified writter buffer. If nil is provided, it will still recover, but won't log.
// Example: os.Stdout, a file opened in write mode, a socket...
func Recover(c *httpserver.Context) {
	// Use "defer" so we can capture a panic
	defer func() {
		if err := recover(); err != nil {
			stack := stack(4)
			Logger.LogResponse(c, "Panic", err)
			Logger.Logf(c, "%s\n", stack)

			if !c.Response.Rendered() {
				c.Response.Header().Set("Content-Type", "application/json; charset=utf-8")
				c.Response.Header().Set(HeaderRequestErrors, "[\"Something went wrong\"]")
				c.Response.JSON(500, "{\"errors\":[\"Something went wrong\"]}")
			} else {
				Logger.Logf(c, "\n  Panic occured after write: error not included in response\n")
			}
		}
	}()

	// Use PerfomRequest instead of ContinueRequest so that any errors that may
	// occur are wrapped by our defer and recover.
	c.PerformRequest()
}

// stack returns a nicely formated stack frame, skipping "skip" frames
func stack(skip int) []byte {
	buf := new(bytes.Buffer) // the returned data
	// As we loop, we open files and read them. These variables record the currently
	// loaded file.
	var lines [][]byte
	var lastFile string
	for i := skip; ; i++ { // Skip the expected number of frames
		pc, file, line, ok := runtime.Caller(i)
		if !ok {
			break
		}

		shouldSkip := false
		for _, exp := range stackFilters {
			if exp.MatchString(file) {
				shouldSkip = true
			}
		}

		// Skip filtered lines, but only if they aren't in the top two lines of the callstack
		if shouldSkip && i > (skip+1) {
			continue
		}

		// Print this much at least.  If we can't find the source, it won't show.
		pathAndPkg := strings.Split(file, "/src/")
		pkgName := strings.Join(pathAndPkg[1:], "/src/")
		fmt.Fprintf(buf, "\n  %s:%d\n", pkgName, line)
		if file != lastFile {
			data, err := ioutil.ReadFile(file)
			if err != nil {
				continue
			}
			lines = bytes.Split(data, []byte{'\n'})
			lastFile = file
		}
		fmt.Fprintf(buf, "\t%s: %s\n", function(pc), source(lines, line))

		if lastLineRegexp.MatchString(file) {
			break
		}
	}
	return buf.Bytes()
}

// source returns a space-trimmed slice of the n'th line.
func source(lines [][]byte, n int) []byte {
	n-- // in stack trace, lines are 1-indexed but our array is 0-indexed
	if n < 0 || n >= len(lines) {
		return dunno
	}
	return bytes.TrimSpace(lines[n])
}

// function returns, if possible, the name of the function containing the PC.
func function(pc uintptr) []byte {
	fn := runtime.FuncForPC(pc)
	if fn == nil {
		return dunno
	}
	name := []byte(fn.Name())

	// The name includes the path name to the package, which is unnecessary
	// since the file name is already included.  Plus, it has center dots.
	//
	// That is, we see:
	//
	//	runtime/debug.*T·ptrmethod
	//
	// and want:
	//
	//	*T.ptrmethod
	//
	// Also the package path might contains dot (e.g. code.google.com/...),
	// so first eliminate the path prefix
	if lastslash := bytes.LastIndex(name, slash); lastslash >= 0 {
		name = name[lastslash+1:]
	}
	if period := bytes.Index(name, dot); period >= 0 {
		name = name[period+1:]
	}
	name = bytes.Replace(name, centerDot, dot, -1)
	return name
}
<|endoftext|>


<commit_before>package bufit

import (
	"bytes"
	"crypto/rand"
	"io"
	"io/ioutil"
	"os"
	"sync"
	"testing"
	"time"
)

func ExampleWriter() {
	buf := newWriter(make([]byte, 0, 10))
	io.Copy(os.Stdout, buf)
	io.Copy(os.Stdout, io.NewSectionReader(*&buf, 0, 100))

	io.WriteString(buf, "Hello ")
	r := io.NewSectionReader(*&buf, 0, int64(buf.Len()))
	io.CopyN(os.Stdout, r, 5)
	io.CopyN(os.Stdout, buf, 5)
	io.WriteString(buf, "World")
	r = io.NewSectionReader(*&buf, 0, int64(buf.Len()))
	io.CopyN(os.Stdout, r, 6)

	io.WriteString(buf, "abcdefg")
	io.Copy(os.Stdout, buf)
	io.Copy(os.Stdout, buf)

	io.WriteString(buf, "Hello World")
	r = io.NewSectionReader(*&buf, 0, int64(buf.Len()))
	io.CopyN(os.Stdout, r, 5)
	io.CopyN(os.Stdout, buf, 4)

	io.WriteString(buf, "abcdefg")
	io.Copy(os.Stdout, buf)
	io.Copy(os.Stdout, buf)
	//Output:
	// HelloHello World WorldabcdefgHelloHello Worldabcdefg
}

type badBuffer []byte

func (b *badBuffer) Write(p []byte) (int, error) {
	*b = append(*b, p...)
	return len(p), nil
}

func (b *badBuffer) Read(p []byte) (n int, err error) {
	n = copy(p, *b)
	*b = (*b)[n:]
	if len(*b) == 0 {
		err = io.EOF
	}
	return n, err
}

func BenchmarkBuffer(b *testing.B) {
	buf := New()
	data, _ := ioutil.ReadAll(io.LimitReader(rand.Reader, 32*1024))

	go func() {
		for i := 0; i < 1000; i++ {
			buf.Write(data)
		}
		buf.Close()
	}()

	r := buf.NextReader()

	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			rr := buf.NextReader()
			io.Copy(ioutil.Discard, rr)
			rr.Close()
		}
	})

	r.Close()
	b.ReportAllocs()
}

func BenchmarkReadWriter(b *testing.B) {
	buf := newWriter(nil)
	data, _ := ioutil.ReadAll(io.LimitReader(rand.Reader, 32*1024))
	temp := make([]byte, 32*1024)
	for i := 0; i < b.N; i++ {
		buf.Write(data)
		io.CopyBuffer(ioutil.Discard, buf, temp)
	}
	b.ReportAllocs()
}

func TestCappedBuffer(t *testing.T) {
	data := []byte("Hello World")
	buf := NewCapped(5)

	go func() {
		for i := 0; i < 5; i++ {
			n, err := buf.Write(data)
			l := buf.Len()
			if err != nil {
				t.Errorf("expected no error, got %s", err)
			}
			if n != len(data) {
				t.Errorf("expected %d bytes to be written, got %d", len(data), n)
			}
			if l > 5 {
				t.Errorf("length shouldn't exceed cap, but got %d", l)
			}

		}
		buf.Close()
	}()

	r := buf.NextReader()
	res, err := ioutil.ReadAll(r)
	if err != nil {
		t.Error(err)
	}

	if !bytes.Equal(res, bytes.Repeat(data, 5)) {
		t.Errorf("expected %s got %s", string(bytes.Repeat(data, 5)), string(res))
	}
}

func TestReadReaderAfterClose(t *testing.T) {
	buf := New()
	r := buf.NextReader()
	r.Close()
	p := make([]byte, 10)
	n, err := r.Read(p)
	if err != io.EOF {
		t.Errorf("expected io.EOF got %v, %v", n, err)
	}
}

func TestCloseReaderTwice(t *testing.T) {
	buf := New()
	r := buf.NextReader()
	r.Close()
	r.Close()
}

func TestNextReaderFromNowOnlySeesNewWrites(t *testing.T) {
	data := []byte("hello world")
	buf := New()
	r := buf.NextReader()

	buf.Write(data)
	if buf.Len() != len(data) {
		t.Errorf("expected len to be %d but got %d", len(data), buf.Len())
	}

	r2 := buf.NextReaderFromNow()

	buf.Write(data)
	buf.Close()

	out, err := ioutil.ReadAll(r2)
	if err != nil {
		t.Errorf("expected no error, got %s", err)
	}

	if !bytes.Equal(out, data) {
		t.Errorf("expected %s, got %s", data, out)
	}

	if buf.Len() != len(data)*2 {
		t.Errorf("expected len to be %d but got %d", len(data)*2, buf.Len())
	}

	r.Close()

	if buf.Len() != 0 {
		t.Errorf("expected len to be %d but got %d", 0, buf.Len())
	}

}

func TestReaderClosesWriter(t *testing.T) {
	buf := NewCapped(1)
	defer buf.Close()

	wait := make(chan struct{})

	go func() {
		// make sure blocking writes get canceled
		if _, err := io.WriteString(buf, "hello"); err != io.ErrClosedPipe {
			t.Errorf("expected %s got %s", io.ErrClosedPipe, err)
		}
		close(wait)
	}()

	buf.OnLastReaderClose(buf.Close)
	buf.NextReader().Close() // should close the blocking write

	select {
	case <-wait:
	case <-time.After(100 * time.Millisecond):
		t.Errorf("write didn't break in time")
	}
}

func assertNumReaders(n int, buf *Buffer, t *testing.T) {
	if v := buf.NumReaders(); v != n {
		t.Errorf("expected %d readers, found %d", n, v)
	}
}

func TestCloseCallback(t *testing.T) {
	called := false
	buf := New()
	buf.OnLastReaderClose(func() error {
		called = true
		return nil
	})

	assertNumReaders(0, buf, t)

	lr := buf.NextReader()
	assertNumReaders(1, buf, t)
	lr.Close()
	assertNumReaders(0, buf, t)
	if !called {
		t.Errorf("expected callback to be called")
	}

	lr = buf.NextReader()
	assertNumReaders(1, buf, t)
	called = false

	for i := 0; i < 10; i++ {
		r := buf.NextReader()
		assertNumReaders(2, buf, t)
		r.Close()
		assertNumReaders(1, buf, t)
		if called {
			t.Errorf("expected callback to not be called")
		}
	}

	assertNumReaders(1, buf, t)
	lr.Close()
	assertNumReaders(0, buf, t)
	if !called {
		t.Errorf("expected callback to be called")
	}
}

func TestConcurrent(t *testing.T) {
	var grp sync.WaitGroup
	buf := New()

	var rs []io.ReadCloser
	for i := 0; i < 1000; i++ {
		rs = append(rs, buf.NextReader())
	}

	testData := bytes.NewBuffer(nil)
	io.CopyN(testData, rand.Reader, 32*1024*10)

	for _, r := range rs {
		grp.Add(1)
		go func(r io.ReadCloser) {
			defer grp.Done()
			defer r.Close()
			data, err := ioutil.ReadAll(r)
			if err != nil {
				t.Error(err)
			}
			if !bytes.Equal(testData.Bytes(), data) {
				t.Error("unexpected result...", testData.Len(), len(data))
			}
		}(r)
	}

	r := bytes.NewReader(testData.Bytes())
	for r.Len() > 0 {
		io.CopyN(buf, r, 32*1024*2)
		<-time.After(100 * time.Millisecond)
	}
	buf.Close()
	grp.Wait()
}

func TestQuitReader(t *testing.T) {
	buf := New()
	r := buf.NextReader()

	wait := make(chan struct{})
	go func() {
		io.Copy(ioutil.Discard, r)
		close(wait)
	}()

	r.Close()
	select {
	case <-wait:
	case <-time.After(100 * time.Millisecond):
		t.Error("timed out waiting for Reader to Close")
	}
}

func TestQuitWriter(t *testing.T) {
	buf := New()
	buf.Close()
	_, err := io.WriteString(buf, ".")
	if err != io.ErrClosedPipe {
		t.Errorf("Writer after Close expected io.ErrClosedPipe but got %v", err)
	}
}

func TestQuitCappedWriter(t *testing.T) {
	buf := NewCapped(2)
	go func() {
		// wait until blocking write has started
		<-time.After(100 * time.Millisecond)
		buf.Close()
	}()
	_, err := io.WriteString(buf, "hello world")
	if err != io.ErrClosedPipe {
		t.Errorf("Writer after Close expected io.ErrClosedPipe but got %v", err)
	}
}

func ExampleBuffer() {
	// Start a new buffer
	buf := New()

	// Create two readers
	r1, r2 := buf.NextReader(), buf.NextReader()

	// Broadcast a message
	io.WriteString(buf, "Hello World\n")

	// Wait
	var grp sync.WaitGroup
	grp.Add(4)

	// Read fast
	go func() {
		defer grp.Done()
		io.Copy(os.Stdout, r1) // "Hello World\n"
	}()

	// Read slow
	go func() {
		defer grp.Done()
		<-time.After(100 * time.Millisecond)
		io.CopyN(os.Stdout, r2, 5) // "Hello"
		<-time.After(time.Second)
		io.Copy(os.Stdout, r2) // "World\n"
	}()

	// Both readers will read the entire buffer! The slow reader
	// won't block the fast one from reading ahead either.

	// Late reader
	// Since this reader joins after all existing readers have Read "Hello"
	// "Hello" has already been cleared from the Buffer, this Reader will only see
	// "World\n" and beyond.
	go func() {
		defer grp.Done()
		<-time.After(500 * time.Millisecond)
		r3 := buf.NextReader()
		io.Copy(os.Stdout, r3) // "World\n"
	}()

	// Short Reader
	// **Important!** if your reader isn't going to read until the buffer is empty
	// you'll need to call Close() when you are done with it to tell the buffer
	// it's done reading data.
	go func() {
		defer grp.Done()
		<-time.After(100 * time.Millisecond)
		r4 := buf.NextReader()
		io.CopyN(os.Stdout, r4, 5) // "Hello"
		r4.Close()                 // tell the buffer you're done reading
	}()

	// **Important!** mark close so that readers can ret. io.EOF
	buf.Close()

	grp.Wait()
	// Output:
	// Hello World
	// HelloHelloHello World
	//  World
}
<commit_msg>adding a benchmark with no lagging reader so buffer size can shrink<commit_after>package bufit

import (
	"bytes"
	"crypto/rand"
	"io"
	"io/ioutil"
	"os"
	"sync"
	"testing"
	"time"
)

func ExampleWriter() {
	buf := newWriter(make([]byte, 0, 10))
	io.Copy(os.Stdout, buf)
	io.Copy(os.Stdout, io.NewSectionReader(*&buf, 0, 100))

	io.WriteString(buf, "Hello ")
	r := io.NewSectionReader(*&buf, 0, int64(buf.Len()))
	io.CopyN(os.Stdout, r, 5)
	io.CopyN(os.Stdout, buf, 5)
	io.WriteString(buf, "World")
	r = io.NewSectionReader(*&buf, 0, int64(buf.Len()))
	io.CopyN(os.Stdout, r, 6)

	io.WriteString(buf, "abcdefg")
	io.Copy(os.Stdout, buf)
	io.Copy(os.Stdout, buf)

	io.WriteString(buf, "Hello World")
	r = io.NewSectionReader(*&buf, 0, int64(buf.Len()))
	io.CopyN(os.Stdout, r, 5)
	io.CopyN(os.Stdout, buf, 4)

	io.WriteString(buf, "abcdefg")
	io.Copy(os.Stdout, buf)
	io.Copy(os.Stdout, buf)
	//Output:
	// HelloHello World WorldabcdefgHelloHello Worldabcdefg
}

type badBuffer []byte

func (b *badBuffer) Write(p []byte) (int, error) {
	*b = append(*b, p...)
	return len(p), nil
}

func (b *badBuffer) Read(p []byte) (n int, err error) {
	n = copy(p, *b)
	*b = (*b)[n:]
	if len(*b) == 0 {
		err = io.EOF
	}
	return n, err
}

func BenchmarkBufferNoPressure(b *testing.B) {
	data, _ := ioutil.ReadAll(io.LimitReader(rand.Reader, 32*1024))

	buf := NewCapped(len(data))

	go func() {
		for i := 0; i < 1000; i++ {
			buf.Write(data)
		}
		buf.Close()
	}()

	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			rr := buf.NextReader()
			io.Copy(ioutil.Discard, rr)
			rr.Close()
		}
	})

	b.ReportAllocs()
}

func BenchmarkBuffer(b *testing.B) {
	buf := New()
	data, _ := ioutil.ReadAll(io.LimitReader(rand.Reader, 32*1024))

	go func() {
		for i := 0; i < 1000; i++ {
			buf.Write(data)
		}
		buf.Close()
	}()

	r := buf.NextReader()

	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			rr := buf.NextReader()
			io.Copy(ioutil.Discard, rr)
			rr.Close()
		}
	})

	r.Close()
	b.ReportAllocs()
}

func BenchmarkReadWriter(b *testing.B) {
	buf := newWriter(nil)
	data, _ := ioutil.ReadAll(io.LimitReader(rand.Reader, 32*1024))
	temp := make([]byte, 32*1024)
	for i := 0; i < b.N; i++ {
		buf.Write(data)
		io.CopyBuffer(ioutil.Discard, buf, temp)
	}
	b.ReportAllocs()
}

func TestCappedBuffer(t *testing.T) {
	data := []byte("Hello World")
	buf := NewCapped(5)

	go func() {
		for i := 0; i < 5; i++ {
			n, err := buf.Write(data)
			l := buf.Len()
			if err != nil {
				t.Errorf("expected no error, got %s", err)
			}
			if n != len(data) {
				t.Errorf("expected %d bytes to be written, got %d", len(data), n)
			}
			if l > 5 {
				t.Errorf("length shouldn't exceed cap, but got %d", l)
			}

		}
		buf.Close()
	}()

	r := buf.NextReader()
	res, err := ioutil.ReadAll(r)
	if err != nil {
		t.Error(err)
	}

	if !bytes.Equal(res, bytes.Repeat(data, 5)) {
		t.Errorf("expected %s got %s", string(bytes.Repeat(data, 5)), string(res))
	}
}

func TestReadReaderAfterClose(t *testing.T) {
	buf := New()
	r := buf.NextReader()
	r.Close()
	p := make([]byte, 10)
	n, err := r.Read(p)
	if err != io.EOF {
		t.Errorf("expected io.EOF got %v, %v", n, err)
	}
}

func TestCloseReaderTwice(t *testing.T) {
	buf := New()
	r := buf.NextReader()
	r.Close()
	r.Close()
}

func TestNextReaderFromNowOnlySeesNewWrites(t *testing.T) {
	data := []byte("hello world")
	buf := New()
	r := buf.NextReader()

	buf.Write(data)
	if buf.Len() != len(data) {
		t.Errorf("expected len to be %d but got %d", len(data), buf.Len())
	}

	r2 := buf.NextReaderFromNow()

	buf.Write(data)
	buf.Close()

	out, err := ioutil.ReadAll(r2)
	if err != nil {
		t.Errorf("expected no error, got %s", err)
	}

	if !bytes.Equal(out, data) {
		t.Errorf("expected %s, got %s", data, out)
	}

	if buf.Len() != len(data)*2 {
		t.Errorf("expected len to be %d but got %d", len(data)*2, buf.Len())
	}

	r.Close()

	if buf.Len() != 0 {
		t.Errorf("expected len to be %d but got %d", 0, buf.Len())
	}

}

func TestReaderClosesWriter(t *testing.T) {
	buf := NewCapped(1)
	defer buf.Close()

	wait := make(chan struct{})

	go func() {
		// make sure blocking writes get canceled
		if _, err := io.WriteString(buf, "hello"); err != io.ErrClosedPipe {
			t.Errorf("expected %s got %s", io.ErrClosedPipe, err)
		}
		close(wait)
	}()

	buf.OnLastReaderClose(buf.Close)
	buf.NextReader().Close() // should close the blocking write

	select {
	case <-wait:
	case <-time.After(100 * time.Millisecond):
		t.Errorf("write didn't break in time")
	}
}

func assertNumReaders(n int, buf *Buffer, t *testing.T) {
	if v := buf.NumReaders(); v != n {
		t.Errorf("expected %d readers, found %d", n, v)
	}
}

func TestCloseCallback(t *testing.T) {
	called := false
	buf := New()
	buf.OnLastReaderClose(func() error {
		called = true
		return nil
	})

	assertNumReaders(0, buf, t)

	lr := buf.NextReader()
	assertNumReaders(1, buf, t)
	lr.Close()
	assertNumReaders(0, buf, t)
	if !called {
		t.Errorf("expected callback to be called")
	}

	lr = buf.NextReader()
	assertNumReaders(1, buf, t)
	called = false

	for i := 0; i < 10; i++ {
		r := buf.NextReader()
		assertNumReaders(2, buf, t)
		r.Close()
		assertNumReaders(1, buf, t)
		if called {
			t.Errorf("expected callback to not be called")
		}
	}

	assertNumReaders(1, buf, t)
	lr.Close()
	assertNumReaders(0, buf, t)
	if !called {
		t.Errorf("expected callback to be called")
	}
}

func TestConcurrent(t *testing.T) {
	var grp sync.WaitGroup
	buf := New()

	var rs []io.ReadCloser
	for i := 0; i < 1000; i++ {
		rs = append(rs, buf.NextReader())
	}

	testData := bytes.NewBuffer(nil)
	io.CopyN(testData, rand.Reader, 32*1024*10)

	for _, r := range rs {
		grp.Add(1)
		go func(r io.ReadCloser) {
			defer grp.Done()
			defer r.Close()
			data, err := ioutil.ReadAll(r)
			if err != nil {
				t.Error(err)
			}
			if !bytes.Equal(testData.Bytes(), data) {
				t.Error("unexpected result...", testData.Len(), len(data))
			}
		}(r)
	}

	r := bytes.NewReader(testData.Bytes())
	for r.Len() > 0 {
		io.CopyN(buf, r, 32*1024*2)
		<-time.After(100 * time.Millisecond)
	}
	buf.Close()
	grp.Wait()
}

func TestQuitReader(t *testing.T) {
	buf := New()
	r := buf.NextReader()

	wait := make(chan struct{})
	go func() {
		io.Copy(ioutil.Discard, r)
		close(wait)
	}()

	r.Close()
	select {
	case <-wait:
	case <-time.After(100 * time.Millisecond):
		t.Error("timed out waiting for Reader to Close")
	}
}

func TestQuitWriter(t *testing.T) {
	buf := New()
	buf.Close()
	_, err := io.WriteString(buf, ".")
	if err != io.ErrClosedPipe {
		t.Errorf("Writer after Close expected io.ErrClosedPipe but got %v", err)
	}
}

func TestQuitCappedWriter(t *testing.T) {
	buf := NewCapped(2)
	go func() {
		// wait until blocking write has started
		<-time.After(100 * time.Millisecond)
		buf.Close()
	}()
	_, err := io.WriteString(buf, "hello world")
	if err != io.ErrClosedPipe {
		t.Errorf("Writer after Close expected io.ErrClosedPipe but got %v", err)
	}
}

func ExampleBuffer() {
	// Start a new buffer
	buf := New()

	// Create two readers
	r1, r2 := buf.NextReader(), buf.NextReader()

	// Broadcast a message
	io.WriteString(buf, "Hello World\n")

	// Wait
	var grp sync.WaitGroup
	grp.Add(4)

	// Read fast
	go func() {
		defer grp.Done()
		io.Copy(os.Stdout, r1) // "Hello World\n"
	}()

	// Read slow
	go func() {
		defer grp.Done()
		<-time.After(100 * time.Millisecond)
		io.CopyN(os.Stdout, r2, 5) // "Hello"
		<-time.After(time.Second)
		io.Copy(os.Stdout, r2) // "World\n"
	}()

	// Both readers will read the entire buffer! The slow reader
	// won't block the fast one from reading ahead either.

	// Late reader
	// Since this reader joins after all existing readers have Read "Hello"
	// "Hello" has already been cleared from the Buffer, this Reader will only see
	// "World\n" and beyond.
	go func() {
		defer grp.Done()
		<-time.After(500 * time.Millisecond)
		r3 := buf.NextReader()
		io.Copy(os.Stdout, r3) // "World\n"
	}()

	// Short Reader
	// **Important!** if your reader isn't going to read until the buffer is empty
	// you'll need to call Close() when you are done with it to tell the buffer
	// it's done reading data.
	go func() {
		defer grp.Done()
		<-time.After(100 * time.Millisecond)
		r4 := buf.NextReader()
		io.CopyN(os.Stdout, r4, 5) // "Hello"
		r4.Close()                 // tell the buffer you're done reading
	}()

	// **Important!** mark close so that readers can ret. io.EOF
	buf.Close()

	grp.Wait()
	// Output:
	// Hello World
	// HelloHelloHello World
	//  World
}
<|endoftext|>


<commit_before>package dclient

import "github.com/fsouza/go-dockerclient"

/*
APIImagesSlice is a type that wraps []docker.APIImages so that it can be sorted
using the "sort" package Interface.
*/
type APIImagesSlice []docker.APIImages

func (slice APIImagesSlice) Len() int {
	return len(slice)
}

func (slice APIImagesSlice) Less(i, j int) bool {
	return slice[i].Created > slice[j].Created
}

func (slice APIImagesSlice) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}

/*
FirstID - figure out what this does.
*/
func (slice APIImagesSlice) FirstID() string {
	return slice[0].ID
}
<commit_msg>Remove unused function<commit_after>package dclient

import "github.com/fsouza/go-dockerclient"

/*
APIImagesSlice is a type that wraps []docker.APIImages so that it can be sorted
using the "sort" package Interface.
*/
type APIImagesSlice []docker.APIImages

func (slice APIImagesSlice) Len() int {
	return len(slice)
}

func (slice APIImagesSlice) Less(i, j int) bool {
	return slice[i].Created > slice[j].Created
}

func (slice APIImagesSlice) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}
<|endoftext|>


<commit_before>//Author: Gary Connelly
//date 22/09/2017

//https://www.youtube.com/watch?v=gh1yOouqFs0

package main

import "fmt"

func main() {
	const myNumber = 5
	count := 1
	var guess int
	fmt.Println("Guess my number: ")
	fmt.Scanf("%v", &guess)

	while guess != myNumber {
		if guess < myNumber {
			fmt.Println("Too small, try again  ")
			fmt.Println("Guess my number: ")
			fmt.Scanf("%v", &guess)
			count++
		} else {
			fmt.Println("Too high, try again  ")
			fmt.Println("Guess my number: ")
			fmt.Scanf("%v", &guess)
			count++
		}
	}//end while
}<commit_msg>editing the counter<commit_after>//Author: Gary Connelly
//date 22/09/2017

//https://www.youtube.com/watch?v=gh1yOouqFs0

package main

import "fmt"

func main() {
	const myNumber = 5
	count := 1
	var guess int
	fmt.Println("Guess my number: ")
	fmt.Scanf("%v", &guess)

	while guess != myNumber {
	count++
		if guess < myNumber {
			fmt.Println("Too small, try again  ")
			fmt.Println("Guess my number: ")
			fmt.Scanf("%v", &guess)

		} else if guess > myNumber {
			fmt.Println("Too high, try again  ")
			fmt.Println("Guess my number: ")
			fmt.Scanf("%v", &guess)

		}
	}//end while

}<|endoftext|>


<commit_before>package pt

type Plane struct {
	Point    Vector
	Normal   Vector
	Material Material
}

func NewPlane(point, normal Vector, material Material) *Plane {
	normal = normal.Normalize()
	return &Plane{point, normal, material}
}

func (p *Plane) Compile() {
}

func (p *Plane) BoundingBox() Box {
	return Box{Vector{-INF, -INF, -INF}, Vector{INF, INF, INF}}
}

func (p *Plane) Intersect(ray Ray) Hit {
	d := p.Normal.Dot(ray.Direction)
	if d < EPS {
		return NoHit
	}
	a := p.Point.Sub(ray.Origin)
	t := a.Dot(p.Normal) / d
	if t < EPS {
		return NoHit
	}
	return Hit{p, t, nil}
}

func (p *Plane) UV(a Vector) Vector {
	return Vector{}
}

func (p *Plane) MaterialAt(a Vector) Material {
	return p.Material
}

func (p *Plane) NormalAt(a Vector) Vector {
	return p.Normal
}
<commit_msg>plane fix<commit_after>package pt

import "math"

type Plane struct {
	Point    Vector
	Normal   Vector
	Material Material
}

func NewPlane(point, normal Vector, material Material) *Plane {
	normal = normal.Normalize()
	return &Plane{point, normal, material}
}

func (p *Plane) Compile() {
}

func (p *Plane) BoundingBox() Box {
	return Box{Vector{-INF, -INF, -INF}, Vector{INF, INF, INF}}
}

func (p *Plane) Intersect(ray Ray) Hit {
	d := p.Normal.Dot(ray.Direction)
	if math.Abs(d) < EPS {
		return NoHit
	}
	a := p.Point.Sub(ray.Origin)
	t := a.Dot(p.Normal) / d
	if t <= 0 {
		return NoHit
	}
	return Hit{p, t, nil}
}

func (p *Plane) UV(a Vector) Vector {
	return Vector{}
}

func (p *Plane) MaterialAt(a Vector) Material {
	return p.Material
}

func (p *Plane) NormalAt(a Vector) Vector {
	return p.Normal
}
<|endoftext|>


<commit_before>package main

import (
	gojson "encoding/json"
	"io/ioutil"
	"os"

	"github.com/cloudfoundry/cli/json"
)

func main() {
	pathToJSONFile := os.Args[1]
	rules, err := json.ParseJSON(pathToJSONFile)
	if err != nil {
		panic(err.Error())
	}

	jsonOutput, jsonErr := gojson.MarshalIndent(rules, "", "   ")
	if jsonErr != nil {
		panic(jsonErr.Error())
	}
	ioutil.WriteFile(pathToJSONFile, jsonOutput, 0644)
}
<commit_msg>Fix travis build by deleting obsolete tool<commit_after><|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"

	"github.com/koding/kite"
)

const publicEcho = "http://echoip.com"
const proxyHostUrl = "koding.com/-/userproxy/"

func getRegisterURL(k *kite.Kite) (*url.URL, error) {
	ip, err := publicIP()
	if err != nil {
		return nil, err
	}

	// http://localhost:8090/userproxy/54.164.243.111/kite

	return &url.URL{
		Scheme: "https",
		Host:   proxyHostUrl + ip.String(),
		Path:   "/kite",
	}, nil
}

// publicIP returns an IP that is supposed to be Public.
func publicIP() (net.IP, error) {
	resp, err := http.Get(publicEcho)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	out, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	n := net.ParseIP(string(out))
	if n == nil {
		return nil, fmt.Errorf("cannot parse ip %s", string(out))
	}

	return n, nil
}
<commit_msg>klient: register with public IP so we know the real IP<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"strconv"

	"github.com/koding/kite"
)

const publicEcho = "http://echoip.com"

func getRegisterURL(k *kite.Kite) (*url.URL, error) {
	ip, err := publicIP()
	if err != nil {
		return nil, err
	}

	scheme := "http"
	if k.TLSConfig != nil {
		scheme = "https"
	}

	return &url.URL{
		Scheme: scheme,
		Host:   ip.String() + ":" + strconv.Itoa(k.Config.Port),
		Path:   "/kite",
	}, nil
}

// publicIP returns an IP that is supposed to be Public.
func publicIP() (net.IP, error) {
	resp, err := http.Get(publicEcho)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	out, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	n := net.ParseIP(string(out))
	if n == nil {
		return nil, fmt.Errorf("cannot parse ip %s", string(out))
	}

	return n, nil
}
<|endoftext|>