<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package index

import (
	"context"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"go.opencensus.io/plugin/ochttp"
	"golang.org/x/discovery/internal"
	"golang.org/x/discovery/internal/derrors"
	"golang.org/x/net/context/ctxhttp"
)

// A Client is used by the etl service to communicate with the module index.
type Client struct {
	// URL of the module index
	url string

	// client used for HTTP requests. It is mutable for testing purposes.
	httpClient *http.Client
}

// New constructs a *Client using the provided rawurl, which is expected to
// be an absolute URI that can be directly passed to http.Get.
func New(rawurl string) (_ *Client, err error) {
	defer derrors.Add(&err, "index.New(%q)", rawurl)

	u, err := url.Parse(rawurl)
	if err != nil {
		return nil, fmt.Errorf("url.Parse(%q): %v", rawurl, err)
	}
	if u.Scheme != "https" {
		return nil, fmt.Errorf("scheme must be https (got %s)", u.Scheme)
	}
	return &Client{url: strings.TrimRight(rawurl, "/"), httpClient: &http.Client{Transport: &ochttp.Transport{}}}, nil
}

func (c *Client) pollURL(since time.Time, limit int) string {
	values := url.Values{}
	values.Set("since", since.Format(time.RFC3339))
	if limit > 0 {
		values.Set("limit", strconv.Itoa(limit))
	}
	return fmt.Sprintf("%s?%s", c.url, values.Encode())
}

// GetVersions queries the index for new versions.
func (c *Client) GetVersions(ctx context.Context, since time.Time, limit int) ([]*internal.IndexVersion, error) {
	u := c.pollURL(since, limit)
	r, err := ctxhttp.Get(ctx, c.httpClient, u)
	if err != nil {
		return nil, fmt.Errorf("index.Client.GetVersions(ctx, %s, %d): ctxhttp.Get(ctx, nil, %q): %v",
			since, limit, u, err)
	}
	defer r.Body.Close()

	var versions []*internal.IndexVersion
	dec := json.NewDecoder(r.Body)

	// The module index returns a stream of JSON objects formatted with newline
	// as the delimiter. For each version log, we want to set source to
	// "proxy-index" and created_at to the time right before the proxy index is
	// queried.
	for dec.More() {
		var l internal.IndexVersion
		if err := dec.Decode(&l); err != nil {
			log.Printf("dec.Decode: %v", err)
			continue
		}
		// The created_at column is without timestamp, so we must normalize to UTC.
		l.Timestamp = l.Timestamp.UTC()
		versions = append(versions, &l)
	}
	return versions, nil
}
<commit_msg>internal/index: make GetVersions fail fast<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package index

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"go.opencensus.io/plugin/ochttp"
	"golang.org/x/discovery/internal"
	"golang.org/x/discovery/internal/derrors"
	"golang.org/x/net/context/ctxhttp"
)

// A Client is used by the etl service to communicate with the module index.
type Client struct {
	// URL of the module index
	url string

	// client used for HTTP requests. It is mutable for testing purposes.
	httpClient *http.Client
}

// New constructs a *Client using the provided rawurl, which is expected to
// be an absolute URI that can be directly passed to http.Get.
func New(rawurl string) (_ *Client, err error) {
	defer derrors.Add(&err, "index.New(%q)", rawurl)

	u, err := url.Parse(rawurl)
	if err != nil {
		return nil, fmt.Errorf("url.Parse(%q): %v", rawurl, err)
	}
	if u.Scheme != "https" {
		return nil, fmt.Errorf("scheme must be https (got %s)", u.Scheme)
	}
	return &Client{url: strings.TrimRight(rawurl, "/"), httpClient: &http.Client{Transport: &ochttp.Transport{}}}, nil
}

func (c *Client) pollURL(since time.Time, limit int) string {
	values := url.Values{}
	values.Set("since", since.Format(time.RFC3339))
	if limit > 0 {
		values.Set("limit", strconv.Itoa(limit))
	}
	return fmt.Sprintf("%s?%s", c.url, values.Encode())
}

// GetVersions queries the index for new versions.
func (c *Client) GetVersions(ctx context.Context, since time.Time, limit int) (_ []*internal.IndexVersion, err error) {
	defer derrors.Wrap(&err, "index.Client.GetVersions(ctx, %s, %d)", since, limit)

	u := c.pollURL(since, limit)
	r, err := ctxhttp.Get(ctx, c.httpClient, u)
	if err != nil {
		return nil, fmt.Errorf("ctxhttp.Get(ctx, nil, %q): %v", u, err)
	}
	defer r.Body.Close()

	var versions []*internal.IndexVersion
	dec := json.NewDecoder(r.Body)

	// The module index returns a stream of JSON objects formatted with newline
	// as the delimiter.
	for dec.More() {
		var l internal.IndexVersion
		if err := dec.Decode(&l); err != nil {
			return nil, fmt.Errorf("decoding JSON: %v", err)
		}
		versions = append(versions, &l)
	}
	return versions, nil
}
<|endoftext|>


<commit_before>// Copyright 2015, Joe Tsai. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE.md file.

package prefix

import (
	"bytes"
	"io"
	"io/ioutil"
	"os"
	"strings"
)

// For some of the common Readers, we wrap and extend them to satisfy the
// compress.BufferedReader interface to improve performance.

type buffer struct {
	*bytes.Buffer
}

type bytesReader struct {
	*bytes.Reader
	limRd   io.LimitedReader
	scratch [512]byte
}

type stringReader struct {
	*strings.Reader
	limRd   io.LimitedReader
	scratch [512]byte
}

func (r *buffer) Buffered() int {
	return r.Len()
}

func (r *buffer) Peek(n int) ([]byte, error) {
	b := r.Bytes()
	if len(b) >= n {
		return b[:n], nil
	}
	return b, io.EOF
}

func (r *buffer) Discard(n int) (int, error) {
	b := r.Next(n)
	if len(b) == n {
		return n, nil
	}
	return len(b), io.EOF
}

func (r *bytesReader) Buffered() int {
	if r.Len() > len(r.scratch) {
		return len(r.scratch)
	}
	return r.Len()
}

func (r *bytesReader) Peek(n int) ([]byte, error) {
	if n > len(r.scratch) {
		return nil, io.ErrShortBuffer
	}
	pos, _ := r.Seek(0, os.SEEK_CUR) // Get current position, never fails
	cnt, err := r.ReadAt(r.scratch[:n], pos)
	return r.scratch[:cnt], err
}

func (r *bytesReader) Discard(n int) (int, error) {
	r.limRd = io.LimitedReader{R: r, N: int64(n)}
	n64, err := io.CopyBuffer(ioutil.Discard, &r.limRd, r.scratch[:])
	if err == nil && n64 < int64(n) {
		err = io.EOF
	}
	return int(n64), err
}

func (r *stringReader) Buffered() int {
	if r.Len() > len(r.scratch) {
		return len(r.scratch)
	}
	return r.Len()
}

func (r *stringReader) Peek(n int) ([]byte, error) {
	if n > len(r.scratch) {
		return nil, io.ErrShortBuffer
	}
	pos, _ := r.Seek(0, os.SEEK_CUR) // Get current position, never fails
	cnt, err := r.ReadAt(r.scratch[:n], pos)
	return r.scratch[:cnt], err
}

func (r *stringReader) Discard(n int) (int, error) {
	r.limRd = io.LimitedReader{R: r, N: int64(n)}
	n64, err := io.CopyBuffer(ioutil.Discard, &r.limRd, r.scratch[:])
	if err == nil && n64 < int64(n) {
		err = io.EOF
	}
	return int(n64), err
}
<commit_msg>internal/prefix: optimize performance of wrappers<commit_after>// Copyright 2015, Joe Tsai. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE.md file.

package prefix

import (
	"bytes"
	"io"
	"os"
	"strings"
)

// For some of the common Readers, we wrap and extend them to satisfy the
// compress.BufferedReader interface to improve performance.

type buffer struct {
	*bytes.Buffer
}

type bytesReader struct {
	*bytes.Reader
	pos int64
	buf []byte
	arr [512]byte
}

type stringReader struct {
	*strings.Reader
	pos int64
	buf []byte
	arr [512]byte
}

func (r *buffer) Buffered() int {
	return r.Len()
}

func (r *buffer) Peek(n int) ([]byte, error) {
	b := r.Bytes()
	if len(b) < n {
		return b, io.EOF
	}
	return b[:n], nil
}

func (r *buffer) Discard(n int) (int, error) {
	b := r.Next(n)
	if len(b) < n {
		return len(b), io.EOF
	}
	return n, nil
}

func (r *bytesReader) Buffered() int {
	if r.Len() > len(r.arr) {
		return len(r.arr)
	}
	return r.Len()
}

func (r *bytesReader) Peek(n int) ([]byte, error) {
	if n > len(r.arr) {
		return nil, io.ErrShortBuffer
	}

	pos, _ := r.Seek(0, os.SEEK_CUR)
	if off := pos - r.pos; off > 0 && off < int64(len(r.buf)) {
		r.buf, r.pos = r.buf[off:], pos
	}
	if len(r.buf) >= n && r.pos == pos {
		return r.buf[:n], nil
	}

	cnt, err := r.ReadAt(r.arr[:], pos)
	r.buf, r.pos = r.arr[:cnt], pos
	if cnt < n {
		return r.arr[:cnt], err
	}
	return r.arr[:n], nil
}

func (r *bytesReader) Discard(n int) (int, error) {
	var err error
	if n > r.Len() {
		n, err = r.Len(), io.EOF
	}
	r.Seek(int64(n), os.SEEK_CUR)
	return n, err
}

func (r *stringReader) Buffered() int {
	if r.Len() > len(r.arr) {
		return len(r.arr)
	}
	return r.Len()
}

func (r *stringReader) Peek(n int) ([]byte, error) {
	if n > len(r.arr) {
		return nil, io.ErrShortBuffer
	}

	pos, _ := r.Seek(0, os.SEEK_CUR)
	if off := pos - r.pos; off > 0 && off < int64(len(r.buf)) {
		r.buf, r.pos = r.buf[off:], pos
	}
	if n <= len(r.buf) && pos == r.pos {
		return r.buf[:n], nil
	}

	cnt, err := r.ReadAt(r.arr[:], pos)
	r.buf, r.pos = r.arr[:cnt], pos
	if cnt < n {
		return r.arr[:cnt], err
	}
	return r.arr[:n], nil
}

func (r *stringReader) Discard(n int) (int, error) {
	var err error
	if n > r.Len() {
		n, err = r.Len(), io.EOF
	}
	r.Seek(int64(n), os.SEEK_CUR)
	return n, err
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package store

import (
	"fmt"
	"regexp"
	"sort"
	"strconv"
	"strings"

	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/validation"

	"k8s.io/kube-state-metrics/v2/pkg/metric"
	"k8s.io/kube-state-metrics/v2/pkg/options"
)

var (
	invalidLabelCharRE = regexp.MustCompile(`[^a-zA-Z0-9_]`)
	matchAllCap        = regexp.MustCompile("([a-z0-9])([A-Z])")
	conditionStatuses  = []v1.ConditionStatus{v1.ConditionTrue, v1.ConditionFalse, v1.ConditionUnknown}
)

func resourceVersionMetric(rv string) []*metric.Metric {
	v, err := strconv.ParseFloat(rv, 64)
	if err != nil {
		return []*metric.Metric{}
	}

	return []*metric.Metric{
		{
			Value: v,
		},
	}

}

func boolFloat64(b bool) float64 {
	if b {
		return 1
	}
	return 0
}

// addConditionMetrics generates one metric for each possible condition
// status. For this function to work properly, the last label in the metric
// description must be the condition.
func addConditionMetrics(cs v1.ConditionStatus) []*metric.Metric {
	ms := make([]*metric.Metric, len(conditionStatuses))

	for i, status := range conditionStatuses {
		ms[i] = &metric.Metric{
			LabelValues: []string{strings.ToLower(string(status))},
			Value:       boolFloat64(cs == status),
		}
	}

	return ms
}

func kubeMapToPrometheusLabels(prefix string, input map[string]string) ([]string, []string) {
	return mapToPrometheusLabels(input, prefix)
}

func mapToPrometheusLabels(labels map[string]string, prefix string) ([]string, []string) {
	labelKeys := make([]string, 0, len(labels))
	labelValues := make([]string, 0, len(labels))

	sortedKeys := make([]string, 0)
	for key := range labels {
		sortedKeys = append(sortedKeys, key)
	}
	sort.Strings(sortedKeys)

	// conflictDesc holds some metadata for resolving potential label conflicts
	type conflictDesc struct {
		// the number of conflicting label keys we saw so far
		count int

		// the offset of the initial conflicting label key, so we could
		// later go back and rename "label_foo" to "label_foo_conflict1"
		initial int
	}

	conflicts := make(map[string]*conflictDesc)
	for _, k := range sortedKeys {
		labelKey := labelName(prefix, k)
		if conflict, ok := conflicts[labelKey]; ok {
			if conflict.count == 1 {
				// this is the first conflict for the label,
				// so we have to go back and rename the initial label that we've already added
				labelKeys[conflict.initial] = labelConflictSuffix(labelKeys[conflict.initial], conflict.count)
			}

			conflict.count++
			labelKey = labelConflictSuffix(labelKey, conflict.count)
		} else {
			// we'll need this info later in case there are conflicts
			conflicts[labelKey] = &conflictDesc{
				count:   1,
				initial: len(labelKeys),
			}
		}
		labelKeys = append(labelKeys, labelKey)
		labelValues = append(labelValues, labels[k])
	}
	return labelKeys, labelValues
}

func labelName(prefix, labelName string) string {
	return prefix + "_" + lintLabelName(sanitizeLabelName(labelName))
}

func sanitizeLabelName(s string) string {
	return invalidLabelCharRE.ReplaceAllString(s, "_")
}

func lintLabelName(s string) string {
	return toSnakeCase(s)
}

func toSnakeCase(s string) string {
	snake := matchAllCap.ReplaceAllString(s, "${1}_${2}")
	return strings.ToLower(snake)
}

func labelConflictSuffix(label string, count int) string {
	return fmt.Sprintf("%s_conflict%d", label, count)
}

func isHugePageResourceName(name v1.ResourceName) bool {
	return strings.HasPrefix(string(name), v1.ResourceHugePagesPrefix)
}

func isAttachableVolumeResourceName(name v1.ResourceName) bool {
	return strings.HasPrefix(string(name), v1.ResourceAttachableVolumesPrefix)
}

func isExtendedResourceName(name v1.ResourceName) bool {
	if isNativeResource(name) || strings.HasPrefix(string(name), v1.DefaultResourceRequestsPrefix) {
		return false
	}
	// Ensure it satisfies the rules in IsQualifiedName() after converted into quota resource name
	nameForQuota := fmt.Sprintf("%s%s", v1.DefaultResourceRequestsPrefix, string(name))
	if errs := validation.IsQualifiedName(nameForQuota); len(errs) != 0 {
		return false
	}
	return true
}

func isNativeResource(name v1.ResourceName) bool {
	return !strings.Contains(string(name), "/") ||
		isPrefixedNativeResource(name)
}

func isPrefixedNativeResource(name v1.ResourceName) bool {
	return strings.Contains(string(name), v1.ResourceDefaultNamespacePrefix)
}

// createPrometheusLabelKeysValues takes in passed kubernetes annotations/labels
// and associated allowed list in kubernetes label format.
// It returns only those allowed annotations/labels that exist in the list and converts them to Prometheus labels.
func createPrometheusLabelKeysValues(prefix string, allKubeData map[string]string, allowList []string) ([]string, []string) {
	allowedKubeData := make(map[string]string)

	if len(allowList) > 0 {
		if allowList[0] == options.LabelWildcard {
			return kubeMapToPrometheusLabels(prefix, allKubeData)
		}

		for _, l := range allowList {
			v, found := allKubeData[l]
			if found {
				allowedKubeData[l] = v
			}
		}
	}
	return kubeMapToPrometheusLabels(prefix, allowedKubeData)
}

// mergeKeyValues merges label keys and values slice pairs into a single slice pair.
// Arguments are passed as equal-length pairs of slices, where the first slice contains keys and second contains values.
// Example: mergeKeyValues(keys1, values1, keys2, values2) => (keys1+keys2, values1+values2)
func mergeKeyValues(keyValues ...[]string) (keys, values []string) {

	capacity := 0
	for i := 0; i < len(keyValues); i += 2 {
		capacity += len(keyValues[i])
	}

	// Allocate one contiguous block, then split it up to keys and values zero'd slices.
	keysValues := make([]string, 0, capacity*2)
	keys = (keysValues[0:capacity:capacity])[:0]
	values = (keysValues[capacity : capacity*2])[:0]

	for i := 0; i < len(keyValues); i += 2 {
		keys = append(keys, keyValues[i]...)
		values = append(values, keyValues[i+1]...)
	}

	return keys, values
}
<commit_msg>Update internal/store/utils.go<commit_after>/*
Copyright 2018 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package store

import (
	"fmt"
	"regexp"
	"sort"
	"strconv"
	"strings"

	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/validation"

	"k8s.io/kube-state-metrics/v2/pkg/metric"
	"k8s.io/kube-state-metrics/v2/pkg/options"
)

var (
	invalidLabelCharRE = regexp.MustCompile(`[^a-zA-Z0-9_]`)
	matchAllCap        = regexp.MustCompile("([a-z0-9])([A-Z])")
	conditionStatuses  = []v1.ConditionStatus{v1.ConditionTrue, v1.ConditionFalse, v1.ConditionUnknown}
)

func resourceVersionMetric(rv string) []*metric.Metric {
	v, err := strconv.ParseFloat(rv, 64)
	if err != nil {
		return []*metric.Metric{}
	}

	return []*metric.Metric{
		{
			Value: v,
		},
	}

}

func boolFloat64(b bool) float64 {
	if b {
		return 1
	}
	return 0
}

// addConditionMetrics generates one metric for each possible condition
// status. For this function to work properly, the last label in the metric
// description must be the condition.
func addConditionMetrics(cs v1.ConditionStatus) []*metric.Metric {
	ms := make([]*metric.Metric, len(conditionStatuses))

	for i, status := range conditionStatuses {
		ms[i] = &metric.Metric{
			LabelValues: []string{strings.ToLower(string(status))},
			Value:       boolFloat64(cs == status),
		}
	}

	return ms
}

func kubeMapToPrometheusLabels(prefix string, input map[string]string) ([]string, []string) {
	return mapToPrometheusLabels(input, prefix)
}

func mapToPrometheusLabels(labels map[string]string, prefix string) ([]string, []string) {
	labelKeys := make([]string, 0, len(labels))
	labelValues := make([]string, 0, len(labels))

	sortedKeys := make([]string, 0)
	for key := range labels {
		sortedKeys = append(sortedKeys, key)
	}
	sort.Strings(sortedKeys)

	// conflictDesc holds some metadata for resolving potential label conflicts
	type conflictDesc struct {
		// the number of conflicting label keys we saw so far
		count int

		// the offset of the initial conflicting label key, so we could
		// later go back and rename "label_foo" to "label_foo_conflict1"
		initial int
	}

	conflicts := make(map[string]*conflictDesc)
	for _, k := range sortedKeys {
		labelKey := labelName(prefix, k)
		if conflict, ok := conflicts[labelKey]; ok {
			if conflict.count == 1 {
				// this is the first conflict for the label,
				// so we have to go back and rename the initial label that we've already added
				labelKeys[conflict.initial] = labelConflictSuffix(labelKeys[conflict.initial], conflict.count)
			}

			conflict.count++
			labelKey = labelConflictSuffix(labelKey, conflict.count)
		} else {
			// we'll need this info later in case there are conflicts
			conflicts[labelKey] = &conflictDesc{
				count:   1,
				initial: len(labelKeys),
			}
		}
		labelKeys = append(labelKeys, labelKey)
		labelValues = append(labelValues, labels[k])
	}
	return labelKeys, labelValues
}

func labelName(prefix, labelName string) string {
	return prefix + "_" + lintLabelName(sanitizeLabelName(labelName))
}

func sanitizeLabelName(s string) string {
	return invalidLabelCharRE.ReplaceAllString(s, "_")
}

func lintLabelName(s string) string {
	return toSnakeCase(s)
}

func toSnakeCase(s string) string {
	snake := matchAllCap.ReplaceAllString(s, "${1}_${2}")
	return strings.ToLower(snake)
}

func labelConflictSuffix(label string, count int) string {
	return fmt.Sprintf("%s_conflict%d", label, count)
}

func isHugePageResourceName(name v1.ResourceName) bool {
	return strings.HasPrefix(string(name), v1.ResourceHugePagesPrefix)
}

func isAttachableVolumeResourceName(name v1.ResourceName) bool {
	return strings.HasPrefix(string(name), v1.ResourceAttachableVolumesPrefix)
}

func isExtendedResourceName(name v1.ResourceName) bool {
	if isNativeResource(name) || strings.HasPrefix(string(name), v1.DefaultResourceRequestsPrefix) {
		return false
	}
	// Ensure it satisfies the rules in IsQualifiedName() after converted into quota resource name
	nameForQuota := fmt.Sprintf("%s%s", v1.DefaultResourceRequestsPrefix, string(name))
	if errs := validation.IsQualifiedName(nameForQuota); len(errs) != 0 {
		return false
	}
	return true
}

func isNativeResource(name v1.ResourceName) bool {
	return !strings.Contains(string(name), "/") ||
		isPrefixedNativeResource(name)
}

func isPrefixedNativeResource(name v1.ResourceName) bool {
	return strings.Contains(string(name), v1.ResourceDefaultNamespacePrefix)
}

// createPrometheusLabelKeysValues takes in passed kubernetes annotations/labels
// and associated allowed list in kubernetes label format.
// It returns only those allowed annotations/labels that exist in the list and converts them to Prometheus labels.
func createPrometheusLabelKeysValues(prefix string, allKubeData map[string]string, allowList []string) ([]string, []string) {
	allowedKubeData := make(map[string]string)

	if len(allowList) > 0 {
		if allowList[0] == options.LabelWildcard {
			return kubeMapToPrometheusLabels(prefix, allKubeData)
		}

		for _, l := range allowList {
			v, found := allKubeData[l]
			if found {
				allowedKubeData[l] = v
			}
		}
	}
	return kubeMapToPrometheusLabels(prefix, allowedKubeData)
}

// mergeKeyValues merges label keys and values slice pairs into a single slice pair.
// Arguments are passed as equal-length pairs of slices, where the first slice contains keys and second contains values.
// Example: mergeKeyValues(keys1, values1, keys2, values2) => (keys1+keys2, values1+values2)
func mergeKeyValues(keyValues ...[]string) (keys, values []string) {
	capacity := 0
	for i := 0; i < len(keyValues); i += 2 {
		capacity += len(keyValues[i])
	}

	// Allocate one contiguous block, then split it up to keys and values zero'd slices.
	keysValues := make([]string, 0, capacity*2)
	keys = (keysValues[0:capacity:capacity])[:0]
	values = (keysValues[capacity : capacity*2])[:0]

	for i := 0; i < len(keyValues); i += 2 {
		keys = append(keys, keyValues[i]...)
		values = append(values, keyValues[i+1]...)
	}

	return keys, values
}
<|endoftext|>


<commit_before>package tensor3

type Vectors []Vector

func NewVectors(cs ...BaseType) (vs Vectors) {
	vs = make(Vectors, (len(cs)+2)/3)
	for i := range vs {
		vs[i] = NewVector(cs[i*3:]...)
	}
	return
}

func (vs Vectors) Index(vr *Vector) (i uint) {
	for j := range vs {
		if &vs[j] == vr {
			return uint(j+1)
		}
	}
	return 0
}

func (vs Vectors) Cross(v Vector) {
	vs.ForEach((*Vector).Cross, v)
}

func (vs Vectors) Add(v Vector) {
	vs.ForEach((*Vector).Add, v)
}

func (vs Vectors) Subtract(v Vector) {
	vs.ForEach((*Vector).Subtract, v)
}

func (vs Vectors) Project(v Vector) {
	vs.ForEach((*Vector).Project, v)
}

func (vs Vectors) Sum() (v Vector) {
	v.ForAll(vs, (*Vector).Add)
	return
}

func (vs Vectors) Multiply(s BaseType) {
	var multiply func(*Vector)
	multiply = func(v *Vector) {
		v.Multiply(s)
	}
	vs.ForEachNoParameter(multiply)
}

func (vs Vectors) Max() (v Vector) {
	v.Set(vs[0])
	v.ForAll(vs[1:], (*Vector).Max)
	return
}

func (vs Vectors) Min() (v Vector) {
	v.Set(vs[0])
	v.ForAll(vs[1:], (*Vector).Min)
	return
}

func (vs Vectors) Middle() (v Vector) {
	v = vs.Max()
	v.Mid(vs.Min())
	return
}

func (vs Vectors) Interpolate(v Vector, f float64) {
	f1 := Base64(1 - f)
	f2 := Base64(f)
	var interpolate func(*Vector, Vector)
	interpolate = func(v *Vector, v2 Vector) {
		v2.Multiply(f1)
		v.Multiply(f2)
		v.Add(v2)
	}
	vs.ForEach(interpolate, v)
}

func (vs Vectors) ForEach(fn func(*Vector, Vector), v Vector) {
	if !Parallel {
		vectorsApply(vs, fn, v)
	} else {
		vectorsApplyChunked(vs, fn, v)
	}
}

func vectorsApply(vs Vectors, fn func(*Vector, Vector), v Vector) {
	for i := range vs {
		fn(&vs[i], v)
	}
}

func vectorsApplyChunked(vs Vectors, fn func(*Vector, Vector), v Vector) {
	done := make(chan struct{}, 1)
	var running uint
	for chunk := range vectorsInChunks(vs, chunkSize(len(vs))) {
		running++
		go func(c Vectors) {
			vectorsApply(c, fn, v)
			done <- struct{}{}
		}(chunk)
	}
	for ; running > 0; running-- {
		<-done
	}
}

// for each vector apply a function with no parameters
func (vs Vectors) ForEachNoParameter(fn func(*Vector)) {
	var inner func(*Vector, Vector)
	inner = func(v *Vector, _ Vector) {
		fn(v)
	}
	vs.ForEach(inner, Vector{})
}

func (vs Vectors) CrossAll(vs2 Vectors) {
	vs.ForAll((*Vector).Cross, vs2)
}

func (vs Vectors) AddAll(vs2 Vectors) {
	vs.ForAll((*Vector).Add, vs2)
}

func (vs Vectors) SubtractAll(vs2 Vectors) {
	vs.ForAll((*Vector).Subtract, vs2)
}

func (vs Vectors) ProjectAll(vs2 Vectors) {
	vs.ForAll((*Vector).Project, vs2)
}

func (vs Vectors) ForAll(fn func(*Vector, Vector), vs2 Vectors) {
	if !Parallel {
		vectorsApplyAll(vs, fn, vs2)
	} else {
		vectorsApplyAllChunked(vs, fn, vs2)
	}
}

func vectorsApplyAll(vs Vectors, fn func(*Vector, Vector), vs2 Vectors) {
	for i := range vs {
		fn(&vs[i], vs2[i])
	}
}

func vectorsApplyAllChunked(vs Vectors, fn func(*Vector, Vector), vs2 Vectors) {
	done := make(chan struct{}, 1)
	var running uint
	// shorten vs to use only what we have in vs2
	if len(vs) > len(vs2) {
		vs = vs[:len(vs2)]
	}
	cs := chunkSize(len(vs))
	chunks2 := vectorsInChunks(vs2, cs)
	for chunk := range vectorsInChunks(vs, cs) {
		running++
		go func(c Vectors) {
			vectorsApplyAll(c, fn, <-chunks2)
			done <- struct{}{}
		}(chunk)
	}
	for ; running > 0; running-- {
		<-done
	}
}

// search Vectors for the two Vector's that return the minimum value from the provided function
func (vs Vectors) SearchMin(toMin func(Vector, Vector) BaseType) (i, j int, value BaseType) {
	// TODO search in chunks
	j=1
	value = toMin(vs[0], vs[1])
	var v1, v2 Vector
	var il, jl int
	for jl, v2 = range vs[2:] {
		nl := toMin(vs[0], v2)
		if nl < value {
			value, j = nl, jl+2
		}
	}

	for il, v1 = range vs[1:] {
		for jl, v2 = range vs[il+2:] {
			nl := toMin(v1, v2)
			if nl < value {
				value, i, j = nl, il+1, jl+il+2
			}
		}
	}
	return
}

func (vs Vectors) SearchMinRegionally(toMin func(Vector, Vector) BaseType) (i, j *Vector) {
	splitPoint:=vs.Middle()
	var notFirst bool
	for vrss:= range vectorsSplitRegionally(vs, splitPoint) {
		if len(vrss)<2 {continue} // rare single point still needs so be checked with 3 region edge points
		// TODO use go routine and channel for results.
		 is,js,vs:=vrss.SearchMin(toMin)
		 if !notFirst || vs<toMin(*vrss[is],*vrss[js]) {
		 	i,j=vrss[is],vrss[js]
		 	notFirst=true
		 }
		 // TODO region perimeter crossing match, if either match point is more min with any of its 3 region edge projected points, then another SearchMin, with the appropriate vrss, is needed.
	}
	return
}
/*
func (b *BaseType) Aggregate(vs Vectors,length,stride int, fn func(*BaseType, Vectors)) {
	for vsc := range vectorSlicesInChunks(vs,chunkSize(len(vs)),length,stride,false) {
		for _,vs:=range(vsc){
			fn(b, vs)
		}
	}
}
*/

// for each vector apply a function with no parameters
func (vs Vectors) ForEachInSlices(length,stride int,wrap bool,fn func(Vectors)) {
	if !Parallel {
		var i int
		for ;i<len(vs)-length+1;i+=stride{
			fn(vs[i:i+length])
		}
		if wrap{
			joinSlice:=make(Vectors,length,length)
			for ;i<len(vs);i+=stride{
				copy(joinSlice,vs[i:])
				copy(joinSlice[len(vs)-i:],vs)
				fn(joinSlice)
			}
		}
	} else {
		vectorsInSlicesApplyChunked(vs,length,stride,wrap, fn)
	}
}

func vectorsInSlicesApply(vss []Vectors,fn func(Vectors)) {
	for _,vs := range vss {
		fn(vs)
	}
}

func vectorsInSlicesApplyChunked(vs Vectors,length,stride int,wrap bool, fn func(Vectors)) {
	done := make(chan struct{}, 1)
	var running uint
	for chunk := range vectorSlicesInChunks(vs, chunkSize(len(vs)),length,stride,wrap) {
		running++
		go func(c []Vectors) {
			vectorsInSlicesApply(c, fn)
			done <- struct{}{}
		}(chunk)
	}
	for ; running > 0; running-- {
		<-done
	}
}

// return a VectorRefs referencing those Vector's that return true from the provided function.
func (vs Vectors) Select(fn func(*Vector)bool) (svs VectorRefs) {
	for i := range vs {
		if fn(&vs[i]){
			svs=append(svs,&vs[i])
		}
	}
	return
}

// return a VectorRefs referecing in Vectors those that are at equal spaced strides.
func (vs Vectors) Stride(s uint) (svs VectorRefs) {
	if s==0 {return}
	is:=int(s)
	svs=make(VectorRefs,len(vs)/is+1)
	for i:= range(svs) {
		svs[i]=&vs[i*is]
	}
	return
}

// return a slice of VectorRefs each referencing those Vector's that returned the slices index value from the provided function.
// or put another way;
// bin Vector's by the functions returned value.
// bins start at 1, a function returning a value of 0 causes the VecRef not to be in any of the returned bins.
func (vs Vectors) Split(fn func(Vector)uint) (ssvs []VectorRefs) {
	for i := range vs {
		ind:= fn(vs[i])
		if ind>0 {
			// pad, if needed, with a series of new VectorRefs to fill up to index. (max index not preknown)
			if ind > uint(len(ssvs)){
				ssvs=append(ssvs,make([]VectorRefs,ind-uint(len(ssvs)))...)
			}
			ssvs[ind-1]=append(ssvs[ind-1],&vs[i])
		}
	}
	return
}

<commit_msg>comment<commit_after>package tensor3

type Vectors []Vector

func NewVectors(cs ...BaseType) (vs Vectors) {
	vs = make(Vectors, (len(cs)+2)/3)
	for i := range vs {
		vs[i] = NewVector(cs[i*3:]...)
	}
	return
}

func (vs Vectors) Index(vr *Vector) (i uint) {
	for j := range vs {
		if &vs[j] == vr {
			return uint(j+1)
		}
	}
	return 0
}

func (vs Vectors) Cross(v Vector) {
	vs.ForEach((*Vector).Cross, v)
}

func (vs Vectors) Add(v Vector) {
	vs.ForEach((*Vector).Add, v)
}

func (vs Vectors) Subtract(v Vector) {
	vs.ForEach((*Vector).Subtract, v)
}

func (vs Vectors) Project(v Vector) {
	vs.ForEach((*Vector).Project, v)
}

func (vs Vectors) Sum() (v Vector) {
	v.ForAll(vs, (*Vector).Add)
	return
}

func (vs Vectors) Multiply(s BaseType) {
	var multiply func(*Vector)
	multiply = func(v *Vector) {
		v.Multiply(s)
	}
	vs.ForEachNoParameter(multiply)
}

func (vs Vectors) Max() (v Vector) {
	v.Set(vs[0])
	v.ForAll(vs[1:], (*Vector).Max)
	return
}

func (vs Vectors) Min() (v Vector) {
	v.Set(vs[0])
	v.ForAll(vs[1:], (*Vector).Min)
	return
}

func (vs Vectors) Middle() (v Vector) {
	v = vs.Max()
	v.Mid(vs.Min())
	return
}

func (vs Vectors) Interpolate(v Vector, f float64) {
	f1 := Base64(1 - f)
	f2 := Base64(f)
	var interpolate func(*Vector, Vector)
	interpolate = func(v *Vector, v2 Vector) {
		v2.Multiply(f1)
		v.Multiply(f2)
		v.Add(v2)
	}
	vs.ForEach(interpolate, v)
}

func (vs Vectors) ForEach(fn func(*Vector, Vector), v Vector) {
	if !Parallel {
		vectorsApply(vs, fn, v)
	} else {
		vectorsApplyChunked(vs, fn, v)
	}
}

func vectorsApply(vs Vectors, fn func(*Vector, Vector), v Vector) {
	for i := range vs {
		fn(&vs[i], v)
	}
}

func vectorsApplyChunked(vs Vectors, fn func(*Vector, Vector), v Vector) {
	done := make(chan struct{}, 1)
	var running uint
	for chunk := range vectorsInChunks(vs, chunkSize(len(vs))) {
		running++
		go func(c Vectors) {
			vectorsApply(c, fn, v)
			done <- struct{}{}
		}(chunk)
	}
	for ; running > 0; running-- {
		<-done
	}
}

// for each vector apply a function with no parameters
func (vs Vectors) ForEachNoParameter(fn func(*Vector)) {
	var inner func(*Vector, Vector)
	inner = func(v *Vector, _ Vector) {
		fn(v)
	}
	vs.ForEach(inner, Vector{})
}

func (vs Vectors) CrossAll(vs2 Vectors) {
	vs.ForAll((*Vector).Cross, vs2)
}

func (vs Vectors) AddAll(vs2 Vectors) {
	vs.ForAll((*Vector).Add, vs2)
}

func (vs Vectors) SubtractAll(vs2 Vectors) {
	vs.ForAll((*Vector).Subtract, vs2)
}

func (vs Vectors) ProjectAll(vs2 Vectors) {
	vs.ForAll((*Vector).Project, vs2)
}

func (vs Vectors) ForAll(fn func(*Vector, Vector), vs2 Vectors) {
	if !Parallel {
		vectorsApplyAll(vs, fn, vs2)
	} else {
		vectorsApplyAllChunked(vs, fn, vs2)
	}
}

func vectorsApplyAll(vs Vectors, fn func(*Vector, Vector), vs2 Vectors) {
	for i := range vs {
		fn(&vs[i], vs2[i])
	}
}

func vectorsApplyAllChunked(vs Vectors, fn func(*Vector, Vector), vs2 Vectors) {
	done := make(chan struct{}, 1)
	var running uint
	// shorten vs to use only what we have in vs2
	if len(vs) > len(vs2) {
		vs = vs[:len(vs2)]
	}
	cs := chunkSize(len(vs))
	chunks2 := vectorsInChunks(vs2, cs)
	for chunk := range vectorsInChunks(vs, cs) {
		running++
		go func(c Vectors) {
			vectorsApplyAll(c, fn, <-chunks2)
			done <- struct{}{}
		}(chunk)
	}
	for ; running > 0; running-- {
		<-done
	}
}

// search Vectors for the two Vector's that return the minimum value from the provided function
func (vs Vectors) SearchMin(toMin func(Vector, Vector) BaseType) (i, j int, value BaseType) {
	// TODO search in chunks
	j=1
	value = toMin(vs[0], vs[1])
	var v1, v2 Vector
	var il, jl int
	for jl, v2 = range vs[2:] {
		nl := toMin(vs[0], v2)
		if nl < value {
			value, j = nl, jl+2
		}
	}

	for il, v1 = range vs[1:] {
		for jl, v2 = range vs[il+2:] {
			nl := toMin(v1, v2)
			if nl < value {
				value, i, j = nl, il+1, jl+il+2
			}
		}
	}
	return
}

func (vs Vectors) SearchMinRegionally(toMin func(Vector, Vector) BaseType) (i, j *Vector) {
	splitPoint:=vs.Middle()
	var notFirst bool
	for vrss:= range vectorsSplitRegionally(vs, splitPoint) {
		if len(vrss)<2 {continue} // rare, when used usefully, single point still needs so be checked with 3 region edge points
		// TODO use go routine and channel for results.
		 is,js,vs:=vrss.SearchMin(toMin)
		 if !notFirst || vs<toMin(*vrss[is],*vrss[js]) {
		 	i,j=vrss[is],vrss[js]
		 	notFirst=true
		 }
		 // TODO region perimeter crossing match, if either match point is more min with any of its 3 region edge projected points, then another SearchMin, with the appropriate vrss, is needed.
	}
	return
}
/*
func (b *BaseType) Aggregate(vs Vectors,length,stride int, fn func(*BaseType, Vectors)) {
	for vsc := range vectorSlicesInChunks(vs,chunkSize(len(vs)),length,stride,false) {
		for _,vs:=range(vsc){
			fn(b, vs)
		}
	}
}
*/

// for each vector apply a function with no parameters
func (vs Vectors) ForEachInSlices(length,stride int,wrap bool,fn func(Vectors)) {
	if !Parallel {
		var i int
		for ;i<len(vs)-length+1;i+=stride{
			fn(vs[i:i+length])
		}
		if wrap{
			joinSlice:=make(Vectors,length,length)
			for ;i<len(vs);i+=stride{
				copy(joinSlice,vs[i:])
				copy(joinSlice[len(vs)-i:],vs)
				fn(joinSlice)
			}
		}
	} else {
		vectorsInSlicesApplyChunked(vs,length,stride,wrap, fn)
	}
}

func vectorsInSlicesApply(vss []Vectors,fn func(Vectors)) {
	for _,vs := range vss {
		fn(vs)
	}
}

func vectorsInSlicesApplyChunked(vs Vectors,length,stride int,wrap bool, fn func(Vectors)) {
	done := make(chan struct{}, 1)
	var running uint
	for chunk := range vectorSlicesInChunks(vs, chunkSize(len(vs)),length,stride,wrap) {
		running++
		go func(c []Vectors) {
			vectorsInSlicesApply(c, fn)
			done <- struct{}{}
		}(chunk)
	}
	for ; running > 0; running-- {
		<-done
	}
}

// return a VectorRefs referencing those Vector's that return true from the provided function.
func (vs Vectors) Select(fn func(*Vector)bool) (svs VectorRefs) {
	for i := range vs {
		if fn(&vs[i]){
			svs=append(svs,&vs[i])
		}
	}
	return
}

// return a VectorRefs referecing in Vectors those that are at equal spaced strides.
func (vs Vectors) Stride(s uint) (svs VectorRefs) {
	if s==0 {return}
	is:=int(s)
	svs=make(VectorRefs,len(vs)/is+1)
	for i:= range(svs) {
		svs[i]=&vs[i*is]
	}
	return
}

// return a slice of VectorRefs each referencing those Vector's that returned the slices index value from the provided function.
// or put another way;
// bin Vector's by the functions returned value.
// bins start at 1, a function returning a value of 0 causes the VecRef not to be in any of the returned bins.
func (vs Vectors) Split(fn func(Vector)uint) (ssvs []VectorRefs) {
	for i := range vs {
		ind:= fn(vs[i])
		if ind>0 {
			// pad, if needed, with a series of new VectorRefs to fill up to index. (max index not preknown)
			if ind > uint(len(ssvs)){
				ssvs=append(ssvs,make([]VectorRefs,ind-uint(len(ssvs)))...)
			}
			ssvs[ind-1]=append(ssvs[ind-1],&vs[i])
		}
	}
	return
}

<|endoftext|>


<commit_before>package proxy

import (
	"bufio"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/getlantern/errors"
)

// HTTP returns a Handler that processes plain-text HTTP requests.
//
// discardFirstRequest: if true, causes the first request to the handler to be
// discarded
//
// idleTimeout: if specified, lets us know to include an appropriate
// KeepAlive: timeout header in the CONNECT response
//
// onRequest: if specified, is called on every read request
//
// onResponse: if specified, is called on every read response
//
// onError: if specified, if there's an error making a round trip, this function
// can return a response to be presented to the client. If the function returns
// no response, nothing is written to the client.
//
// dial: the function that's used to dial upstream
func HTTP(
	discardFirstRequest bool,
	idleTimeout time.Duration,
	onRequest func(req *http.Request) *http.Request,
	onResponse func(resp *http.Response) *http.Response,
	onError func(req *http.Request, err error) *http.Response,
	dial DialFunc,
) Interceptor {
	// Apply defaults
	if onRequest == nil {
		onRequest = defaultOnRequest
	}
	if onResponse == nil {
		onResponse = defaultOnResponse
	}
	if onError == nil {
		onError = defaultOnError
	}
	if idleTimeout > 0 {
		origOnResponse := onResponse
		onResponse = func(resp *http.Response) *http.Response {
			resp = origOnResponse(resp)
			addIdleKeepAlive(resp.Header, idleTimeout)
			return resp
		}
	}

	ic := &httpInterceptor{
		discardFirstRequest: discardFirstRequest,
		idleTimeout:         idleTimeout,
		onRequest:           onRequest,
		onResponse:          onResponse,
		onError:             onError,
		dial:                dial,
	}
	return ic.intercept
}

type httpInterceptor struct {
	discardFirstRequest bool
	idleTimeout         time.Duration
	onRequest           func(req *http.Request) *http.Request
	onResponse          func(resp *http.Response) *http.Response
	onError             func(req *http.Request, err error) *http.Response
	dial                DialFunc
}

func (ic *httpInterceptor) intercept(w http.ResponseWriter, req *http.Request) error {
	var downstream net.Conn
	var downstreamBuffered *bufio.ReadWriter
	tr := &http.Transport{
		Dial:                ic.dial,
		IdleConnTimeout:     ic.idleTimeout,
		MaxIdleConnsPerHost: 8, // just to keep clients from getting too greedy
	}
	var err error

	closeDownstream := false
	defer func() {
		if closeDownstream {
			if closeErr := downstream.Close(); closeErr != nil {
				log.Tracef("Error closing downstream connection: %s", closeErr)
			}
		}
		tr.CloseIdleConnections()
	}()

	// Hijack underlying connection.
	downstream, downstreamBuffered, err = w.(http.Hijacker).Hijack()
	if err != nil {
		fullErr := errors.New("Unable to hijack connection: %s", err)
		respondBadGateway(w, fullErr)
		return fullErr
	}
	closeDownstream = true

	return ic.processRequests(req.RemoteAddr, req, downstream, downstreamBuffered, tr)
}

func (ic *httpInterceptor) processRequests(remoteAddr string, req *http.Request, downstream net.Conn, downstreamBuffered *bufio.ReadWriter, tr *http.Transport) error {
	var readErr error

	first := true
	for {
		discardRequest := first && ic.discardFirstRequest
		if discardRequest {
			err := ic.onRequest(req).Write(ioutil.Discard)
			if err != nil {
				return errors.New("Error discarding first request: %v", err)
			}
		} else {
			resp, err := tr.RoundTrip(prepareRequest(ic.onRequest(req)))
			if err != nil {
				errResp := ic.onError(req, err)
				if errResp != nil {
					errResp.Request = req
					ic.writeResponse(downstream, errResp)
				}
				return err
			}
			writeErr := ic.writeResponse(downstream, resp)
			if writeErr != nil {
				if isUnexpected(writeErr) {
					return errors.New("Unable to write response to downstream: %v", writeErr)
				}
				// Error is not unexpected, but we're done
				return nil
			}
		}

		if req.Close {
			// Client signaled that they would close the connection after this
			// request, finish
			return nil
		}

		req, readErr = http.ReadRequest(downstreamBuffered.Reader)
		if readErr != nil {
			if isUnexpected(readErr) {
				return errors.New("Unable to read next request from downstream: %v", readErr)
			}
			return nil
		}

		// Preserve remote address from original request
		req.RemoteAddr = remoteAddr
		first = false
	}
}

func (ic *httpInterceptor) writeResponse(downstream io.Writer, resp *http.Response) error {
	out := downstream
	belowHTTP11 := !resp.Request.ProtoAtLeast(1, 1)
	if belowHTTP11 && resp.StatusCode < 200 {
		// HTTP 1.0 doesn't define status codes below 200, discard response
		// see http://coad.measurement-factory.com/cgi-bin/coad/SpecCgi?spec_id=rfc2616#excerpt/rfc2616/859a092cb26bde76c25284196171c94d
		out = ioutil.Discard
	} else {
		resp = ic.onResponse(prepareResponse(resp, belowHTTP11))
	}
	return resp.Write(out)
}

// prepareRequest prepares the request in line with the HTTP spec for proxies.
func prepareRequest(req *http.Request) *http.Request {
	outReq := new(http.Request)
	// Beware, this will make a shallow copy. We have to copy all maps
	*outReq = *req

	outReq.Proto = "HTTP/1.1"
	outReq.ProtoMajor = 1
	outReq.ProtoMinor = 1
	// Overwrite close flag: keep persistent connection for the backend servers
	outReq.Close = false

	// Request Header
	outReq.Header = make(http.Header)
	copyHeadersForForwarding(outReq.Header, req.Header)
	// Ensure we have a HOST header (important for Go 1.6+ because http.Server
	// strips the HOST header from the inbound request)
	outReq.Header.Set("Host", req.Host)

	// Request URL
	outReq.URL = cloneURL(req.URL)
	// We know that is going to be HTTP always because HTTPS isn't forwarded.
	// We need to hardcode it here because req.URL.Scheme can be undefined, since
	// client request don't need to use absolute URIs
	outReq.URL.Scheme = "http"
	// We need to make sure the host is defined in the URL (not the actual URI)
	outReq.URL.Host = req.Host
	outReq.URL.RawQuery = req.URL.RawQuery
	outReq.Body = req.Body

	userAgent := req.UserAgent()
	if userAgent == "" {
		outReq.Header.Del("User-Agent")
	} else {
		outReq.Header.Set("User-Agent", userAgent)
	}

	return outReq
}

// prepareResponse prepares the response in line with the HTTP spec
func prepareResponse(resp *http.Response, belowHTTP11 bool) *http.Response {
	origHeader := resp.Header
	resp.Header = make(http.Header)
	copyHeadersForForwarding(resp.Header, origHeader)
	// Below added due to CoAdvisor test failure
	if resp.Header.Get("Date") == "" {
		resp.Header.Set("Date", time.Now().Format(time.RFC850))
	}
	if belowHTTP11 {
		// Also, make sure we're not sending chunked transfer encoding to 1.0 clients
		resp.TransferEncoding = nil
	}
	return resp
}

// cloneURL provides update safe copy by avoiding shallow copying User field
func cloneURL(i *url.URL) *url.URL {
	out := *i
	if i.User != nil {
		out.User = &(*i.User)
	}
	return &out
}

// copyHeadersForForwarding will copy the headers but filter those that shouldn't be
// forwarded
func copyHeadersForForwarding(dst, src http.Header) {
	var extraHopByHopHeaders []string
	for k, vv := range src {
		switch k {
		// Skip hop-by-hop headers, ref section 13.5.1 of http://www.ietf.org/rfc/rfc2616.txt
		case "Connection":
			// section 14.10 of rfc2616
			// the slice is short typically, don't bother sort it to speed up lookup
			extraHopByHopHeaders = vv
		case "Keep-Alive":
		case "Proxy-Authenticate":
		case "Proxy-Authorization":
		case "TE":
		case "Trailers":
		case "Transfer-Encoding":
		case "Upgrade":
		default:
			if !contains(k, extraHopByHopHeaders) {
				for _, v := range vv {
					dst.Add(k, v)
				}
			}
		}
	}
}

func contains(k string, s []string) bool {
	for _, h := range s {
		if k == h {
			return true
		}
	}
	return false
}

func isUnexpected(err error) bool {
	text := err.Error()
	return !strings.HasSuffix(text, "EOF") && !strings.Contains(text, "use of closed network connection") && !strings.Contains(text, "Use of idled network connection") && !strings.Contains(text, "broken pipe")
}

func defaultOnRequest(req *http.Request) *http.Request {
	return req
}

func defaultOnResponse(resp *http.Response) *http.Response {
	return resp
}

func defaultOnError(req *http.Request, err error) *http.Response {
	return nil
}
<commit_msg>Reduced MaxIdleConnsPerHost<commit_after>package proxy

import (
	"bufio"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/getlantern/errors"
)

// HTTP returns a Handler that processes plain-text HTTP requests.
//
// discardFirstRequest: if true, causes the first request to the handler to be
// discarded
//
// idleTimeout: if specified, lets us know to include an appropriate
// KeepAlive: timeout header in the CONNECT response
//
// onRequest: if specified, is called on every read request
//
// onResponse: if specified, is called on every read response
//
// onError: if specified, if there's an error making a round trip, this function
// can return a response to be presented to the client. If the function returns
// no response, nothing is written to the client.
//
// dial: the function that's used to dial upstream
func HTTP(
	discardFirstRequest bool,
	idleTimeout time.Duration,
	onRequest func(req *http.Request) *http.Request,
	onResponse func(resp *http.Response) *http.Response,
	onError func(req *http.Request, err error) *http.Response,
	dial DialFunc,
) Interceptor {
	// Apply defaults
	if onRequest == nil {
		onRequest = defaultOnRequest
	}
	if onResponse == nil {
		onResponse = defaultOnResponse
	}
	if onError == nil {
		onError = defaultOnError
	}
	if idleTimeout > 0 {
		origOnResponse := onResponse
		onResponse = func(resp *http.Response) *http.Response {
			resp = origOnResponse(resp)
			addIdleKeepAlive(resp.Header, idleTimeout)
			return resp
		}
	}

	ic := &httpInterceptor{
		discardFirstRequest: discardFirstRequest,
		idleTimeout:         idleTimeout,
		onRequest:           onRequest,
		onResponse:          onResponse,
		onError:             onError,
		dial:                dial,
	}
	return ic.intercept
}

type httpInterceptor struct {
	discardFirstRequest bool
	idleTimeout         time.Duration
	onRequest           func(req *http.Request) *http.Request
	onResponse          func(resp *http.Response) *http.Response
	onError             func(req *http.Request, err error) *http.Response
	dial                DialFunc
}

func (ic *httpInterceptor) intercept(w http.ResponseWriter, req *http.Request) error {
	var downstream net.Conn
	var downstreamBuffered *bufio.ReadWriter
	tr := &http.Transport{
		Dial:                ic.dial,
		IdleConnTimeout:     ic.idleTimeout,
		MaxIdleConnsPerHost: 1, // since we have one transport per downstream connection, we don't need more than this
	}
	var err error

	closeDownstream := false
	defer func() {
		if closeDownstream {
			if closeErr := downstream.Close(); closeErr != nil {
				log.Tracef("Error closing downstream connection: %s", closeErr)
			}
		}
		tr.CloseIdleConnections()
	}()

	// Hijack underlying connection.
	downstream, downstreamBuffered, err = w.(http.Hijacker).Hijack()
	if err != nil {
		fullErr := errors.New("Unable to hijack connection: %s", err)
		respondBadGateway(w, fullErr)
		return fullErr
	}
	closeDownstream = true

	return ic.processRequests(req.RemoteAddr, req, downstream, downstreamBuffered, tr)
}

func (ic *httpInterceptor) processRequests(remoteAddr string, req *http.Request, downstream net.Conn, downstreamBuffered *bufio.ReadWriter, tr *http.Transport) error {
	var readErr error

	first := true
	for {
		discardRequest := first && ic.discardFirstRequest
		if discardRequest {
			err := ic.onRequest(req).Write(ioutil.Discard)
			if err != nil {
				return errors.New("Error discarding first request: %v", err)
			}
		} else {
			resp, err := tr.RoundTrip(prepareRequest(ic.onRequest(req)))
			if err != nil {
				errResp := ic.onError(req, err)
				if errResp != nil {
					errResp.Request = req
					ic.writeResponse(downstream, errResp)
				}
				return err
			}
			writeErr := ic.writeResponse(downstream, resp)
			if writeErr != nil {
				if isUnexpected(writeErr) {
					return errors.New("Unable to write response to downstream: %v", writeErr)
				}
				// Error is not unexpected, but we're done
				return nil
			}
		}

		if req.Close {
			// Client signaled that they would close the connection after this
			// request, finish
			return nil
		}

		req, readErr = http.ReadRequest(downstreamBuffered.Reader)
		if readErr != nil {
			if isUnexpected(readErr) {
				return errors.New("Unable to read next request from downstream: %v", readErr)
			}
			return nil
		}

		// Preserve remote address from original request
		req.RemoteAddr = remoteAddr
		first = false
	}
}

func (ic *httpInterceptor) writeResponse(downstream io.Writer, resp *http.Response) error {
	out := downstream
	belowHTTP11 := !resp.Request.ProtoAtLeast(1, 1)
	if belowHTTP11 && resp.StatusCode < 200 {
		// HTTP 1.0 doesn't define status codes below 200, discard response
		// see http://coad.measurement-factory.com/cgi-bin/coad/SpecCgi?spec_id=rfc2616#excerpt/rfc2616/859a092cb26bde76c25284196171c94d
		out = ioutil.Discard
	} else {
		resp = ic.onResponse(prepareResponse(resp, belowHTTP11))
	}
	return resp.Write(out)
}

// prepareRequest prepares the request in line with the HTTP spec for proxies.
func prepareRequest(req *http.Request) *http.Request {
	outReq := new(http.Request)
	// Beware, this will make a shallow copy. We have to copy all maps
	*outReq = *req

	outReq.Proto = "HTTP/1.1"
	outReq.ProtoMajor = 1
	outReq.ProtoMinor = 1
	// Overwrite close flag: keep persistent connection for the backend servers
	outReq.Close = false

	// Request Header
	outReq.Header = make(http.Header)
	copyHeadersForForwarding(outReq.Header, req.Header)
	// Ensure we have a HOST header (important for Go 1.6+ because http.Server
	// strips the HOST header from the inbound request)
	outReq.Header.Set("Host", req.Host)

	// Request URL
	outReq.URL = cloneURL(req.URL)
	// We know that is going to be HTTP always because HTTPS isn't forwarded.
	// We need to hardcode it here because req.URL.Scheme can be undefined, since
	// client request don't need to use absolute URIs
	outReq.URL.Scheme = "http"
	// We need to make sure the host is defined in the URL (not the actual URI)
	outReq.URL.Host = req.Host
	outReq.URL.RawQuery = req.URL.RawQuery
	outReq.Body = req.Body

	userAgent := req.UserAgent()
	if userAgent == "" {
		outReq.Header.Del("User-Agent")
	} else {
		outReq.Header.Set("User-Agent", userAgent)
	}

	return outReq
}

// prepareResponse prepares the response in line with the HTTP spec
func prepareResponse(resp *http.Response, belowHTTP11 bool) *http.Response {
	origHeader := resp.Header
	resp.Header = make(http.Header)
	copyHeadersForForwarding(resp.Header, origHeader)
	// Below added due to CoAdvisor test failure
	if resp.Header.Get("Date") == "" {
		resp.Header.Set("Date", time.Now().Format(time.RFC850))
	}
	if belowHTTP11 {
		// Also, make sure we're not sending chunked transfer encoding to 1.0 clients
		resp.TransferEncoding = nil
	}
	return resp
}

// cloneURL provides update safe copy by avoiding shallow copying User field
func cloneURL(i *url.URL) *url.URL {
	out := *i
	if i.User != nil {
		out.User = &(*i.User)
	}
	return &out
}

// copyHeadersForForwarding will copy the headers but filter those that shouldn't be
// forwarded
func copyHeadersForForwarding(dst, src http.Header) {
	var extraHopByHopHeaders []string
	for k, vv := range src {
		switch k {
		// Skip hop-by-hop headers, ref section 13.5.1 of http://www.ietf.org/rfc/rfc2616.txt
		case "Connection":
			// section 14.10 of rfc2616
			// the slice is short typically, don't bother sort it to speed up lookup
			extraHopByHopHeaders = vv
		case "Keep-Alive":
		case "Proxy-Authenticate":
		case "Proxy-Authorization":
		case "TE":
		case "Trailers":
		case "Transfer-Encoding":
		case "Upgrade":
		default:
			if !contains(k, extraHopByHopHeaders) {
				for _, v := range vv {
					dst.Add(k, v)
				}
			}
		}
	}
}

func contains(k string, s []string) bool {
	for _, h := range s {
		if k == h {
			return true
		}
	}
	return false
}

func isUnexpected(err error) bool {
	text := err.Error()
	return !strings.HasSuffix(text, "EOF") && !strings.Contains(text, "use of closed network connection") && !strings.Contains(text, "Use of idled network connection") && !strings.Contains(text, "broken pipe")
}

func defaultOnRequest(req *http.Request) *http.Request {
	return req
}

func defaultOnResponse(resp *http.Response) *http.Response {
	return resp
}

func defaultOnError(req *http.Request, err error) *http.Response {
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"net/url"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

var proxyTestVersion = &version{
	name: "foo",
	sequins: &sequins{
		config: sequinsConfig{
			Sharding: shardingConfig{
				ProxyTimeout:      duration{30 * time.Millisecond},
				ProxyStageTimeout: duration{10 * time.Millisecond},
			},
		},
	},
}

func readAll(t *testing.T, r io.Reader) string {
	b, err := ioutil.ReadAll(r)
	require.NoError(t, err)
	return string(b)
}

func httptestHost(s *httptest.Server) string {
	parsed, _ := url.Parse(s.URL)
	return parsed.Host
}

func TestProxySinglePeer(t *testing.T) {
	goodPeer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, "all good")
	}))

	peers := []string{httptestHost(goodPeer)}
	r, _ := http.NewRequest("GET", "http://localhost", nil)
	res, peer, err := proxyTestVersion.proxy(r, peers)

	require.NoError(t, err, "simple proxying should work")
	assert.NotNil(t, res, "simple proxying should work")

	assert.Equal(t, httptestHost(goodPeer), peer, "the returned peer should be correct")
	assert.Equal(t, "all good\n", readAll(t, res.Body))
}

func TestProxySlowPeer(t *testing.T) {
	slowPeer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		time.Sleep(100 * time.Millisecond)
		fmt.Fprintln(w, "sorry, did you need something?")
	}))

	goodPeer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, "all good")
	}))

	notReachedPeer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		require.FailNow(t, "proxying should succeed before getting to a third peer")
	}))

	peers := []string{httptestHost(slowPeer), httptestHost(goodPeer), httptestHost(notReachedPeer)}
	r, _ := http.NewRequest("GET", "http://localhost", nil)
	res, peer, err := proxyTestVersion.proxy(r, peers)
	require.NoError(t, err, "proxying should work on the second peer")
	require.NotNil(t, res, "proxying should work on the second peer")

	assert.Equal(t, httptestHost(goodPeer), peer, "the returned peer should be correct")
	assert.Equal(t, "all good\n", readAll(t, res.Body))
}

func TestProxyErrorPeer(t *testing.T) {
	errorPeer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(500)
	}))

	goodPeer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, "all good")
	}))

	notReachedPeer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		assert.Fail(t, "proxying should succeed before getting to a third peer")
	}))

	peers := []string{httptestHost(errorPeer), httptestHost(goodPeer), httptestHost(notReachedPeer)}
	r, _ := http.NewRequest("GET", "http://localhost", nil)
	res, peer, err := proxyTestVersion.proxy(r, peers)

	require.NoError(t, err, "proxying should work on the second peer")
	require.NotNil(t, res, "proxying should work on the second peer")

	assert.Equal(t, httptestHost(goodPeer), peer, "the returned peer should be correct")
	assert.Equal(t, "all good\n", readAll(t, res.Body))
}

func TestProxySlowPeerErrorPeer(t *testing.T) {
	slowPeer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		time.Sleep(15 * time.Millisecond)
		fmt.Fprintln(w, "all good, sorry to keep you waiting")
	}))

	errorPeer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(500)
	}))

	peers := []string{httptestHost(slowPeer), httptestHost(errorPeer)}
	r, _ := http.NewRequest("GET", "http://localhost", nil)
	res, peer, err := proxyTestVersion.proxy(r, peers)

	require.NoError(t, err, "proxying should work on the first peer")
	require.NotNil(t, res, "proxying should work on the first peer")

	assert.Equal(t, httptestHost(slowPeer), peer, "the returned peer should be correct")
	assert.Equal(t, "all good, sorry to keep you waiting\n", readAll(t, res.Body))
}

func TestProxyTimeout(t *testing.T) {
	slowPeer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		time.Sleep(100 * time.Millisecond)
		fmt.Fprintln(w, "sorry, did you need something?")
	}))

	notReachedPeer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		assert.Fail(t, "proxying should never try a fourth peer")
	}))

	peers := []string{
		httptestHost(slowPeer),
		httptestHost(slowPeer),
		httptestHost(slowPeer),
		httptestHost(notReachedPeer),
	}

	r, _ := http.NewRequest("GET", "http://localhost", nil)
	res, peer, err := proxyTestVersion.proxy(r, peers)

	assert.Equal(t, errProxyTimeout, err, "proxying should time out with all slow peers")
	assert.Nil(t, res, "proxying should time out with all slow peers")
	assert.Equal(t, "", peer, "peer should be empty if proxying timed out")
}

func TestProxyErrors(t *testing.T) {
	errorPeer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(500)
	}))

	peers := []string{
		httptestHost(errorPeer),
		httptestHost(errorPeer),
		httptestHost(errorPeer),
	}

	r, _ := http.NewRequest("GET", "http://localhost", nil)
	res, peer, err := proxyTestVersion.proxy(r, peers)

	assert.Equal(t, errNoAvailablePeers, err, "proxying should return errNoAvailablePeers if all error")
	assert.Nil(t, res, "proxying should return errNoAvailablePeers if all error")
	assert.Equal(t, "", peer, "peer should be empty if proxying timed out")
}
<commit_msg>Increase sleep times in proxy tests<commit_after>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"net/url"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

var proxyTestVersion = &version{
	name: "foo",
	sequins: &sequins{
		config: sequinsConfig{
			Sharding: shardingConfig{
				ProxyTimeout:      duration{300 * time.Millisecond},
				ProxyStageTimeout: duration{100 * time.Millisecond},
			},
		},
	},
}

func readAll(t *testing.T, r io.Reader) string {
	b, err := ioutil.ReadAll(r)
	require.NoError(t, err)
	return string(b)
}

func httptestHost(s *httptest.Server) string {
	parsed, _ := url.Parse(s.URL)
	return parsed.Host
}

func TestProxySinglePeer(t *testing.T) {
	goodPeer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, "all good")
	}))

	peers := []string{httptestHost(goodPeer)}
	r, _ := http.NewRequest("GET", "http://localhost", nil)
	res, peer, err := proxyTestVersion.proxy(r, peers)

	require.NoError(t, err, "simple proxying should work")
	assert.NotNil(t, res, "simple proxying should work")

	assert.Equal(t, httptestHost(goodPeer), peer, "the returned peer should be correct")
	assert.Equal(t, "all good\n", readAll(t, res.Body))
}

func TestProxySlowPeer(t *testing.T) {
	slowPeer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		time.Sleep(1 * time.Second)
		fmt.Fprintln(w, "sorry, did you need something?")
	}))

	goodPeer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, "all good")
	}))

	notReachedPeer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		require.FailNow(t, "proxying should succeed before getting to a third peer")
	}))

	peers := []string{httptestHost(slowPeer), httptestHost(goodPeer), httptestHost(notReachedPeer)}
	r, _ := http.NewRequest("GET", "http://localhost", nil)
	res, peer, err := proxyTestVersion.proxy(r, peers)
	require.NoError(t, err, "proxying should work on the second peer")
	require.NotNil(t, res, "proxying should work on the second peer")

	assert.Equal(t, httptestHost(goodPeer), peer, "the returned peer should be correct")
	assert.Equal(t, "all good\n", readAll(t, res.Body))
}

func TestProxyErrorPeer(t *testing.T) {
	errorPeer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(500)
	}))

	goodPeer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, "all good")
	}))

	notReachedPeer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		assert.Fail(t, "proxying should succeed before getting to a third peer")
	}))

	peers := []string{httptestHost(errorPeer), httptestHost(goodPeer), httptestHost(notReachedPeer)}
	r, _ := http.NewRequest("GET", "http://localhost", nil)
	res, peer, err := proxyTestVersion.proxy(r, peers)

	require.NoError(t, err, "proxying should work on the second peer")
	require.NotNil(t, res, "proxying should work on the second peer")

	assert.Equal(t, httptestHost(goodPeer), peer, "the returned peer should be correct")
	assert.Equal(t, "all good\n", readAll(t, res.Body))
}

func TestProxySlowPeerErrorPeer(t *testing.T) {
	slowPeer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		time.Sleep(150 * time.Millisecond)
		fmt.Fprintln(w, "all good, sorry to keep you waiting")
	}))

	errorPeer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(500)
	}))

	peers := []string{httptestHost(slowPeer), httptestHost(errorPeer)}
	r, _ := http.NewRequest("GET", "http://localhost", nil)
	res, peer, err := proxyTestVersion.proxy(r, peers)

	require.NoError(t, err, "proxying should work on the first peer")
	require.NotNil(t, res, "proxying should work on the first peer")

	assert.Equal(t, httptestHost(slowPeer), peer, "the returned peer should be correct")
	assert.Equal(t, "all good, sorry to keep you waiting\n", readAll(t, res.Body))
}

func TestProxyTimeout(t *testing.T) {
	slowPeer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		time.Sleep(1 * time.Second)
		fmt.Fprintln(w, "sorry, did you need something?")
	}))

	notReachedPeer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		assert.Fail(t, "proxying should never try a fourth peer")
	}))

	peers := []string{
		httptestHost(slowPeer),
		httptestHost(slowPeer),
		httptestHost(slowPeer),
		httptestHost(notReachedPeer),
	}

	r, _ := http.NewRequest("GET", "http://localhost", nil)
	res, peer, err := proxyTestVersion.proxy(r, peers)

	assert.Equal(t, errProxyTimeout, err, "proxying should time out with all slow peers")
	assert.Nil(t, res, "proxying should time out with all slow peers")
	assert.Equal(t, "", peer, "peer should be empty if proxying timed out")
}

func TestProxyErrors(t *testing.T) {
	errorPeer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(500)
	}))

	peers := []string{
		httptestHost(errorPeer),
		httptestHost(errorPeer),
		httptestHost(errorPeer),
	}

	r, _ := http.NewRequest("GET", "http://localhost", nil)
	res, peer, err := proxyTestVersion.proxy(r, peers)

	assert.Equal(t, errNoAvailablePeers, err, "proxying should return errNoAvailablePeers if all error")
	assert.Nil(t, res, "proxying should return errNoAvailablePeers if all error")
	assert.Equal(t, "", peer, "peer should be empty if proxying timed out")
}
<|endoftext|>


<commit_before>package common

import (
	"fmt"
	"strings"
)

type SystemLog struct {
	debugOn     bool
	productions []string
	debugLines  []string
	debugDepth  int
	errors      []string
	clarificationQuestion string
	clarificationOptions *Options
	ok          bool
	depth		int
}

func NewSystemLog(debugOn bool) *SystemLog {
	log := SystemLog{ debugOn: debugOn }
	log.Clear()

	return &log
}

func (log *SystemLog) Clear() {
	log.productions = []string{}
	log.debugLines = []string{}
	log.debugDepth = 0
	log.errors = []string{}
	log.clarificationQuestion = ""
	log.clarificationOptions = &Options{}
	log.ok = true
}

func (log *SystemLog) EnableDebug() {
	log.debugOn = true
}

func (log *SystemLog) DisableDebug() {
	log.debugOn = true
}

func (log *SystemLog) ToggleDebug() {
	log.debugOn = !log.debugOn
}

func (log *SystemLog) AddProduction(name string, production string) {
	stmt := strings.Repeat("  ", log.debugDepth) + name + ": " + production + " "
	log.productions = append(log.productions, stmt)
}

func (log *SystemLog) StartProduction(name string, production string) {
	log.AddProduction(name, production)
	log.debugDepth++
}

func (log *SystemLog) EndProduction(name string, production string) {
	log.debugDepth--
	log.AddProduction(name, production)
}

func (log *SystemLog) AddError(error string) {
	log.ok = false
	log.errors = append(log.errors, error)
	log.AddProduction("ERROR", error)
}

func (log *SystemLog) SetClarificationRequest(question string, options *Options) {
	log.clarificationQuestion = question
	log.clarificationOptions = options
}

func (log *SystemLog) GetClarificationQuestion() string {
	return log.clarificationQuestion
}

func (log *SystemLog) GetClarificationOptions() *Options {
	return log.clarificationOptions
}

func (log *SystemLog) IsOk() bool {
	return log.ok
}

func (log *SystemLog) IsDone() bool {
	return !log.ok || log.clarificationQuestion != ""
}

func (log *SystemLog) StartDebug(text string, vals ...interface{}) {

	if !log.debugOn {
		return
	}

	stmt := strings.Repeat("  ", log.debugDepth) + text + " "
	for _, val := range vals {
		stmt += fmt.Sprintf("%v", val) + " "
	}

	log.debugLines = append(log.debugLines, stmt)
	log.debugDepth++
}

func (log *SystemLog) EndDebug(text string, vals ...interface{}) {

	if !log.debugOn {
		return
	}

	if log.debugDepth > 0 {
		log.debugDepth--
	}

	stmt := strings.Repeat("  ", log.debugDepth) + text + " "
	for _, val := range vals {
		stmt += fmt.Sprintf("%v", val) + " "
	}

	log.debugLines = append(log.debugLines, stmt)
}

func (log *SystemLog) GetDebugLines() []string {
	return log.debugLines
}

func (log *SystemLog) GetProductions() []string {
	return log.productions
}

func (log *SystemLog) GetErrors() []string {
	return log.errors
}

func (log *SystemLog) String() string {
	s := ""

	if !log.IsOk() {
		s += "\n"
		for _, error := range log.errors {
			s += "ERROR: " + error + "\n"
		}
		s += "\n"
	}

	for _, production := range log.GetProductions() {
		s += fmt.Sprintln(production)
	}

	for _, debugLine := range log.debugLines {
		s += debugLine + "\n"
	}

	return s
}
<commit_msg>productions with lines<commit_after>package common

import (
	"fmt"
	"strings"
)

type SystemLog struct {
	debugOn     bool
	productions []string
	debugLines  []string
	debugDepth  int
	errors      []string
	clarificationQuestion string
	clarificationOptions *Options
	ok          bool
	depth		int
}

func NewSystemLog(debugOn bool) *SystemLog {
	log := SystemLog{ debugOn: debugOn }
	log.Clear()

	return &log
}

func (log *SystemLog) Clear() {
	log.productions = []string{}
	log.debugLines = []string{}
	log.debugDepth = 0
	log.errors = []string{}
	log.clarificationQuestion = ""
	log.clarificationOptions = &Options{}
	log.ok = true
}

func (log *SystemLog) EnableDebug() {
	log.debugOn = true
}

func (log *SystemLog) DisableDebug() {
	log.debugOn = true
}

func (log *SystemLog) ToggleDebug() {
	log.debugOn = !log.debugOn
}

func (log *SystemLog) AddProduction(name string, production string) {
	stmt := strings.Repeat("| ", log.debugDepth) + name + ": " + production + " "
	log.productions = append(log.productions, stmt)
}

func (log *SystemLog) StartProduction(name string, production string) {
	log.AddProduction("╭ " + name, production)
	log.debugDepth++
}

func (log *SystemLog) EndProduction(name string, production string) {
	log.debugDepth--
	log.AddProduction("╰ " + name, production)
}

func (log *SystemLog) AddError(error string) {
	log.ok = false
	log.errors = append(log.errors, error)
	log.AddProduction("ERROR", error)
}

func (log *SystemLog) SetClarificationRequest(question string, options *Options) {
	log.clarificationQuestion = question
	log.clarificationOptions = options
}

func (log *SystemLog) GetClarificationQuestion() string {
	return log.clarificationQuestion
}

func (log *SystemLog) GetClarificationOptions() *Options {
	return log.clarificationOptions
}

func (log *SystemLog) IsOk() bool {
	return log.ok
}

func (log *SystemLog) IsDone() bool {
	return !log.ok || log.clarificationQuestion != ""
}

func (log *SystemLog) StartDebug(text string, vals ...interface{}) {

	if !log.debugOn {
		return
	}

	stmt := strings.Repeat("  ", log.debugDepth) + text + " "
	for _, val := range vals {
		stmt += fmt.Sprintf("%v", val) + " "
	}

	log.debugLines = append(log.debugLines, stmt)
	log.debugDepth++
}

func (log *SystemLog) EndDebug(text string, vals ...interface{}) {

	if !log.debugOn {
		return
	}

	if log.debugDepth > 0 {
		log.debugDepth--
	}

	stmt := strings.Repeat("  ", log.debugDepth) + text + " "
	for _, val := range vals {
		stmt += fmt.Sprintf("%v", val) + " "
	}

	log.debugLines = append(log.debugLines, stmt)
}

func (log *SystemLog) GetDebugLines() []string {
	return log.debugLines
}

func (log *SystemLog) GetProductions() []string {
	return log.productions
}

func (log *SystemLog) GetErrors() []string {
	return log.errors
}

func (log *SystemLog) String() string {
	s := ""

	if !log.IsOk() {
		s += "\n"
		for _, error := range log.errors {
			s += "ERROR: " + error + "\n"
		}
		s += "\n"
	}

	for _, production := range log.GetProductions() {
		s += fmt.Sprintln(production)
	}

	for _, debugLine := range log.debugLines {
		s += debugLine + "\n"
	}

	return s
}
<|endoftext|>


<commit_before>/* access.go : Implement strided memory access to see the effects of cache
Author: James Fairbanks
Date: 2012-09-02
Liscence: BSD
*/

package main

import (
	"fmt"
	"github.com/jpfairbanks/timing"
	"math/rand"
	"flag"
)

//Dtype: the data type of the arrays
type Dtype int64

//Strider: a slice of type dtype with a fixed stride
type Strider struct {
	data   []Dtype
	stride int
}

//Fill: fill a strided array so that we can walk it later
func (s Strider) Fill() {
	n := len(s.data)
	iterations := s.stride
	for k := 0; k < iterations; k++ {
		for i := 0; i < len(s.data)-k; i += 1 {
			s.data[(s.stride*i+k) % n] = Dtype((s.stride*(i+1) + k) % n)
		}
	}
	for i:=1; i < s.stride; i++{
		//fmt.Printf("%d ", i)
		s.data[(n-i)-1] = s.data[(n-i)-1] + 1
	}
	s.data[n-1] = 0
}

//Walk: walk a strided array by the stride
func (s Strider) Walk() Dtype {
	var v Dtype
	ptr := s.data[0]
	//fmt.Printf("stride: %d, ", s.stride)
	for ptr != 0 {
		v += ptr
		ptr = s.data[ptr]
	}
	return v
}

var scaleptr *uint
func init(){
	scaleptr = flag.Uint("scale", 10, "set the length of the array to access as a power of 2")
	flag.Parse()
}
//main: time.Time the Walk
func main() {
	var n, s int
	var scale uint
	scale = *scaleptr
	n = 2 << scale
	//fmt.Printf("scale:%d,", scale)
	s = 1
	k := 3
	stepsize := make([]int, k)
	tg := timing.New(k)
	correctsum := Dtype((n * (n - 1)) / 2)
	stepsize[0] = 1
	for i := 1; i < len(stepsize); i++ {
		stepsize[i] = 2 * stepsize[i-1]
	}
	//fmt.Printf("strides:%v,\n",stepsize)
	sdr := Strider{make([]Dtype, n), s}
	var sum Dtype
	for i, k := range stepsize {
		sdr.stride = k
		sdr.Fill()
		tg.Tic(i)
		for iter := 0; iter < 10; iter++ {
			sum = sdr.Walk()
		}
		tg.Toc(i)
		if  correctsum != sum {
			fmt.Println("we did not hit all the elements")
			fmt.Println(correctsum, sum)
		}
	}

	perm := rand.Perm(n)
	ptg := timing.New(1)
	ptg.Tic(0)
	var randsum Dtype
	for _, j := range perm {
		randsum += sdr.data[j]
	}
	ptg.Toc(0)
	if randsum != correctsum {
		fmt.Printf("random ordered produced wrong sum: %v, %v,\n", randsum, correctsum)
	}
	tg.Resolve()
	fmt.Println(tg.TupleString("\n"))
	ptg.Resolve()
	fmt.Println("Random: ", ptg)
}
<commit_msg>change output format\n random case takes as many iterations as strided cases.<commit_after>/* access.go : Implement strided memory access to see the effects of cache
Author: James Fairbanks
Date: 2012-09-02
Liscence: BSD
*/

package main

import (
	"fmt"
	"github.com/jpfairbanks/timing"
	"math/rand"
	"flag"
)

//Dtype: the data type of the arrays
type Dtype int64

//Strider: a slice of type dtype with a fixed stride
type Strider struct {
	data   []Dtype
	stride int
}

//Fill: fill a strided array so that we can walk it later
func (s Strider) Fill() {
	n := len(s.data)
	iterations := s.stride
	for k := 0; k < iterations; k++ {
		for i := 0; i < len(s.data)-k; i += 1 {
			s.data[(s.stride*i+k) % n] = Dtype((s.stride*(i+1) + k) % n)
		}
	}
	for i:=1; i < s.stride; i++{
		//fmt.Printf("%d ", i)
		s.data[(n-i)-1] = s.data[(n-i)-1] + 1
	}
	s.data[n-1] = 0
}

//Walk: walk a strided array by the stride
func (s Strider) Walk() Dtype {
	var v Dtype
	ptr := s.data[0]
	//fmt.Printf("stride: %d, ", s.stride)
	for ptr != 0 {
		v += ptr
		ptr = s.data[ptr]
	}
	return v
}

var scaleptr *uint
func init(){
	scaleptr = flag.Uint("scale", 10, "set the length of the array to access as a power of 2")
	flag.Parse()
}
//main: time.Time the Walk
func main() {
	var n, s int
	var scale uint
	scale = *scaleptr
	n = 2 << scale
	//fmt.Printf("scale:%d,", scale)
	s = 1
	k := 6
	stepsize := make([]int, k)
	tg := timing.New(k)
	correctsum := Dtype((n * (n - 1)) / 2)
	stepsize[0] = 1
	for i := 1; i < len(stepsize); i++ {
		stepsize[i] = 2 * stepsize[i-1]
	}
	//fmt.Printf("strides:%v,\n",stepsize)
	sdr := Strider{make([]Dtype, n), s}
	var sum Dtype
	for i, k := range stepsize {
		sdr.stride = k
		sdr.Fill()
		tg.Tic(i)
		for iter := 0; iter < 10; iter++ {
			sum = sdr.Walk()
		}
		tg.Toc(i)
		if  correctsum != sum {
			fmt.Println("we did not hit all the elements")
			fmt.Println(correctsum, sum)
		}
	}

	perm := rand.Perm(n)
	ptg := timing.New(1)
	ptg.Tic(0)
	var randsum Dtype
	for run:=0; run < 10; run++{
		randsum = 0
		for _, j := range perm {
			randsum += sdr.data[j]
		}
	}
	ptg.Toc(0)
	if randsum != correctsum {
		fmt.Printf("random ordered produced wrong sum: %v, %v,\n", randsum, correctsum)
	}
	tg.Resolve()
	fmt.Print(tg)
	ptg.Resolve()
	fmt.Printf(" %v\n", ptg)
}
<|endoftext|>


<commit_before>package commands

import (
	"os"
	"os/exec"
	"regexp"
	"runtime"
	"strings"

	"github.com/phase2/rig/cli/util"
	"github.com/urfave/cli"
)

type Dns struct {
	BaseCommand
}

func (cmd *Dns) Commands() cli.Command {
	return cli.Command{
		Name:  "dns",
		Usage: "Start DNS services on the docker-machine",
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:   "nameservers",
				Value:  "8.8.8.8:53",
				Usage:  "Comma separated list of fallback names servers.",
				EnvVar: "RIG_NAMESERVERS",
			},
		},
		Before: cmd.Before,
		Action: cmd.Run,
	}
}

func (cmd *Dns) Run(c *cli.Context) error {
	if cmd.machine.IsRunning() {
		cmd.out.Info.Println("Configuring DNS")
		cmd.ConfigureDns(cmd.machine, c.String("nameservers"))
		cmd.ConfigureRoutes(cmd.machine)
	} else {
		cmd.out.Error.Fatalf("Machine '%s' is not running.", cmd.machine.Name)
	}

	return nil
}

// Remove the host filter from the xhyve bridge interface
func (cmd *Dns) RemoveHostFilter(ipAddr string) {
	// #1: route -n get <machineIP> to find the interface name
	routeData, err := exec.Command("route", "-n", "get", ipAddr).CombinedOutput()
	if err != nil {
		cmd.out.Warning.Println("Unable to determine bridge interface to remove hostfilter")
		return
	}
	ifaceRegexp := regexp.MustCompile(`interface:\s(\w+)`)
	iface := ifaceRegexp.FindStringSubmatch(string(routeData))[1]

	// #2: ifconfig <interface name> to get the details
	ifaceData, err := exec.Command("ifconfig", iface).CombinedOutput()
	if err != nil {
		cmd.out.Warning.Println("Unable to determine member to remove hostfilter")
		return
	}
	memberRegexp := regexp.MustCompile(`member:\s(\w+)\s`)
	member := memberRegexp.FindStringSubmatch(string(ifaceData))[1]

	// #4: ifconfig <bridge> -hostfilter <member>
	util.StreamCommand(exec.Command("sudo", "ifconfig", iface, "-hostfilter", member))
}

func (cmd *Dns) ConfigureRoutes(machine Machine) {
	cmd.out.Info.Println("Setting up local networking (may require your admin password)")

	machineIp := machine.GetIP()
	bridgeIp := machine.GetBridgeIP()
	if runtime.GOOS == "windows" {
		exec.Command("runas", "/noprofile", "/user:Administrator", "route", "DELETE", "172.17.0.0").Run()
		util.StreamCommand(exec.Command("runas", "/noprofile", "/user:Administrator", "route", "-p", "ADD", "172.17.0.0/16", machineIp))
	} else {
		if machine.IsXhyve() {
			cmd.RemoveHostFilter(machine.GetIP())
		}
		exec.Command("sudo", "mkdir", "-p", "/etc/resolver").Run()
		exec.Command("bash", "-c", "echo \"nameserver "+bridgeIp+"\" | sudo tee /etc/resolver/vm").Run()
		exec.Command("sudo", "route", "-n", "delete", "-net", "172.17.0.0").Run()
		util.StreamCommand(exec.Command("sudo", "route", "-n", "add", "172.17.0.0/16", machineIp))

		if _, err := os.Stat("/usr/sbin/discoveryutil"); err == nil {
			// Put this here for people running OS X 10.10.0 to 10.10.3 (oy vey.)
			cmd.out.Verbose.Println("Restarting discoveryutil to flush DNS caches")
			util.StreamCommand(exec.Command("sudo", "launchctl", "unload", "-w", "/System/Library/LaunchDaemons/com.apple.discoveryd.plist"))
			util.StreamCommand(exec.Command("sudo", "launchctl", "load", "-w", "/System/Library/LaunchDaemons/com.apple.discoveryd.plist"))
		} else {
			// Reset DNS. We have seen this suddenly make /etc/resolver/vm work.
			cmd.out.Verbose.Println("Restarting mDNSResponder to flush DNS caches")
			util.StreamCommand(exec.Command("sudo", "launchctl", "unload", "-w", "/System/Library/LaunchDaemons/com.apple.mDNSResponder.plist"))
			util.StreamCommand(exec.Command("sudo", "launchctl", "load", "-w", "/System/Library/LaunchDaemons/com.apple.mDNSResponder.plist"))
		}
	}
}

func (cmd *Dns) ConfigureDns(machine Machine, nameservers string) {
	dnsServers := strings.Split(nameservers, ",")

	machine.SetEnv()
	bridgeIp := machine.GetBridgeIP()

	// Start dnsdock
	exec.Command("docker", "stop", "dnsdock").Run()
	exec.Command("docker", "rm", "dnsdock").Run()

	args := []string{
		"run",
		"-d",
		"--restart=always",
		"-v", "/var/run/docker.sock:/var/run/docker.sock",
		"-l", "com.dnsdock.name=dnsdock",
		"-l", "com.dnsdock.image=outrigger",
		"--name", "dnsdock",
		"-p", bridgeIp + ":53:53/udp",
		"aacebedo/dnsdock:v1.16.1-amd64",
		"--domain=vm",
	}
	for _, server := range dnsServers {
		args = append(args, "--nameserver="+server)
	}
	util.ForceStreamCommand(exec.Command("docker", args...))
}
<commit_msg>Restart mDNSResponder differently. 10.12.4 introduced System Integrity Protections around launchctl<commit_after>package commands

import (
	"os"
	"os/exec"
	"regexp"
	"runtime"
	"strings"

	"github.com/phase2/rig/cli/util"
	"github.com/urfave/cli"
)

type Dns struct {
	BaseCommand
}

func (cmd *Dns) Commands() cli.Command {
	return cli.Command{
		Name:  "dns",
		Usage: "Start DNS services on the docker-machine",
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:   "nameservers",
				Value:  "8.8.8.8:53",
				Usage:  "Comma separated list of fallback names servers.",
				EnvVar: "RIG_NAMESERVERS",
			},
		},
		Before: cmd.Before,
		Action: cmd.Run,
	}
}

func (cmd *Dns) Run(c *cli.Context) error {
	if cmd.machine.IsRunning() {
		cmd.out.Info.Println("Configuring DNS")
		cmd.ConfigureDns(cmd.machine, c.String("nameservers"))
		cmd.ConfigureRoutes(cmd.machine)
	} else {
		cmd.out.Error.Fatalf("Machine '%s' is not running.", cmd.machine.Name)
	}

	return nil
}

// Remove the host filter from the xhyve bridge interface
func (cmd *Dns) RemoveHostFilter(ipAddr string) {
	// #1: route -n get <machineIP> to find the interface name
	routeData, err := exec.Command("route", "-n", "get", ipAddr).CombinedOutput()
	if err != nil {
		cmd.out.Warning.Println("Unable to determine bridge interface to remove hostfilter")
		return
	}
	ifaceRegexp := regexp.MustCompile(`interface:\s(\w+)`)
	iface := ifaceRegexp.FindStringSubmatch(string(routeData))[1]

	// #2: ifconfig <interface name> to get the details
	ifaceData, err := exec.Command("ifconfig", iface).CombinedOutput()
	if err != nil {
		cmd.out.Warning.Println("Unable to determine member to remove hostfilter")
		return
	}
	memberRegexp := regexp.MustCompile(`member:\s(\w+)\s`)
	member := memberRegexp.FindStringSubmatch(string(ifaceData))[1]

	// #4: ifconfig <bridge> -hostfilter <member>
	util.StreamCommand(exec.Command("sudo", "ifconfig", iface, "-hostfilter", member))
}

func (cmd *Dns) ConfigureRoutes(machine Machine) {
	cmd.out.Info.Println("Setting up local networking (may require your admin password)")

	machineIp := machine.GetIP()
	bridgeIp := machine.GetBridgeIP()
	if runtime.GOOS == "windows" {
		exec.Command("runas", "/noprofile", "/user:Administrator", "route", "DELETE", "172.17.0.0").Run()
		util.StreamCommand(exec.Command("runas", "/noprofile", "/user:Administrator", "route", "-p", "ADD", "172.17.0.0/16", machineIp))
	} else {
		if machine.IsXhyve() {
			cmd.RemoveHostFilter(machine.GetIP())
		}
		exec.Command("sudo", "mkdir", "-p", "/etc/resolver").Run()
		exec.Command("bash", "-c", "echo \"nameserver "+bridgeIp+"\" | sudo tee /etc/resolver/vm").Run()
		exec.Command("sudo", "route", "-n", "delete", "-net", "172.17.0.0").Run()
		util.StreamCommand(exec.Command("sudo", "route", "-n", "add", "172.17.0.0/16", machineIp))

		if _, err := os.Stat("/usr/sbin/discoveryutil"); err == nil {
			// Put this here for people running OS X 10.10.0 to 10.10.3 (oy vey.)
			cmd.out.Verbose.Println("Restarting discoveryutil to flush DNS caches")
			util.StreamCommand(exec.Command("sudo", "launchctl", "unload", "-w", "/System/Library/LaunchDaemons/com.apple.discoveryd.plist"))
			util.StreamCommand(exec.Command("sudo", "launchctl", "load", "-w", "/System/Library/LaunchDaemons/com.apple.discoveryd.plist"))
		} else {
			// Reset DNS cache. We have seen this suddenly make /etc/resolver/vm work.
			cmd.out.Verbose.Println("Restarting mDNSResponder to flush DNS caches")
			util.StreamCommand(exec.Command("sudo", "killall", "-HUP", "mDNSResponder"))
		}
	}
}

func (cmd *Dns) ConfigureDns(machine Machine, nameservers string) {
	dnsServers := strings.Split(nameservers, ",")

	machine.SetEnv()
	bridgeIp := machine.GetBridgeIP()

	// Start dnsdock
	exec.Command("docker", "stop", "dnsdock").Run()
	exec.Command("docker", "rm", "dnsdock").Run()

	args := []string{
		"run",
		"-d",
		"--restart=always",
		"-v", "/var/run/docker.sock:/var/run/docker.sock",
		"-l", "com.dnsdock.name=dnsdock",
		"-l", "com.dnsdock.image=outrigger",
		"--name", "dnsdock",
		"-p", bridgeIp + ":53:53/udp",
		"aacebedo/dnsdock:v1.16.1-amd64",
		"--domain=vm",
	}
	for _, server := range dnsServers {
		args = append(args, "--nameserver="+server)
	}
	util.ForceStreamCommand(exec.Command("docker", args...))
}
<|endoftext|>


<commit_before>package msg

import (
	"fmt"
	"github.com/viant/endly/testing/validator"
	"github.com/viant/toolbox/data"
	"github.com/viant/toolbox/url"
)

const defaultTimeoutMs = 10000

//CreateRequest represents a create resource request
type CreateRequest struct {
	Credentials string
	Resources   []*ResourceSetup
}

func (r *CreateRequest) Init() error {
	if len(r.Resources) == 0 {
		return nil
	}
	for _, resource := range r.Resources {
		if err := resource.Init(); err != nil {
			return err
		}
		if resource.Credentials == "" {
			resource.Credentials = r.Credentials
		}
	}
	return nil
}

func (r *CreateRequest) Validate() error {
	if len(r.Resources) == 0 {
		return fmt.Errorf("resources was empty")
	}
	for _, resource := range r.Resources {
		if err := resource.Validate(); err != nil {
			return err
		}

	}
	return nil
}

//CreateResponse represents a create resource response
type CreateResponse struct {
	Resources []*Resource
}

//DeleteRequest represents a delete resource request
type DeleteRequest struct {
	Credentials string
	Resources   []*Resource
}

func (r *DeleteRequest) Init() error {
	if len(r.Resources) == 0 {
		return nil
	}
	for _, resource := range r.Resources {
		_ = resource.Init()
		if resource.Credentials == "" {
			resource.Credentials = r.Credentials
		}
	}
	return nil
}

//DeleteResponse represents a delete resource response
type DeleteResponse struct{}

//PushRequest represents push request
type PushRequest struct {
	Credentials   string
	Dest          *Resource
	Messages      []*Message
	Source        *url.Resource
	TimeoutMs     int
	UDF           string
	isInitialized bool
}

func (r *PushRequest) Init() error {
	if r.isInitialized {
		return nil
	}
	if r.Dest != nil {
		if err := r.Dest.Init(); err != nil {
			return err
		}
		if r.Dest.Credentials == "" {
			r.Dest.Credentials = r.Credentials
		}
	}
	if r.TimeoutMs == 0 {
		r.TimeoutMs = defaultTimeoutMs
	}
	return nil
}

func (r *PushRequest) Validate() error {
	if r.Dest == nil {
		return fmt.Errorf("dest was empty")
	}
	if len(r.Messages) == 0 && r.Source == nil {
		return fmt.Errorf("messages were empty")
	}
	return nil
}

//PushResponse represents a push response
type PushResponse struct {
	Results []Result
}

//PullRequest represents a pull request
type PullRequest struct {
	Credentials string
	Source      *Resource
	TimeoutMs   int
	Count       int
	Nack        bool `description:"flag indicates that the client will not or cannot process a Message passed to the Subscriber.Receive callback."`
	UDF         string
	Expect      interface{}
}

func (r *PullRequest) Init() error {
	if r.TimeoutMs == 0 {
		r.TimeoutMs = defaultTimeoutMs
	}
	if r.Source.Credentials == "" {
		r.Source.Credentials = r.Credentials
	}
	return r.Source.Init()
}

func (r *PullRequest) Validate() error {
	if r.Source == nil {
		return fmt.Errorf("source was empty")
	}
	return nil
}

//PullRequest represents a pull response
type PullResponse struct {
	Messages []*Message
	Assert   *validator.AssertResponse
}

type Message struct {
	ID          string
	Subject     string
	Attributes  map[string]interface{}
	Data        interface{}
	Transformed interface{} `description:"udf transformed data"`
}

func (m *Message) Expand(state data.Map) *Message {
	var result = &Message{
		Attributes: make(map[string]interface{}),
		Subject:    m.Subject,
	}
	if len(m.Attributes) > 0 {
		for k, v := range m.Attributes {
			result.Attributes[state.ExpandAsText(k)] = state.Expand(v)
		}
	}
	if m.Data != nil {
		result.Data = state.Expand(m.Data)
	}
	return result
}

type Result interface{}
<commit_msg>added description<commit_after>package msg

import (
	"fmt"
	"github.com/viant/endly/testing/validator"
	"github.com/viant/toolbox/data"
	"github.com/viant/toolbox/url"
)

const defaultTimeoutMs = 10000

//CreateRequest represents a create resource request
type CreateRequest struct {
	Credentials string
	Resources   []*ResourceSetup
}

func (r *CreateRequest) Init() error {
	if len(r.Resources) == 0 {
		return nil
	}
	for _, resource := range r.Resources {
		if err := resource.Init(); err != nil {
			return err
		}
		if resource.Credentials == "" {
			resource.Credentials = r.Credentials
		}
	}
	return nil
}

func (r *CreateRequest) Validate() error {
	if len(r.Resources) == 0 {
		return fmt.Errorf("resources was empty")
	}
	for _, resource := range r.Resources {
		if err := resource.Validate(); err != nil {
			return err
		}

	}
	return nil
}

//CreateResponse represents a create resource response
type CreateResponse struct {
	Resources []*Resource
}

//DeleteRequest represents a delete resource request
type DeleteRequest struct {
	Credentials string
	Resources   []*Resource
}

func (r *DeleteRequest) Init() error {
	if len(r.Resources) == 0 {
		return nil
	}
	for _, resource := range r.Resources {
		_ = resource.Init()
		if resource.Credentials == "" {
			resource.Credentials = r.Credentials
		}
	}
	return nil
}

//DeleteResponse represents a delete resource response
type DeleteResponse struct{}

//PushRequest represents push request
type PushRequest struct {
	Credentials   string
	Dest          *Resource
	Messages      []*Message
	Source        *url.Resource `description:"source for message data  currently JSON or new line delimitered json are supported that conform []*Message type "`
	TimeoutMs     int
	UDF           string
	isInitialized bool
}

func (r *PushRequest) Init() error {
	if r.isInitialized {
		return nil
	}
	if r.Dest != nil {
		if err := r.Dest.Init(); err != nil {
			return err
		}
		if r.Dest.Credentials == "" {
			r.Dest.Credentials = r.Credentials
		}
	}
	if r.TimeoutMs == 0 {
		r.TimeoutMs = defaultTimeoutMs
	}
	return nil
}

func (r *PushRequest) Validate() error {
	if r.Dest == nil {
		return fmt.Errorf("dest was empty")
	}
	if len(r.Messages) == 0 && r.Source == nil {
		return fmt.Errorf("messages were empty")
	}
	return nil
}

//PushResponse represents a push response
type PushResponse struct {
	Results []Result
}

//PullRequest represents a pull request
type PullRequest struct {
	Credentials string
	Source      *Resource
	TimeoutMs   int
	Count       int
	Nack        bool `description:"flag indicates that the client will not or cannot process a Message passed to the Subscriber.Receive callback."`
	UDF         string
	Expect      interface{}
}

func (r *PullRequest) Init() error {
	if r.TimeoutMs == 0 {
		r.TimeoutMs = defaultTimeoutMs
	}
	if r.Source.Credentials == "" {
		r.Source.Credentials = r.Credentials
	}
	return r.Source.Init()
}

func (r *PullRequest) Validate() error {
	if r.Source == nil {
		return fmt.Errorf("source was empty")
	}
	return nil
}

//PullRequest represents a pull response
type PullResponse struct {
	Messages []*Message
	Assert   *validator.AssertResponse
}

type Message struct {
	ID          string
	Subject     string
	Attributes  map[string]interface{}
	Data        interface{}
	Transformed interface{} `description:"udf transformed data"`
}

func (m *Message) Expand(state data.Map) *Message {
	var result = &Message{
		Attributes: make(map[string]interface{}),
		Subject:    m.Subject,
	}
	if len(m.Attributes) > 0 {
		for k, v := range m.Attributes {
			result.Attributes[state.ExpandAsText(k)] = state.Expand(v)
		}
	}
	if m.Data != nil {
		result.Data = state.Expand(m.Data)
	}
	return result
}

type Result interface{}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"net/http"
	"html/template"
)

// Represents a page in the wiki.
type Page struct {
	Title string
	// io library works with bytes, so that's why we use a byte slice.
	Body []byte
}

// Persistence function.
// "p" is the receiver of this function. Interesting.
// Will return nil if save works.
func (p *Page) save() error {
	filename := "data/" + p.Title + ".txt"
	return ioutil.WriteFile(filename, p.Body, 0600)
}

// Load data from filesystem.
func loadPage(title string) (*Page, error) {
	filename := "data/" + title + ".txt"
	body, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	return &Page{Title: title, Body: body}, nil
}

func viewHandler(w http.ResponseWriter, r *http.Request) {
	title := r.URL.Path[len("/view/"):]
	p, err := loadPage(title)
	if err != nil {
		http.NotFound(w, r)
		return
	}
	t, _ := template.ParseFiles("templates/view.html")
	t.Execute(w, p)
}

func editHandler(w http.ResponseWriter, r *http.Request) {
	title := r.URL.Path[len("/edit"):]
	p, err := loadPage(title)
	templateFile := "edit"
	if err != nil {
		p = &Page{Title: title}
		templateFile = "create"
	}
	t, _ := template.ParseFiles("templates/" + templateFile + ".html")
	t.Execute(w, p)
}

func saveHandler(w http.ResponseWriter, r *http.Request) {
	http.NotFound(w, r)
}

func main() {
	http.HandleFunc("/view/", viewHandler)
	http.HandleFunc("/edit/", editHandler)
	http.HandleFunc("/save/", saveHandler)
	http.ListenAndServe(":8080", nil)
}

<commit_msg>Added renderTemplate func. Redirect from /view to /edit if not found.<commit_after>package main

import (
	"io/ioutil"
	"net/http"
	"html/template"
)

// Represents a page in the wiki.
type Page struct {
	Title string
	// io library works with bytes, so that's why we use a byte slice.
	Body []byte
}

// Persistence function.
// "p" is the receiver of this function. Interesting.
// Will return nil if save works.
func (p *Page) save() error {
	filename := "data/" + p.Title + ".txt"
	return ioutil.WriteFile(filename, p.Body, 0600)
}

// Load data from filesystem.
func loadPage(title string) (*Page, error) {
	filename := "data/" + title + ".txt"
	body, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	return &Page{Title: title, Body: body}, nil
}

func viewHandler(w http.ResponseWriter, r *http.Request) {
	title := r.URL.Path[len("/view/"):]
	p, err := loadPage(title)
	if err != nil {
		// If we attempt to view a page that doesn't exist, lets create one.
		http.Redirect(w, r, "/edit/" + title, http.StatusFound)
		return
	}
	renderTemplate(w, "view", p)
}

func editHandler(w http.ResponseWriter, r *http.Request) {
	title := r.URL.Path[len("/edit"):]
	p, err := loadPage(title)
	tmpl := "edit"
	if err != nil {
		p = &Page{Title: title}
		tmpl = "create"
	}
	renderTemplate(w, tmpl, p)
}

func saveHandler(w http.ResponseWriter, r *http.Request) {
	http.NotFound(w, r)
}

func renderTemplate(w http.ResponseWriter, tmpl string, p *Page) {
	t, _ := template.ParseFiles("templates/" + tmpl + ".html")
	t.Execute(w, p)
}

func main() {
	http.HandleFunc("/view/", viewHandler)
	http.HandleFunc("/edit/", editHandler)
	http.HandleFunc("/save/", saveHandler)
	http.ListenAndServe(":8080", nil)
}

<|endoftext|>


<commit_before>package golapa

import (
	 "fmt"
	"net/http"
	"text/template"
	"net/url"
	"log"
	"os"
	"time"
)

type RequestHandler interface {
	Get(w http.ResponseWriter, r *http.Request, name string, tpl *template.Template)
	Post(w http.ResponseWriter, r *http.Request, tpl *template.Template, data url.Values, template_path string)
}

type StandardRequestHandler struct {}

func (srh *StandardRequestHandler) Get(w http.ResponseWriter, r *http.Request, name string, tpl *template.Template) {
	switch name {
	case "c/terms":
		tpl.Lookup("terms.tpl").Execute(w, nil)
	case "c/privacy":
		tpl.Lookup("privacy.tpl").Execute(w, nil)
	default:
		tpl.Lookup("main.tpl").Execute(w, nil)

	}
}

func (srh *StandardRequestHandler) Post(w http.ResponseWriter,  r *http.Request, tpl *template.Template, data url.Values, template_path string) {
	sender := os.Getenv("EMAIL_SENDER")
	subject := os.Getenv("EMAIL_SUBJECT")

	email := data.Get("email")
	name := data.Get("name")

	if name != "" && email != "" {
		log.Print(name)
		log.Print(email)

		// Create goroutine to send email via
		go func(){
			mailer := &StandardEmailer{template_path}
			msg, err := mailer.CreateMessage(sender, subject, r.FormValue("name"), r.FormValue("email"))
			if err != nil {
				log.Print("Could not create email: %v", err)
			}
			mailer.Send(msg)
			time.Sleep(5000 * time.Millisecond)

		}()

		// OK
	}
	tpl.Lookup("post.tpl").Execute(w, data)

type BaseHandler struct {
	TemplatePath	string
}

func (bh *BaseHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	bh.serve(w, req, new(StandardRequestHandler))
}

func (bh *BaseHandler) serve(w http.ResponseWriter, r *http.Request, rh RequestHandler) {
	tpl, err := template.ParseFiles(fmt.Sprintf("%s/main.tpl", bh.TemplatePath), fmt.Sprintf("%s/post.tpl", bh.TemplatePath), fmt.Sprintf("%s/terms.tpl", bh.TemplatePath), fmt.Sprintf("%s/privacy.tpl", bh.TemplatePath), fmt.Sprintf("%s/header.tpl", bh.TemplatePath), fmt.Sprintf("%s/footer.tpl", bh.TemplatePath))
	if err != nil {
		http.Error(w, "Internal error", http.StatusInternalServerError)
		log.Print(err.Error())
		return
	}
	switch r.Method {
	case "GET":
		rh.Get(w, r, r.URL.Path[len("/"):], tpl)

	case "POST":
		err := r.ParseForm()
		if err != nil {
			http.Error(w, "Internal error", http.StatusInternalServerError)
			return
		}
		rh.Post(w, r, tpl, r.PostForm, bh.TemplatePath)
	default:
		http.Error(w, "Method not implemented", http.StatusInternalServerError)
	}
}
<commit_msg>Fixing syntax error<commit_after>package golapa

import (
	 "fmt"
	"net/http"
	"text/template"
	"net/url"
	"log"
	"os"
	"time"
)

type RequestHandler interface {
	Get(w http.ResponseWriter, r *http.Request, name string, tpl *template.Template)
	Post(w http.ResponseWriter, r *http.Request, tpl *template.Template, data url.Values, template_path string)
}

type StandardRequestHandler struct {}

func (srh *StandardRequestHandler) Get(w http.ResponseWriter, r *http.Request, name string, tpl *template.Template) {
	switch name {
	case "c/terms":
		tpl.Lookup("terms.tpl").Execute(w, nil)
	case "c/privacy":
		tpl.Lookup("privacy.tpl").Execute(w, nil)
	default:
		tpl.Lookup("main.tpl").Execute(w, nil)

	}
}

func (srh *StandardRequestHandler) Post(w http.ResponseWriter,  r *http.Request, tpl *template.Template, data url.Values, template_path string) {
	sender := os.Getenv("EMAIL_SENDER")
	subject := os.Getenv("EMAIL_SUBJECT")

	email := data.Get("email")
	name := data.Get("name")

	if name != "" && email != "" {
		log.Print(name)
		log.Print(email)

		// Create goroutine to send email via
		go func(){
			mailer := &StandardEmailer{template_path}
			msg, err := mailer.CreateMessage(sender, subject, r.FormValue("name"), r.FormValue("email"))
			if err != nil {
				log.Print("Could not create email: %v", err)
			}
			mailer.Send(msg)
			time.Sleep(5000 * time.Millisecond)

		}()

		// OK
	}
	tpl.Lookup("post.tpl").Execute(w, data)
}

type BaseHandler struct {
	TemplatePath	string
}

func (bh *BaseHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	bh.serve(w, req, new(StandardRequestHandler))
}

func (bh *BaseHandler) serve(w http.ResponseWriter, r *http.Request, rh RequestHandler) {
	tpl, err := template.ParseFiles(fmt.Sprintf("%s/main.tpl", bh.TemplatePath), fmt.Sprintf("%s/post.tpl", bh.TemplatePath), fmt.Sprintf("%s/terms.tpl", bh.TemplatePath), fmt.Sprintf("%s/privacy.tpl", bh.TemplatePath), fmt.Sprintf("%s/header.tpl", bh.TemplatePath), fmt.Sprintf("%s/footer.tpl", bh.TemplatePath))
	if err != nil {
		http.Error(w, "Internal error", http.StatusInternalServerError)
		log.Print(err.Error())
		return
	}
	switch r.Method {
	case "GET":
		rh.Get(w, r, r.URL.Path[len("/"):], tpl)

	case "POST":
		err := r.ParseForm()
		if err != nil {
			http.Error(w, "Internal error", http.StatusInternalServerError)
			return
		}
		rh.Post(w, r, tpl, r.PostForm, bh.TemplatePath)
	default:
		http.Error(w, "Method not implemented", http.StatusInternalServerError)
	}
}
<|endoftext|>