<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"math"
	"strconv"
	"sync/atomic"
	"time"

	"github.com/dustin/go-couchstore"
	"github.com/dustin/go-jsonpointer"
)

var timeoutError = errors.New("query timed out")

type processOut struct {
	key   int64
	value interface{}
	err   error
}

type processIn struct {
	dbname string
	key    int64
	infos  []*couchstore.DocInfo
	ptrs   []string
	reds   []Reducer
	before time.Time
	out    chan<- processOut
}

type queryIn struct {
	dbname    string
	from      string
	to        string
	group     int
	ptrs      []string
	reds      []Reducer
	start     time.Time
	before    time.Time
	started   int32
	totalKeys int32
	out       chan processOut
	cherr     chan error
}

func processDoc(collection [][]*string, doc string, ptrs []string) {

	j := map[string]interface{}{}
	err := json.Unmarshal([]byte(doc), &j)
	if err != nil {
		for i := range ptrs {
			collection[i] = append(collection[i], nil)
		}
		return
	}
	for i, p := range ptrs {
		val := jsonpointer.Get(j, p)
		switch x := val.(type) {
		case string:
			collection[i] = append(collection[i], &x)
		case int, uint, int64, float64, uint64, bool:
			v := fmt.Sprintf("%v", val)
			collection[i] = append(collection[i], &v)
		default:
			log.Printf("Ignoring %T", val)
			collection[i] = append(collection[i], nil)
		}
	}
}

func process_docs(dbname string, key int64, infos []*couchstore.DocInfo,
	ptrs []string, reds []Reducer, ch chan<- processOut) {

	result := processOut{key, nil, nil}

	db, err := dbopen(dbname)
	if err != nil {
		result.err = err
		ch <- result
		return
	}
	defer db.Close()

	collection := make([][]*string, len(ptrs))

	for _, di := range infos {
		doc, err := db.GetFromDocInfo(di)
		if err == nil {
			processDoc(collection, doc.Value(), ptrs)
		} else {
			for i := range collection {
				collection[i] = append(collection[i], nil)
			}
		}
	}

	result.value = reduce(collection, reds)

	ch <- result
}

func docProcessor(ch <-chan processIn) {
	for pi := range ch {
		if time.Now().Before(pi.before) {
			process_docs(pi.dbname, pi.key, pi.infos, pi.ptrs,
				pi.reds, pi.out)
		} else {
			pi.out <- processOut{pi.key, nil, timeoutError}
		}
	}
}

func runQuery(q *queryIn) {
	db, err := dbopen(q.dbname)
	if err != nil {
		log.Printf("Error opening db: %v - %v", q.dbname, err)
		q.cherr <- err
		return
	}
	defer db.Close()

	chunk := int64(time.Duration(q.group) * time.Millisecond)

	infos := []*couchstore.DocInfo{}
	prevg := int64(0)

	err = db.Walk(q.from, func(d *couchstore.Couchstore,
		di *couchstore.DocInfo) error {
		if q.to != "" && di.ID() >= q.to {
			return couchstore.StopIteration
		}

		k := parseKey(di.ID())
		g := (k / chunk) * chunk
		atomic.AddInt32(&q.totalKeys, 1)

		if g != prevg && len(infos) > 0 {
			atomic.AddInt32(&q.started, 1)
			processorInput <- processIn{q.dbname, prevg, infos,
				q.ptrs, q.reds, q.before, q.out}

			infos = infos[:0]
		}
		infos = append(infos, di)
		prevg = g

		return nil
	})

	if err == nil && len(infos) > 0 {
		atomic.AddInt32(&q.started, 1)
		processorInput <- processIn{q.dbname, prevg, infos,
			q.ptrs, q.reds, q.before, q.out}
	}

	q.cherr <- err
}

func queryExecutor(ch <-chan *queryIn) {
	for q := range ch {
		if time.Now().Before(q.before) {
			runQuery(q)
		} else {
			log.Printf("Timed out query that's %v late",
				time.Since(q.before))
			q.cherr <- timeoutError
		}
	}
}

func executeQuery(dbname, from, to string, group int,
	ptrs []string, reds []Reducer) *queryIn {

	now := time.Now()

	rv := &queryIn{
		dbname: dbname,
		from:   from,
		to:     to,
		group:  group,
		ptrs:   ptrs,
		reds:   reds,
		start:  now,
		before: now.Add(*queryTimeout),
		out:    make(chan processOut),
		cherr:  make(chan error),
	}
	queryInput <- rv
	return rv
}

var processorInput chan processIn
var queryInput chan *queryIn

type Reducer func(input []*string) interface{}

func reduce(collection [][]*string, reducers []Reducer) []interface{} {
	rv := make([]interface{}, len(collection))
	for i, a := range collection {
		rv[i] = reducers[i](a)
	}
	return rv
}

func convertTofloat64(in []*string) []float64 {
	rv := make([]float64, 0, len(in))
	for _, v := range in {
		if v != nil {
			x, err := strconv.ParseFloat(*v, 64)
			if err == nil {
				rv = append(rv, x)
			}
		}
	}
	return rv
}

var reducers = map[string]Reducer{
	"identity": func(input []*string) interface{} {
		return input
	},
	"any": func(input []*string) interface{} {
		for _, v := range input {
			if v != nil {
				return *v
			}
		}
		return nil
	},
	"count": func(input []*string) interface{} {
		rv := 0
		for _, v := range input {
			if v != nil {
				rv++
			}
		}
		return rv
	},
	"sum": func(input []*string) interface{} {
		rv := float64(0)
		for _, v := range convertTofloat64(input) {
			rv += v
		}
		return rv
	},
	"sumsq": func(input []*string) interface{} {
		rv := float64(0)
		for _, v := range convertTofloat64(input) {
			rv += (v * v)
		}
		return rv
	},
	"max": func(input []*string) interface{} {
		rv := float64(math.MinInt64)
		for _, v := range convertTofloat64(input) {
			if v > rv {
				rv = v
			}
		}
		return rv
	},
	"min": func(input []*string) interface{} {
		rv := float64(math.MaxInt64)
		for _, v := range convertTofloat64(input) {
			if v < rv {
				rv = v
			}
		}
		return rv
	},
	"avg": func(input []*string) interface{} {
		nums := convertTofloat64(input)
		sum := float64(0)
		for _, v := range nums {
			sum += v
		}
		return float64(sum) / float64(len(nums))
	},
}
<commit_msg>Allocate a new collection of docs on each spin.<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"math"
	"strconv"
	"sync/atomic"
	"time"

	"github.com/dustin/go-couchstore"
	"github.com/dustin/go-jsonpointer"
)

var timeoutError = errors.New("query timed out")

type processOut struct {
	key   int64
	value interface{}
	err   error
}

type processIn struct {
	dbname string
	key    int64
	infos  []*couchstore.DocInfo
	ptrs   []string
	reds   []Reducer
	before time.Time
	out    chan<- processOut
}

type queryIn struct {
	dbname    string
	from      string
	to        string
	group     int
	ptrs      []string
	reds      []Reducer
	start     time.Time
	before    time.Time
	started   int32
	totalKeys int32
	out       chan processOut
	cherr     chan error
}

func processDoc(collection [][]*string, doc string, ptrs []string) {

	j := map[string]interface{}{}
	err := json.Unmarshal([]byte(doc), &j)
	if err != nil {
		for i := range ptrs {
			collection[i] = append(collection[i], nil)
		}
		return
	}
	for i, p := range ptrs {
		val := jsonpointer.Get(j, p)
		switch x := val.(type) {
		case string:
			collection[i] = append(collection[i], &x)
		case int, uint, int64, float64, uint64, bool:
			v := fmt.Sprintf("%v", val)
			collection[i] = append(collection[i], &v)
		default:
			log.Printf("Ignoring %T", val)
			collection[i] = append(collection[i], nil)
		}
	}
}

func process_docs(dbname string, key int64, infos []*couchstore.DocInfo,
	ptrs []string, reds []Reducer, ch chan<- processOut) {

	result := processOut{key, nil, nil}

	db, err := dbopen(dbname)
	if err != nil {
		result.err = err
		ch <- result
		return
	}
	defer db.Close()

	collection := make([][]*string, len(ptrs))

	for _, di := range infos {
		doc, err := db.GetFromDocInfo(di)
		if err == nil {
			processDoc(collection, doc.Value(), ptrs)
		} else {
			for i := range collection {
				collection[i] = append(collection[i], nil)
			}
		}
	}

	result.value = reduce(collection, reds)

	ch <- result
}

func docProcessor(ch <-chan processIn) {
	for pi := range ch {
		if time.Now().Before(pi.before) {
			process_docs(pi.dbname, pi.key, pi.infos, pi.ptrs,
				pi.reds, pi.out)
		} else {
			pi.out <- processOut{pi.key, nil, timeoutError}
		}
	}
}

func runQuery(q *queryIn) {
	db, err := dbopen(q.dbname)
	if err != nil {
		log.Printf("Error opening db: %v - %v", q.dbname, err)
		q.cherr <- err
		return
	}
	defer db.Close()

	chunk := int64(time.Duration(q.group) * time.Millisecond)

	infos := []*couchstore.DocInfo{}
	prevg := int64(0)

	err = db.Walk(q.from, func(d *couchstore.Couchstore,
		di *couchstore.DocInfo) error {
		if q.to != "" && di.ID() >= q.to {
			return couchstore.StopIteration
		}

		k := parseKey(di.ID())
		g := (k / chunk) * chunk
		atomic.AddInt32(&q.totalKeys, 1)

		if g != prevg && len(infos) > 0 {
			atomic.AddInt32(&q.started, 1)
			processorInput <- processIn{q.dbname, prevg, infos,
				q.ptrs, q.reds, q.before, q.out}

			infos = make([]*couchstore.DocInfo, 0, len(infos))
		}
		infos = append(infos, di)
		prevg = g

		return nil
	})

	if err == nil && len(infos) > 0 {
		atomic.AddInt32(&q.started, 1)
		processorInput <- processIn{q.dbname, prevg, infos,
			q.ptrs, q.reds, q.before, q.out}
	}

	q.cherr <- err
}

func queryExecutor(ch <-chan *queryIn) {
	for q := range ch {
		if time.Now().Before(q.before) {
			runQuery(q)
		} else {
			log.Printf("Timed out query that's %v late",
				time.Since(q.before))
			q.cherr <- timeoutError
		}
	}
}

func executeQuery(dbname, from, to string, group int,
	ptrs []string, reds []Reducer) *queryIn {

	now := time.Now()

	rv := &queryIn{
		dbname: dbname,
		from:   from,
		to:     to,
		group:  group,
		ptrs:   ptrs,
		reds:   reds,
		start:  now,
		before: now.Add(*queryTimeout),
		out:    make(chan processOut),
		cherr:  make(chan error),
	}
	queryInput <- rv
	return rv
}

var processorInput chan processIn
var queryInput chan *queryIn

type Reducer func(input []*string) interface{}

func reduce(collection [][]*string, reducers []Reducer) []interface{} {
	rv := make([]interface{}, len(collection))
	for i, a := range collection {
		rv[i] = reducers[i](a)
	}
	return rv
}

func convertTofloat64(in []*string) []float64 {
	rv := make([]float64, 0, len(in))
	for _, v := range in {
		if v != nil {
			x, err := strconv.ParseFloat(*v, 64)
			if err == nil {
				rv = append(rv, x)
			}
		}
	}
	return rv
}

var reducers = map[string]Reducer{
	"identity": func(input []*string) interface{} {
		return input
	},
	"any": func(input []*string) interface{} {
		for _, v := range input {
			if v != nil {
				return *v
			}
		}
		return nil
	},
	"count": func(input []*string) interface{} {
		rv := 0
		for _, v := range input {
			if v != nil {
				rv++
			}
		}
		return rv
	},
	"sum": func(input []*string) interface{} {
		rv := float64(0)
		for _, v := range convertTofloat64(input) {
			rv += v
		}
		return rv
	},
	"sumsq": func(input []*string) interface{} {
		rv := float64(0)
		for _, v := range convertTofloat64(input) {
			rv += (v * v)
		}
		return rv
	},
	"max": func(input []*string) interface{} {
		rv := float64(math.MinInt64)
		for _, v := range convertTofloat64(input) {
			if v > rv {
				rv = v
			}
		}
		return rv
	},
	"min": func(input []*string) interface{} {
		rv := float64(math.MaxInt64)
		for _, v := range convertTofloat64(input) {
			if v < rv {
				rv = v
			}
		}
		return rv
	},
	"avg": func(input []*string) interface{} {
		nums := convertTofloat64(input)
		sum := float64(0)
		for _, v := range nums {
			sum += v
		}
		return float64(sum) / float64(len(nums))
	},
}
<|endoftext|>


<commit_before>package qmd

import (
	"fmt"

	"github.com/bitly/go-nsq"
)

type QueueConfig struct {
	HostNSQDAddr string   `toml:"host_nsqd_address"`
	NSQDAddrs    []string `toml:"nsqd_addresses"`
	LookupdAddrs []string `toml:"lookupd_addresses"`
}

func (qc *QueueConfig) Clean() error {
	if len(qc.LookupdAddrs) == 0 {
		if len(qc.NSQDAddrs) == 0 {
			return fmt.Errorf("Both LookupdAddresses and NSQDAddresses are missing")
		}
	}
	return nil
}

func ConnectConsumer(qc *QueueConfig, consumer *nsq.Consumer) error {
	var err error

	// Connect consumers to NSQLookupd
	if qc.LookupdAddrs != nil || len(qc.LookupdAddrs) != 0 {
		log.Info("Connecting Consumer to the following NSQLookupds %s", qc.LookupdAddrs)
		err = consumer.ConnectToNSQLookupds(qc.LookupdAddrs)
		if err != nil {
			return err
		}
	}
	// Connect consumers to NSQD
	if qc.NSQDAddrs != nil || len(qc.NSQDAddrs) != 0 {
		log.Info("Connecting Consumer to the following NSQDs %s", qc.NSQDAddrs)
		err = consumer.ConnectToNSQDs(qc.NSQDAddrs)
		if err != nil {
			return err
		}
	}
	return fmt.Errorf("Couldn't connect to either NSQDs or NSQLookupds")
}
<commit_msg>Fixed typo<commit_after>package qmd

import (
	"fmt"

	"github.com/alexcesaro/log"
	"github.com/bitly/go-nsq"
)

type QueueConfig struct {
	HostNSQDAddr string   `toml:"host_nsqd_address"`
	NSQDAddrs    []string `toml:"nsqd_addresses"`
	LookupdAddrs []string `toml:"lookupd_addresses"`
}

func (qc *QueueConfig) Clean() error {
	if len(qc.LookupdAddrs) == 0 {
		if len(qc.NSQDAddrs) == 0 {
			return fmt.Errorf("Both LookupdAddresses and NSQDAddresses are missing")
		}
	}
	return nil
}

func ConnectConsumer(qc *QueueConfig, consumer *nsq.Consumer) error {
	var err error

	// Connect consumers to NSQLookupd
	if qc.LookupdAddrs != nil && len(qc.LookupdAddrs) != 0 {
		log.Info("Connecting Consumer to the following NSQLookupds %s", qc.LookupdAddrs)
		err = consumer.ConnectToNSQLookupds(qc.LookupdAddrs)
		if err != nil {
			return err
		}
	}
	// Connect consumers to NSQD
	if qc.NSQDAddrs != nil && len(qc.NSQDAddrs) != 0 {
		log.Info("Connecting Consumer to the following NSQDs %s", qc.NSQDAddrs)
		err = consumer.ConnectToNSQDs(qc.NSQDAddrs)
		if err != nil {
			return err
		}
	}
	return fmt.Errorf("Couldn't connect to either NSQDs or NSQLookupds")
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package shared

import (
	"context"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/google/go-containerregistry/pkg/name"
	"golang.org/x/sync/errgroup"
	appsv1 "k8s.io/api/apps/v1"

	pkgTest "knative.dev/pkg/test"
	"knative.dev/pkg/test/spoof"
	"knative.dev/serving/test"
)

const scaleToZeroGracePeriod = 30 * time.Second

// WaitForScaleToZero will wait for the specified deployment to scale to 0 replicas.
// Will wait up to 6 times the scaleToZeroGracePeriod (30 seconds) before failing.
func WaitForScaleToZero(t pkgTest.TLegacy, deploymentName string, clients *test.Clients) error {
	t.Helper()
	t.Logf("Waiting for %q to scale to zero", deploymentName)

	return pkgTest.WaitForDeploymentState(
		context.Background(),
		clients.KubeClient,
		deploymentName,
		func(d *appsv1.Deployment) (bool, error) {
			return d.Status.ReadyReplicas == 0, nil
		},
		"DeploymentIsScaledDown",
		test.ServingNamespace,
		scaleToZeroGracePeriod*6,
	)
}

// ValidateImageDigest validates the image digest.
func ValidateImageDigest(imageName string, imageDigest string) (bool, error) {
	ref, err := name.ParseReference(pkgTest.ImagePath(imageName))
	if err != nil {
		return false, err
	}

	digest, err := name.NewDigest(imageDigest)
	if err != nil {
		return false, err
	}

	return ref.Context().String() == digest.Context().String(), nil
}

// sendRequests sends "num" requests to "url", returning a string for each spoof.Response.Body.
func sendRequests(client spoof.Interface, url *url.URL, num int) ([]string, error) {
	responses := make([]string, num)

	// Launch "num" requests, recording the responses we get in "responses".
	g, _ := errgroup.WithContext(context.Background())
	for i := 0; i < num; i++ {
		// We don't index into "responses" inside the goroutine to avoid a race, see #1545.
		result := &responses[i]
		g.Go(func() error {
			req, err := http.NewRequest(http.MethodGet, url.String(), nil)
			if err != nil {
				return err
			}

			resp, err := client.Do(req)
			if err != nil {
				return err
			}

			*result = string(resp.Body)
			return nil
		})
	}
	return responses, g.Wait()
}

func substrInList(key string, targets []string) string {
	for _, t := range targets {
		if strings.Contains(key, t) {
			return t
		}
	}
	return ""
}

// checkResponses verifies that each "expectedResponse" is present in "actualResponses" at least "min" times.
func checkResponses(t pkgTest.TLegacy, num, min int, domain string, expectedResponses, actualResponses []string) error {
	// counts maps the expected response body to the number of matching requests we saw.
	counts := make(map[string]int, len(expectedResponses))
	// badCounts maps the unexpected response body to the number of matching requests we saw.
	badCounts := make(map[string]int)

	// counts := eval(
	//   SELECT body, count(*) AS total
	//   FROM $actualResponses
	//   WHERE body IN $expectedResponses
	//   GROUP BY body
	// )
	for i, ar := range actualResponses {
		if er := substrInList(ar, expectedResponses); er != "" {
			counts[er]++
		} else {
			badCounts[ar]++
			t.Logf("For domain %s: got unexpected response for request %d", domain, i)
		}
	}

	// Print unexpected responses for debugging purposes
	for badResponse, count := range badCounts {
		t.Logf("For domain %s: saw unexpected response %q %d times.", domain, badResponse, count)
	}

	// Verify that we saw each entry in "expectedResponses" at least "min" times.
	// check(SELECT body FROM $counts WHERE total < $min)
	totalMatches := 0
	errMsg := []string{}
	for _, er := range expectedResponses {
		count := counts[er]
		if count < min {
			errMsg = append(errMsg,
				fmt.Sprintf("domain %s failed: want at least %d, got %d for response %q",
					domain, min, count, er))
		}

		t.Logf("For domain %s: wanted at least %d, got %d requests.", domain, min, count)
		totalMatches += count
	}
	// Verify that the total expected responses match the number of requests made.
	if totalMatches < num {
		errMsg = append(errMsg,
			fmt.Sprintf("domain %s: saw expected responses %d times, wanted %d", domain, totalMatches, num))
	}
	if len(errMsg) == 0 {
		// If we made it here, the implementation conforms. Congratulations!
		return nil
	}
	return errors.New(strings.Join(errMsg, ","))
}

// CheckDistribution sends "num" requests to "domain", then validates that
// we see each body in "expectedResponses" at least "min" times.
func CheckDistribution(t pkgTest.TLegacy, clients *test.Clients, url *url.URL, num, min int, expectedResponses []string) error {
	ctx := context.Background()
	client, err := pkgTest.NewSpoofingClient(ctx, clients.KubeClient, t.Logf, url.Hostname(), test.ServingFlags.ResolvableDomain, test.AddRootCAtoTransport(ctx, t.Logf, clients, test.ServingFlags.Https))
	if err != nil {
		return err
	}

	t.Logf("Performing %d concurrent requests to %s", num, url)
	actualResponses, err := sendRequests(client, url, num)
	if err != nil {
		return err
	}

	return checkResponses(t, num, min, url.Hostname(), expectedResponses, actualResponses)
}
<commit_msg>Remove downstream usage of spoof.Interface (#9401)<commit_after>/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package shared

import (
	"context"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/google/go-containerregistry/pkg/name"
	"golang.org/x/sync/errgroup"
	appsv1 "k8s.io/api/apps/v1"

	pkgTest "knative.dev/pkg/test"
	"knative.dev/pkg/test/spoof"
	"knative.dev/serving/test"
)

const scaleToZeroGracePeriod = 30 * time.Second

// WaitForScaleToZero will wait for the specified deployment to scale to 0 replicas.
// Will wait up to 6 times the scaleToZeroGracePeriod (30 seconds) before failing.
func WaitForScaleToZero(t pkgTest.TLegacy, deploymentName string, clients *test.Clients) error {
	t.Helper()
	t.Logf("Waiting for %q to scale to zero", deploymentName)

	return pkgTest.WaitForDeploymentState(
		context.Background(),
		clients.KubeClient,
		deploymentName,
		func(d *appsv1.Deployment) (bool, error) {
			return d.Status.ReadyReplicas == 0, nil
		},
		"DeploymentIsScaledDown",
		test.ServingNamespace,
		scaleToZeroGracePeriod*6,
	)
}

// ValidateImageDigest validates the image digest.
func ValidateImageDigest(imageName string, imageDigest string) (bool, error) {
	ref, err := name.ParseReference(pkgTest.ImagePath(imageName))
	if err != nil {
		return false, err
	}

	digest, err := name.NewDigest(imageDigest)
	if err != nil {
		return false, err
	}

	return ref.Context().String() == digest.Context().String(), nil
}

// sendRequests sends "num" requests to "url", returning a string for each spoof.Response.Body.
func sendRequests(client *spoof.SpoofingClient, url *url.URL, num int) ([]string, error) {
	responses := make([]string, num)

	// Launch "num" requests, recording the responses we get in "responses".
	g, _ := errgroup.WithContext(context.Background())
	for i := 0; i < num; i++ {
		// We don't index into "responses" inside the goroutine to avoid a race, see #1545.
		result := &responses[i]
		g.Go(func() error {
			req, err := http.NewRequest(http.MethodGet, url.String(), nil)
			if err != nil {
				return err
			}

			resp, err := client.Do(req)
			if err != nil {
				return err
			}

			*result = string(resp.Body)
			return nil
		})
	}
	return responses, g.Wait()
}

func substrInList(key string, targets []string) string {
	for _, t := range targets {
		if strings.Contains(key, t) {
			return t
		}
	}
	return ""
}

// checkResponses verifies that each "expectedResponse" is present in "actualResponses" at least "min" times.
func checkResponses(t pkgTest.TLegacy, num, min int, domain string, expectedResponses, actualResponses []string) error {
	// counts maps the expected response body to the number of matching requests we saw.
	counts := make(map[string]int, len(expectedResponses))
	// badCounts maps the unexpected response body to the number of matching requests we saw.
	badCounts := make(map[string]int)

	// counts := eval(
	//   SELECT body, count(*) AS total
	//   FROM $actualResponses
	//   WHERE body IN $expectedResponses
	//   GROUP BY body
	// )
	for i, ar := range actualResponses {
		if er := substrInList(ar, expectedResponses); er != "" {
			counts[er]++
		} else {
			badCounts[ar]++
			t.Logf("For domain %s: got unexpected response for request %d", domain, i)
		}
	}

	// Print unexpected responses for debugging purposes
	for badResponse, count := range badCounts {
		t.Logf("For domain %s: saw unexpected response %q %d times.", domain, badResponse, count)
	}

	// Verify that we saw each entry in "expectedResponses" at least "min" times.
	// check(SELECT body FROM $counts WHERE total < $min)
	totalMatches := 0
	errMsg := []string{}
	for _, er := range expectedResponses {
		count := counts[er]
		if count < min {
			errMsg = append(errMsg,
				fmt.Sprintf("domain %s failed: want at least %d, got %d for response %q",
					domain, min, count, er))
		}

		t.Logf("For domain %s: wanted at least %d, got %d requests.", domain, min, count)
		totalMatches += count
	}
	// Verify that the total expected responses match the number of requests made.
	if totalMatches < num {
		errMsg = append(errMsg,
			fmt.Sprintf("domain %s: saw expected responses %d times, wanted %d", domain, totalMatches, num))
	}
	if len(errMsg) == 0 {
		// If we made it here, the implementation conforms. Congratulations!
		return nil
	}
	return errors.New(strings.Join(errMsg, ","))
}

// CheckDistribution sends "num" requests to "domain", then validates that
// we see each body in "expectedResponses" at least "min" times.
func CheckDistribution(t pkgTest.TLegacy, clients *test.Clients, url *url.URL, num, min int, expectedResponses []string) error {
	ctx := context.Background()
	client, err := pkgTest.NewSpoofingClient(ctx, clients.KubeClient, t.Logf, url.Hostname(), test.ServingFlags.ResolvableDomain, test.AddRootCAtoTransport(ctx, t.Logf, clients, test.ServingFlags.Https))
	if err != nil {
		return err
	}

	t.Logf("Performing %d concurrent requests to %s", num, url)
	actualResponses, err := sendRequests(client, url, num)
	if err != nil {
		return err
	}

	return checkResponses(t, num, min, url.Hostname(), expectedResponses, actualResponses)
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"go/ast"
	"go/printer"
	"os"
	"strings"
)

func creat(name string) *os.File {
	f, err := os.Open(name, os.O_WRONLY|os.O_CREAT|os.O_TRUNC, 0666)
	if err != nil {
		fatal("%s", err)
	}
	return f
}

// writeDefs creates output files to be compiled by 6g, 6c, and gcc.
// (The comments here say 6g and 6c but the code applies to the 8 and 5 tools too.)
func (p *Prog) writeDefs() {
	pkgroot := os.Getenv("GOROOT") + "/pkg/" + os.Getenv("GOOS") + "_" + os.Getenv("GOARCH")
	path := p.PackagePath
	if !strings.HasPrefix(path, "/") {
		path = pkgroot + "/" + path
	}

	fgo2 := creat("_cgo_gotypes.go")
	fc := creat("_cgo_defun.c")

	// Write second Go output: definitions of _C_xxx.
	// In a separate file so that the import of "unsafe" does not
	// pollute the original file.
	fmt.Fprintf(fgo2, "// Created by cgo - DO NOT EDIT\n")
	fmt.Fprintf(fgo2, "package %s\n\n", p.Package)
	fmt.Fprintf(fgo2, "import \"unsafe\"\n\n")
	fmt.Fprintf(fgo2, "type _ unsafe.Pointer\n\n")

	for name, def := range p.Typedef {
		fmt.Fprintf(fgo2, "type %s ", name)
		printer.Fprint(fgo2, def)
		fmt.Fprintf(fgo2, "\n")
	}
	fmt.Fprintf(fgo2, "type _C_void [0]byte\n")

	fmt.Fprintf(fc, cProlog, pkgroot, pkgroot, pkgroot, pkgroot, p.Package, p.Package)

	for name, def := range p.Vardef {
		fmt.Fprintf(fc, "#pragma dynld %s·_C_%s %s \"%s.so\"\n", p.Package, name, name, path)
		fmt.Fprintf(fgo2, "var _C_%s ", name)
		printer.Fprint(fgo2, &ast.StarExpr{X: def.Go})
		fmt.Fprintf(fgo2, "\n")
	}
	fmt.Fprintf(fc, "\n")

	for name, value := range p.Constdef {
		fmt.Fprintf(fgo2, "const %s = %s\n", name, value)
	}

	for name, value := range p.Enumdef {
		fmt.Fprintf(fgo2, "const %s = %d\n", name, value)
	}
	fmt.Fprintf(fgo2, "\n")

	for name, def := range p.Funcdef {
		// Go func declaration.
		d := &ast.FuncDecl{
			Name: ast.NewIdent("_C_" + name),
			Type: def.Go,
		}
		printer.Fprint(fgo2, d)
		fmt.Fprintf(fgo2, "\n")

		if name == "CString" || name == "GoString" {
			// The builtins are already defined in the C prolog.
			continue
		}

		// Construct a gcc struct matching the 6c argument frame.
		// Assumes that in gcc, char is 1 byte, short 2 bytes, int 4 bytes, long long 8 bytes.
		// These assumptions are checked by the gccProlog.
		// Also assumes that 6c convention is to word-align the
		// input and output parameters.
		structType := "struct {\n"
		off := int64(0)
		npad := 0
		for i, t := range def.Params {
			if off%t.Align != 0 {
				pad := t.Align - off%t.Align
				structType += fmt.Sprintf("\t\tchar __pad%d[%d];\n", npad, pad)
				off += pad
				npad++
			}
			structType += fmt.Sprintf("\t\t%s p%d;\n", t.C, i)
			off += t.Size
		}
		if off%p.PtrSize != 0 {
			pad := p.PtrSize - off%p.PtrSize
			structType += fmt.Sprintf("\t\tchar __pad%d[%d];\n", npad, pad)
			off += pad
			npad++
		}
		if t := def.Result; t != nil {
			if off%t.Align != 0 {
				pad := t.Align - off%t.Align
				structType += fmt.Sprintf("\t\tchar __pad%d[%d];\n", npad, pad)
				off += pad
				npad++
			}
			structType += fmt.Sprintf("\t\t%s r;\n", t.C)
			off += t.Size
		}
		if off%p.PtrSize != 0 {
			pad := p.PtrSize - off%p.PtrSize
			structType += fmt.Sprintf("\t\tchar __pad%d[%d];\n", npad, pad)
			off += pad
			npad++
		}
		if len(def.Params) == 0 && def.Result == nil {
			structType += "\t\tchar unused;\n" // avoid empty struct
			off++
		}
		structType += "\t}"
		argSize := off

		// C wrapper calls into gcc, passing a pointer to the argument frame.
		// Also emit #pragma to get a pointer to the gcc wrapper.
		fmt.Fprintf(fc, "#pragma dynld _cgo_%s _cgo_%s \"%s.so\"\n", name, name, path)
		fmt.Fprintf(fc, "void (*_cgo_%s)(void*);\n", name)
		fmt.Fprintf(fc, "\n")
		fmt.Fprintf(fc, "void\n")
		fmt.Fprintf(fc, "%s·_C_%s(struct{uint8 x[%d];}p)\n", p.Package, name, argSize)
		fmt.Fprintf(fc, "{\n")
		fmt.Fprintf(fc, "\tcgocall(_cgo_%s, &p);\n", name)
		fmt.Fprintf(fc, "}\n")
		fmt.Fprintf(fc, "\n")
	}

	fgo2.Close()
	fc.Close()
}

// writeOutput creates stubs for a specific source file to be compiled by 6g
// (The comments here say 6g and 6c but the code applies to the 8 and 5 tools too.)
func (p *Prog) writeOutput(srcfile string) {
	base := srcfile
	if strings.HasSuffix(base, ".go") {
		base = base[0 : len(base)-3]
	}
	fgo1 := creat(base + ".cgo1.go")
	fgcc := creat(base + ".cgo2.c")

	// Write Go output: Go input with rewrites of C.xxx to _C_xxx.
	fmt.Fprintf(fgo1, "// Created by cgo - DO NOT EDIT\n")
	fmt.Fprintf(fgo1, "//line %s:1\n", srcfile)
	printer.Fprint(fgo1, p.AST)

	// While we process the vars and funcs, also write 6c and gcc output.
	// Gcc output starts with the preamble.
	fmt.Fprintf(fgcc, "%s\n", p.Preamble)
	fmt.Fprintf(fgcc, "%s\n", gccProlog)

	for name, def := range p.Funcdef {
		_, ok := p.OutDefs[name]
		if name == "CString" || name == "GoString" || ok {
			// The builtins are already defined in the C prolog, and we don't
			// want to duplicate function definitions we've already done.
			continue
		}
		p.OutDefs[name] = true

		// Construct a gcc struct matching the 6c argument frame.
		// Assumes that in gcc, char is 1 byte, short 2 bytes, int 4 bytes, long long 8 bytes.
		// These assumptions are checked by the gccProlog.
		// Also assumes that 6c convention is to word-align the
		// input and output parameters.
		structType := "struct {\n"
		off := int64(0)
		npad := 0
		for i, t := range def.Params {
			if off%t.Align != 0 {
				pad := t.Align - off%t.Align
				structType += fmt.Sprintf("\t\tchar __pad%d[%d];\n", npad, pad)
				off += pad
				npad++
			}
			structType += fmt.Sprintf("\t\t%s p%d;\n", t.C, i)
			off += t.Size
		}
		if off%p.PtrSize != 0 {
			pad := p.PtrSize - off%p.PtrSize
			structType += fmt.Sprintf("\t\tchar __pad%d[%d];\n", npad, pad)
			off += pad
			npad++
		}
		if t := def.Result; t != nil {
			if off%t.Align != 0 {
				pad := t.Align - off%t.Align
				structType += fmt.Sprintf("\t\tchar __pad%d[%d];\n", npad, pad)
				off += pad
				npad++
			}
			structType += fmt.Sprintf("\t\t%s r;\n", t.C)
			off += t.Size
		}
		if off%p.PtrSize != 0 {
			pad := p.PtrSize - off%p.PtrSize
			structType += fmt.Sprintf("\t\tchar __pad%d[%d];\n", npad, pad)
			off += pad
			npad++
		}
		if len(def.Params) == 0 && def.Result == nil {
			structType += "\t\tchar unused;\n" // avoid empty struct
			off++
		}
		structType += "\t}"

		// Gcc wrapper unpacks the C argument struct
		// and calls the actual C function.
		fmt.Fprintf(fgcc, "void\n")
		fmt.Fprintf(fgcc, "_cgo_%s(void *v)\n", name)
		fmt.Fprintf(fgcc, "{\n")
		fmt.Fprintf(fgcc, "\t%s *a = v;\n", structType)
		fmt.Fprintf(fgcc, "\t")
		if def.Result != nil {
			fmt.Fprintf(fgcc, "a->r = ")
		}
		fmt.Fprintf(fgcc, "%s(", name)
		for i := range def.Params {
			if i > 0 {
				fmt.Fprintf(fgcc, ", ")
			}
			fmt.Fprintf(fgcc, "a->p%d", i)
		}
		fmt.Fprintf(fgcc, ");\n")
		fmt.Fprintf(fgcc, "}\n")
		fmt.Fprintf(fgcc, "\n")
	}

	fgo1.Close()
	fgcc.Close()
}

const gccProlog = `
// Usual nonsense: if x and y are not equal, the type will be invalid
// (have a negative array count) and an inscrutable error will come
// out of the compiler and hopefully mention "name".
#define __cgo_compile_assert_eq(x, y, name) typedef char name[(x-y)*(x-y)*-2+1];

// Check at compile time that the sizes we use match our expectations.
#define __cgo_size_assert(t, n) __cgo_compile_assert_eq(sizeof(t), n, _cgo_sizeof_##t##_is_not_##n)

__cgo_size_assert(char, 1)
__cgo_size_assert(short, 2)
__cgo_size_assert(int, 4)
typedef long long __cgo_long_long;
__cgo_size_assert(__cgo_long_long, 8)
__cgo_size_assert(float, 4)
__cgo_size_assert(double, 8)
`

const builtinProlog = `
typedef struct { char *p; int n; } _GoString_;
_GoString_ GoString(char *p);
char *CString(_GoString_);
`

const cProlog = `
#include "runtime.h"
#include "cgocall.h"

#pragma dynld initcgo initcgo "%s/libcgo.so"
#pragma dynld libcgo_thread_start libcgo_thread_start "%s/libcgo.so"
#pragma dynld _cgo_malloc _cgo_malloc "%s/libcgo.so"
#pragma dynld _cgo_free free "%s/libcgo.so"

void
%s·_C_GoString(int8 *p, String s)
{
	s = gostring((byte*)p);
	FLUSH(&s);
}

void
%s·_C_CString(String s, int8 *p)
{
	p = cmalloc(s.len+1);
	mcpy((byte*)p, s.str, s.len);
	p[s.len] = 0;
	FLUSH(&p);
}
`
<commit_msg>cgo: update for package global name space<commit_after>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"go/ast"
	"go/printer"
	"os"
	"strings"
)

func creat(name string) *os.File {
	f, err := os.Open(name, os.O_WRONLY|os.O_CREAT|os.O_TRUNC, 0666)
	if err != nil {
		fatal("%s", err)
	}
	return f
}

// writeDefs creates output files to be compiled by 6g, 6c, and gcc.
// (The comments here say 6g and 6c but the code applies to the 8 and 5 tools too.)
func (p *Prog) writeDefs() {
	pkgroot := os.Getenv("GOROOT") + "/pkg/" + os.Getenv("GOOS") + "_" + os.Getenv("GOARCH")
	path := p.PackagePath
	if !strings.HasPrefix(path, "/") {
		path = pkgroot + "/" + path
	}

	fgo2 := creat("_cgo_gotypes.go")
	fc := creat("_cgo_defun.c")

	// Write second Go output: definitions of _C_xxx.
	// In a separate file so that the import of "unsafe" does not
	// pollute the original file.
	fmt.Fprintf(fgo2, "// Created by cgo - DO NOT EDIT\n")
	fmt.Fprintf(fgo2, "package %s\n\n", p.Package)
	fmt.Fprintf(fgo2, "import \"unsafe\"\n\n")
	fmt.Fprintf(fgo2, "type _ unsafe.Pointer\n\n")

	for name, def := range p.Typedef {
		fmt.Fprintf(fgo2, "type %s ", name)
		printer.Fprint(fgo2, def)
		fmt.Fprintf(fgo2, "\n")
	}
	fmt.Fprintf(fgo2, "type _C_void [0]byte\n")

	fmt.Fprintf(fc, cProlog, pkgroot, pkgroot, pkgroot, pkgroot)

	for name, def := range p.Vardef {
		fmt.Fprintf(fc, "#pragma dynld ·_C_%s %s \"%s.so\"\n", name, name, path)
		fmt.Fprintf(fgo2, "var _C_%s ", name)
		printer.Fprint(fgo2, &ast.StarExpr{X: def.Go})
		fmt.Fprintf(fgo2, "\n")
	}
	fmt.Fprintf(fc, "\n")

	for name, value := range p.Constdef {
		fmt.Fprintf(fgo2, "const %s = %s\n", name, value)
	}

	for name, value := range p.Enumdef {
		fmt.Fprintf(fgo2, "const %s = %d\n", name, value)
	}
	fmt.Fprintf(fgo2, "\n")

	for name, def := range p.Funcdef {
		// Go func declaration.
		d := &ast.FuncDecl{
			Name: ast.NewIdent("_C_" + name),
			Type: def.Go,
		}
		printer.Fprint(fgo2, d)
		fmt.Fprintf(fgo2, "\n")

		if name == "CString" || name == "GoString" {
			// The builtins are already defined in the C prolog.
			continue
		}

		// Construct a gcc struct matching the 6c argument frame.
		// Assumes that in gcc, char is 1 byte, short 2 bytes, int 4 bytes, long long 8 bytes.
		// These assumptions are checked by the gccProlog.
		// Also assumes that 6c convention is to word-align the
		// input and output parameters.
		structType := "struct {\n"
		off := int64(0)
		npad := 0
		for i, t := range def.Params {
			if off%t.Align != 0 {
				pad := t.Align - off%t.Align
				structType += fmt.Sprintf("\t\tchar __pad%d[%d];\n", npad, pad)
				off += pad
				npad++
			}
			structType += fmt.Sprintf("\t\t%s p%d;\n", t.C, i)
			off += t.Size
		}
		if off%p.PtrSize != 0 {
			pad := p.PtrSize - off%p.PtrSize
			structType += fmt.Sprintf("\t\tchar __pad%d[%d];\n", npad, pad)
			off += pad
			npad++
		}
		if t := def.Result; t != nil {
			if off%t.Align != 0 {
				pad := t.Align - off%t.Align
				structType += fmt.Sprintf("\t\tchar __pad%d[%d];\n", npad, pad)
				off += pad
				npad++
			}
			structType += fmt.Sprintf("\t\t%s r;\n", t.C)
			off += t.Size
		}
		if off%p.PtrSize != 0 {
			pad := p.PtrSize - off%p.PtrSize
			structType += fmt.Sprintf("\t\tchar __pad%d[%d];\n", npad, pad)
			off += pad
			npad++
		}
		if len(def.Params) == 0 && def.Result == nil {
			structType += "\t\tchar unused;\n" // avoid empty struct
			off++
		}
		structType += "\t}"
		argSize := off

		// C wrapper calls into gcc, passing a pointer to the argument frame.
		// Also emit #pragma to get a pointer to the gcc wrapper.
		fmt.Fprintf(fc, "#pragma dynld _cgo_%s _cgo_%s \"%s.so\"\n", name, name, path)
		fmt.Fprintf(fc, "void (*_cgo_%s)(void*);\n", name)
		fmt.Fprintf(fc, "\n")
		fmt.Fprintf(fc, "void\n")
		fmt.Fprintf(fc, "·_C_%s(struct{uint8 x[%d];}p)\n", name, argSize)
		fmt.Fprintf(fc, "{\n")
		fmt.Fprintf(fc, "\tcgocall(_cgo_%s, &p);\n", name)
		fmt.Fprintf(fc, "}\n")
		fmt.Fprintf(fc, "\n")
	}

	fgo2.Close()
	fc.Close()
}

// writeOutput creates stubs for a specific source file to be compiled by 6g
// (The comments here say 6g and 6c but the code applies to the 8 and 5 tools too.)
func (p *Prog) writeOutput(srcfile string) {
	base := srcfile
	if strings.HasSuffix(base, ".go") {
		base = base[0 : len(base)-3]
	}
	fgo1 := creat(base + ".cgo1.go")
	fgcc := creat(base + ".cgo2.c")

	// Write Go output: Go input with rewrites of C.xxx to _C_xxx.
	fmt.Fprintf(fgo1, "// Created by cgo - DO NOT EDIT\n")
	fmt.Fprintf(fgo1, "//line %s:1\n", srcfile)
	printer.Fprint(fgo1, p.AST)

	// While we process the vars and funcs, also write 6c and gcc output.
	// Gcc output starts with the preamble.
	fmt.Fprintf(fgcc, "%s\n", p.Preamble)
	fmt.Fprintf(fgcc, "%s\n", gccProlog)

	for name, def := range p.Funcdef {
		_, ok := p.OutDefs[name]
		if name == "CString" || name == "GoString" || ok {
			// The builtins are already defined in the C prolog, and we don't
			// want to duplicate function definitions we've already done.
			continue
		}
		p.OutDefs[name] = true

		// Construct a gcc struct matching the 6c argument frame.
		// Assumes that in gcc, char is 1 byte, short 2 bytes, int 4 bytes, long long 8 bytes.
		// These assumptions are checked by the gccProlog.
		// Also assumes that 6c convention is to word-align the
		// input and output parameters.
		structType := "struct {\n"
		off := int64(0)
		npad := 0
		for i, t := range def.Params {
			if off%t.Align != 0 {
				pad := t.Align - off%t.Align
				structType += fmt.Sprintf("\t\tchar __pad%d[%d];\n", npad, pad)
				off += pad
				npad++
			}
			structType += fmt.Sprintf("\t\t%s p%d;\n", t.C, i)
			off += t.Size
		}
		if off%p.PtrSize != 0 {
			pad := p.PtrSize - off%p.PtrSize
			structType += fmt.Sprintf("\t\tchar __pad%d[%d];\n", npad, pad)
			off += pad
			npad++
		}
		if t := def.Result; t != nil {
			if off%t.Align != 0 {
				pad := t.Align - off%t.Align
				structType += fmt.Sprintf("\t\tchar __pad%d[%d];\n", npad, pad)
				off += pad
				npad++
			}
			structType += fmt.Sprintf("\t\t%s r;\n", t.C)
			off += t.Size
		}
		if off%p.PtrSize != 0 {
			pad := p.PtrSize - off%p.PtrSize
			structType += fmt.Sprintf("\t\tchar __pad%d[%d];\n", npad, pad)
			off += pad
			npad++
		}
		if len(def.Params) == 0 && def.Result == nil {
			structType += "\t\tchar unused;\n" // avoid empty struct
			off++
		}
		structType += "\t}"

		// Gcc wrapper unpacks the C argument struct
		// and calls the actual C function.
		fmt.Fprintf(fgcc, "void\n")
		fmt.Fprintf(fgcc, "_cgo_%s(void *v)\n", name)
		fmt.Fprintf(fgcc, "{\n")
		fmt.Fprintf(fgcc, "\t%s *a = v;\n", structType)
		fmt.Fprintf(fgcc, "\t")
		if def.Result != nil {
			fmt.Fprintf(fgcc, "a->r = ")
		}
		fmt.Fprintf(fgcc, "%s(", name)
		for i := range def.Params {
			if i > 0 {
				fmt.Fprintf(fgcc, ", ")
			}
			fmt.Fprintf(fgcc, "a->p%d", i)
		}
		fmt.Fprintf(fgcc, ");\n")
		fmt.Fprintf(fgcc, "}\n")
		fmt.Fprintf(fgcc, "\n")
	}

	fgo1.Close()
	fgcc.Close()
}

const gccProlog = `
// Usual nonsense: if x and y are not equal, the type will be invalid
// (have a negative array count) and an inscrutable error will come
// out of the compiler and hopefully mention "name".
#define __cgo_compile_assert_eq(x, y, name) typedef char name[(x-y)*(x-y)*-2+1];

// Check at compile time that the sizes we use match our expectations.
#define __cgo_size_assert(t, n) __cgo_compile_assert_eq(sizeof(t), n, _cgo_sizeof_##t##_is_not_##n)

__cgo_size_assert(char, 1)
__cgo_size_assert(short, 2)
__cgo_size_assert(int, 4)
typedef long long __cgo_long_long;
__cgo_size_assert(__cgo_long_long, 8)
__cgo_size_assert(float, 4)
__cgo_size_assert(double, 8)
`

const builtinProlog = `
typedef struct { char *p; int n; } _GoString_;
_GoString_ GoString(char *p);
char *CString(_GoString_);
`

const cProlog = `
#include "runtime.h"
#include "cgocall.h"

#pragma dynld initcgo initcgo "%s/libcgo.so"
#pragma dynld libcgo_thread_start libcgo_thread_start "%s/libcgo.so"
#pragma dynld _cgo_malloc _cgo_malloc "%s/libcgo.so"
#pragma dynld _cgo_free free "%s/libcgo.so"

void
·_C_GoString(int8 *p, String s)
{
	s = gostring((byte*)p);
	FLUSH(&s);
}

void
·_C_CString(String s, int8 *p)
{
	p = cmalloc(s.len+1);
	mcpy((byte*)p, s.str, s.len);
	p[s.len] = 0;
	FLUSH(&p);
}
`
<|endoftext|>


<commit_before>// Copyright (c) 2014 Soichiro Kashima
// Licensed under MIT license.

package main

import (
	"encoding/hex"
	"encoding/xml"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

const (
	OutputHeader = `// DO NOT EDIT.
// This file is automatically generated by rdotm tool.
// https://github.com/ksoichiro/rdotm

`
)

// Command line options
type Options struct {
	ResDir   string
	OutDir   string
	Class    string
	Clean    bool
	Localize bool
}

// Resource model structure
type Resources struct {
	Language string   `xml:"-"`
	Strings  []String `xml:"string"`
	Colors   []Color  `xml:"color"`
}

type String struct {
	Name  string `xml:"name,attr"`
	Value string `xml:",chardata"`
}

type Color struct {
	Name  string `xml:"name,attr"`
	Value string `xml:",chardata"`
}

func main() {
	// Get command line options
	var (
		resDir   = flag.String("res", "", "Resource(res) directory path. Required.")
		outDir   = flag.String("out", "", "Output directory path. Required.")
		class    = flag.String("class", "R", "Class name to overwrite default value(R). Optional.")
		clean    = flag.Bool("clean", false, "Clean output directory before execution.")
		localize = flag.Bool("localize", false, "Enable localization using NSLocalizedStringFromTable.")
	)
	flag.Parse()
	if *resDir == "" || *outDir == "" {
		// Exit if the required options are empty
		flag.Usage()
		os.Exit(1)
	}

	// Parse resource XML files and generate source code
	parse(&Options{
		ResDir:   *resDir,
		OutDir:   *outDir,
		Class:    *class,
		Clean:    *clean,
		Localize: *localize})
}

func parse(opt *Options) {
	// Clean and create output directory if needed
	if opt.Clean {
		// Discard all files in the output directory
		os.RemoveAll(opt.OutDir)
	}
	os.MkdirAll(opt.OutDir, 0777)

	// Parse all of the files in res/values/*.xml
	var res Resources
	if opt.Localize {
		resSubDirs, _ := ioutil.ReadDir(opt.ResDir)
		for i := range resSubDirs {
			// Get only values directories
			valuesDir := resSubDirs[i]
			if matched, _ := regexp.MatchString("^values", valuesDir.Name()); !matched {
				continue
			}

			var lang string
			var r Resources
			if valuesDir.Name() == "values" {
				// Base language
				lang = "Base"
				r = parseLang(filepath.Join(opt.ResDir, valuesDir.Name()))
				// Output only base language to Objective-C source
				res = r
			} else {
				re := regexp.MustCompile("values-([a-zA-Z]+)")
				groups := re.FindStringSubmatch(valuesDir.Name())
				if groups == nil {
					// Not supported
					continue
				} else {
					// Maybe supported language
					lang = groups[1]
				}

				r = parseLang(filepath.Join(opt.ResDir, valuesDir.Name()))
			}
			// Create R.strings
			printLocalizableStrings(&r, opt, lang)
		}
	} else {
		valuesDir := filepath.Join(opt.ResDir, "values")
		res = parseLang(valuesDir)
	}
	printAsObjectiveC(&res, opt)
}

func parseLang(valuesDir string) (res Resources) {
	files, _ := ioutil.ReadDir(valuesDir)
	for j := range files {
		entry := files[j]
		if matched, _ := regexp.MatchString(".xml$", entry.Name()); !matched {
			continue
		}
		entryPath := filepath.Join(valuesDir, entry.Name())
		r := parseXml(entryPath)
		if 0 < len(r.Strings) {
			res.Strings = append(res.Strings, r.Strings...)
		}
		if 0 < len(r.Colors) {
			res.Colors = append(res.Colors, r.Colors...)
		}
	}
	return res
}

func parseXml(filename string) (res Resources) {
	xmlFile, err := os.Open(filename)
	if err != nil {
		fmt.Println("Error opening file", err)
		return
	}
	defer xmlFile.Close()

	b, _ := ioutil.ReadAll(xmlFile)
	err = xml.Unmarshal(b, &res)
	if err != nil {
		fmt.Println("Error unmarshaling XML file", err)
		return
	}

	return res
}

func printLocalizableStrings(res *Resources, opt *Options, lang string) {
	class := opt.Class

	// Create language separated directory
	langDir := filepath.Join(opt.OutDir, lang+".lproj")
	os.MkdirAll(langDir, 0777)

	filename := filepath.Join(langDir, class+".strings")
	f, _ := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0666)
	defer f.Close()

	f.WriteString(OutputHeader)

	// String
	for i := range res.Strings {
		s := res.Strings[i]
		f.WriteString(fmt.Sprintf("\"%s\" = \"%s\";\n", s.Name, s.Value))
	}

	f.Close()
}

func printAsObjectiveC(res *Resources, opt *Options) {
	class := opt.Class

	// Print header file(.h)
	filename := filepath.Join(opt.OutDir, class+".h")
	f, _ := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0666)
	defer f.Close()

	f.WriteString(OutputHeader)
	f.WriteString(fmt.Sprintf(`#import <UIKit/UIKit.h>

@interface %s : NSObject

`, class))

	// String
	for i := range res.Strings {
		s := res.Strings[i]
		// Method definition
		f.WriteString(fmt.Sprintf("+ (NSString *)string_%s;\n", s.Name))
	}

	// Color
	for i := range res.Colors {
		s := res.Colors[i]
		// Method definition
		f.WriteString(fmt.Sprintf("+ (UIColor *)color_%s;\n", s.Name))
	}

	f.WriteString(`
@end
`)
	f.Close()

	// Print implementation file(.m)
	filename = filepath.Join(opt.OutDir, class+".m")
	f, _ = os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0666)
	defer f.Close()

	// Import header file
	f.WriteString(OutputHeader)
	f.WriteString(fmt.Sprintf(`#import "%s.h"

@implementation %s

`, class, class))

	// String
	for i := range res.Strings {
		s := res.Strings[i]
		// Method implementation
		if opt.Localize {
			// Read from LANG.lproj/R.strings
			f.WriteString(fmt.Sprintf("+ (NSString *)string_%s { return NSLocalizedStringFromTable(@\"%s\", @\"%s\", nil); }\n", s.Name, s.Name, class))
		} else {
			f.WriteString(fmt.Sprintf("+ (NSString *)string_%s { return @\"%s\"; }\n", s.Name, s.Value))
		}
	}

	// Color
	for i := range res.Colors {
		s := res.Colors[i]
		// Method implementation
		a, r, g, b := hexToInt(s.Value)
		f.WriteString(fmt.Sprintf("+ (UIColor *)color_%s { return [UIColor colorWithRed:%d/255.0 green:%d/255.0 blue:%d/255.0 alpha:%d/255.0]; }\n", s.Name, r, g, b, a))
	}

	f.WriteString(`
@end
`)
	f.Close()
}

func hexToInt(hexString string) (a, r, g, b int) {
	raw := hexString
	// Remove prefix '#'
	if strings.HasPrefix(raw, "#") {
		braw := []byte(raw)
		raw = string(braw[1:])
	}

	// Format hex string
	if len(raw) == 8 {
		// AARRGGBB: Do nothing
	} else if len(raw) == 6 {
		// RRGGBB: Insert alpha(FF)
		raw = "FF" + raw
	} else if len(raw) == 4 {
		// ARGB: Duplicate each hex
		braw := []byte(raw)
		sa := string(braw[0:1])
		sr := string(braw[1:2])
		sg := string(braw[2:3])
		sb := string(braw[3:4])
		raw = sa + sa + sr + sr + sg + sg + sb + sb
	} else if len(raw) == 3 {
		// RGB: Insert alpha(F) and duplicate each hex
		raw = "F" + raw
		braw := []byte(raw)
		sa := string(braw[0:1])
		sr := string(braw[1:2])
		sg := string(braw[2:3])
		sb := string(braw[3:4])
		raw = sa + sa + sr + sr + sg + sg + sb + sb
	}
	bytes, _ := hex.DecodeString(raw)
	a = int(bytes[0])
	r = int(bytes[1])
	g = int(bytes[2])
	b = int(bytes[3])
	return
}
<commit_msg>Closes #5: Merge branch '5_add_content_to_comment'<commit_after>// Copyright (c) 2014 Soichiro Kashima
// Licensed under MIT license.

package main

import (
	"encoding/hex"
	"encoding/xml"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

const (
	OutputHeader = `// DO NOT EDIT.
// This file is automatically generated by rdotm tool.
// https://github.com/ksoichiro/rdotm

`
)

// Command line options
type Options struct {
	ResDir   string
	OutDir   string
	Class    string
	Clean    bool
	Localize bool
}

// Resource model structure
type Resources struct {
	Language string   `xml:"-"`
	Strings  []String `xml:"string"`
	Colors   []Color  `xml:"color"`
}

type String struct {
	Name  string `xml:"name,attr"`
	Value string `xml:",chardata"`
}

type Color struct {
	Name  string `xml:"name,attr"`
	Value string `xml:",chardata"`
}

func main() {
	// Get command line options
	var (
		resDir   = flag.String("res", "", "Resource(res) directory path. Required.")
		outDir   = flag.String("out", "", "Output directory path. Required.")
		class    = flag.String("class", "R", "Class name to overwrite default value(R). Optional.")
		clean    = flag.Bool("clean", false, "Clean output directory before execution.")
		localize = flag.Bool("localize", false, "Enable localization using NSLocalizedStringFromTable.")
	)
	flag.Parse()
	if *resDir == "" || *outDir == "" {
		// Exit if the required options are empty
		flag.Usage()
		os.Exit(1)
	}

	// Parse resource XML files and generate source code
	parse(&Options{
		ResDir:   *resDir,
		OutDir:   *outDir,
		Class:    *class,
		Clean:    *clean,
		Localize: *localize})
}

func parse(opt *Options) {
	// Clean and create output directory if needed
	if opt.Clean {
		// Discard all files in the output directory
		os.RemoveAll(opt.OutDir)
	}
	os.MkdirAll(opt.OutDir, 0777)

	// Parse all of the files in res/values/*.xml
	var res Resources
	if opt.Localize {
		resSubDirs, _ := ioutil.ReadDir(opt.ResDir)
		for i := range resSubDirs {
			// Get only values directories
			valuesDir := resSubDirs[i]
			if matched, _ := regexp.MatchString("^values", valuesDir.Name()); !matched {
				continue
			}

			var lang string
			var r Resources
			if valuesDir.Name() == "values" {
				// Base language
				lang = "Base"
				r = parseLang(filepath.Join(opt.ResDir, valuesDir.Name()))
				// Output only base language to Objective-C source
				res = r
			} else {
				re := regexp.MustCompile("values-([a-zA-Z]+)")
				groups := re.FindStringSubmatch(valuesDir.Name())
				if groups == nil {
					// Not supported
					continue
				} else {
					// Maybe supported language
					lang = groups[1]
				}

				r = parseLang(filepath.Join(opt.ResDir, valuesDir.Name()))
			}
			// Create R.strings
			printLocalizableStrings(&r, opt, lang)
		}
	} else {
		valuesDir := filepath.Join(opt.ResDir, "values")
		res = parseLang(valuesDir)
	}
	printAsObjectiveC(&res, opt)
}

func parseLang(valuesDir string) (res Resources) {
	files, _ := ioutil.ReadDir(valuesDir)
	for j := range files {
		entry := files[j]
		if matched, _ := regexp.MatchString(".xml$", entry.Name()); !matched {
			continue
		}
		entryPath := filepath.Join(valuesDir, entry.Name())
		r := parseXml(entryPath)
		if 0 < len(r.Strings) {
			res.Strings = append(res.Strings, r.Strings...)
		}
		if 0 < len(r.Colors) {
			res.Colors = append(res.Colors, r.Colors...)
		}
	}
	return res
}

func parseXml(filename string) (res Resources) {
	xmlFile, err := os.Open(filename)
	if err != nil {
		fmt.Println("Error opening file", err)
		return
	}
	defer xmlFile.Close()

	b, _ := ioutil.ReadAll(xmlFile)
	err = xml.Unmarshal(b, &res)
	if err != nil {
		fmt.Println("Error unmarshaling XML file", err)
		return
	}

	return res
}

func printLocalizableStrings(res *Resources, opt *Options, lang string) {
	class := opt.Class

	// Create language separated directory
	langDir := filepath.Join(opt.OutDir, lang+".lproj")
	os.MkdirAll(langDir, 0777)

	filename := filepath.Join(langDir, class+".strings")
	f, _ := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0666)
	defer f.Close()

	f.WriteString(OutputHeader)

	// String
	for i := range res.Strings {
		s := res.Strings[i]
		f.WriteString(fmt.Sprintf("\"%s\" = \"%s\";\n", s.Name, s.Value))
	}

	f.Close()
}

func printAsObjectiveC(res *Resources, opt *Options) {
	class := opt.Class

	// Print header file(.h)
	filename := filepath.Join(opt.OutDir, class+".h")
	f, _ := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0666)
	defer f.Close()

	f.WriteString(OutputHeader)
	f.WriteString(fmt.Sprintf(`#import <UIKit/UIKit.h>

@interface %s : NSObject

`, class))

	// String
	for i := range res.Strings {
		s := res.Strings[i]
		// Method definition
		f.WriteString(fmt.Sprintf(`/** %s */
+ (NSString *)string_%s;
`, s.Value, s.Name))
	}

	// Color
	for i := range res.Colors {
		s := res.Colors[i]
		// Method definition
		f.WriteString(fmt.Sprintf(`/** %s */
+ (UIColor *)color_%s;
`, s.Value, s.Name))
	}

	f.WriteString(`
@end
`)
	f.Close()

	// Print implementation file(.m)
	filename = filepath.Join(opt.OutDir, class+".m")
	f, _ = os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0666)
	defer f.Close()

	// Import header file
	f.WriteString(OutputHeader)
	f.WriteString(fmt.Sprintf(`#import "%s.h"

@implementation %s

`, class, class))

	// String
	for i := range res.Strings {
		s := res.Strings[i]
		// Method implementation
		if opt.Localize {
			// Read from LANG.lproj/R.strings
			f.WriteString(fmt.Sprintf("+ (NSString *)string_%s { return NSLocalizedStringFromTable(@\"%s\", @\"%s\", nil); }\n", s.Name, s.Name, class))
		} else {
			f.WriteString(fmt.Sprintf("+ (NSString *)string_%s { return @\"%s\"; }\n", s.Name, s.Value))
		}
	}

	// Color
	for i := range res.Colors {
		s := res.Colors[i]
		// Method implementation
		a, r, g, b := hexToInt(s.Value)
		f.WriteString(fmt.Sprintf("+ (UIColor *)color_%s { return [UIColor colorWithRed:%d/255.0 green:%d/255.0 blue:%d/255.0 alpha:%d/255.0]; }\n", s.Name, r, g, b, a))
	}

	f.WriteString(`
@end
`)
	f.Close()
}

func hexToInt(hexString string) (a, r, g, b int) {
	raw := hexString
	// Remove prefix '#'
	if strings.HasPrefix(raw, "#") {
		braw := []byte(raw)
		raw = string(braw[1:])
	}

	// Format hex string
	if len(raw) == 8 {
		// AARRGGBB: Do nothing
	} else if len(raw) == 6 {
		// RRGGBB: Insert alpha(FF)
		raw = "FF" + raw
	} else if len(raw) == 4 {
		// ARGB: Duplicate each hex
		braw := []byte(raw)
		sa := string(braw[0:1])
		sr := string(braw[1:2])
		sg := string(braw[2:3])
		sb := string(braw[3:4])
		raw = sa + sa + sr + sr + sg + sg + sb + sb
	} else if len(raw) == 3 {
		// RGB: Insert alpha(F) and duplicate each hex
		raw = "F" + raw
		braw := []byte(raw)
		sa := string(braw[0:1])
		sr := string(braw[1:2])
		sg := string(braw[2:3])
		sb := string(braw[3:4])
		raw = sa + sa + sr + sr + sg + sg + sb + sb
	}
	bytes, _ := hex.DecodeString(raw)
	a = int(bytes[0])
	r = int(bytes[1])
	g = int(bytes[2])
	b = int(bytes[3])
	return
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/xeipuuv/gojsonreference"
)

type JsonSchemaBundler interface {
	GetRootSchema() (JsonSchema)
	GetSchema(string) (JsonSchema)
}

type bundler struct {
	rootRef *gojsonreference.JsonReference
	loader JsonSchemaLoader
}

func NewJsonSchemaBundler(loader JsonSchemaLoader, rootRef *gojsonreference.JsonReference) (JsonSchemaBundler) {
	b := &bundler{
		loader,
		rootRef,
	}
	return b
}

func (b *bundler) GetRootSchema() (JsonSchema) {
	return b.loader.Load(b.rootRef)
}

func (b *bundler) GetSchema(refString string) (JsonSchema) {
	return b.loader.Load(NewRelativeJsonReference(b.rootRef, NewJsonReference(refString)))
}

<commit_msg>fixed bug<commit_after>package main

import (
	"github.com/xeipuuv/gojsonreference"
)

type JsonSchemaBundler interface {
	GetRootSchema() (JsonSchema)
	GetSchema(string) (JsonSchema)
}

type bundler struct {
	loader JsonSchemaLoader
	rootRef gojsonreference.JsonReference
}

func NewJsonSchemaBundler(loader JsonSchemaLoader, rootRef gojsonreference.JsonReference) (JsonSchemaBundler) {
	b := &bundler{
		loader,
		rootRef,
	}
	return b
}

func (b *bundler) GetRootSchema() (JsonSchema) {
	return b.loader.Load(b.rootRef)
}

func (b *bundler) GetSchema(refString string) (JsonSchema) {
	ref := NewRelativeJsonReference(b.rootRef, NewJsonReference(refString))
	schema := b.loader.Load(ref)
	if schema.Ref != "" {
		return b.GetSchema(schema.Ref)
	}
	return schema
}
<|endoftext|>


<commit_before>// Package presence provides simple user presence system
package presence

import (
	"errors"
	"fmt"
	"strconv"
	"sync"
	"time"

	gredis "github.com/garyburd/redigo/redis"
	"github.com/koding/redis"
)

// Prefix for presence package
var PresencePrefix = "presence"

// Redis holds the required connection data for redis
type Redis struct {
	// main redis connection
	redis *redis.RedisSession

	// inactiveDuration specifies no-probe allowance time
	inactiveDuration time.Duration

	// receiving offline events pattern
	becameOfflinePattern string

	// receiving online events pattern
	becameOnlinePattern string

	// errChan pipe all errors  the this channel
	errChan chan error

	// closed holds the status of connection
	closed bool

	//psc holds the pubsub channel if opened
	psc *gredis.PubSubConn

	// holds event channel
	events chan Event

	// lock for Redis struct
	mu sync.Mutex
}

// NewRedis creates a Redis for any broker system that is architected to use,
// communicate, forward events to the presence system
func NewRedis(server string, db int, inactiveDuration time.Duration) (Backend, error) {
	redis, err := redis.NewRedisSession(&redis.RedisConf{Server: server, DB: db})
	if err != nil {
		return nil, err
	}
	redis.SetPrefix(PresencePrefix)

	return &Redis{
		redis:                redis,
		becameOfflinePattern: fmt.Sprintf("__keyevent@%d__:expired", db),
		becameOnlinePattern:  fmt.Sprintf("__keyevent@%d__:set", db),
		inactiveDuration:     inactiveDuration,
		errChan:              make(chan error, 1),
	}, nil
}

// Online resets the expiration time for any given key
// if key doesnt exists, it means user is now online and should be set as online
// Whenever application gets any prob from a client
// should call this function
func (s *Redis) Online(ids ...string) error {
	if len(ids) == 1 {
		// if member exits increase ttl
		if s.redis.Expire(ids[0], s.inactiveDuration) == nil {
			return nil
		}

		// if member doesnt exist set it
		return s.redis.Setex(ids[0], s.inactiveDuration, ids[0])
	}

	existance, err := s.sendMultiExpire(ids, s.inactiveDurationString())
	if err != nil {
		return err
	}

	return s.sendMultiSetIfRequired(ids, existance)
}

// Offline sets given ids as offline, ignores any error
func (s *Redis) Offline(ids ...string) error {
	if len(ids) == 1 {
		if s.redis.Expire(ids[0], time.Second*0) == nil {
			return nil
		}
	}

	s.sendMultiExpire(ids, "0")
	return nil
}

// sendMultiSetIfRequired accepts set of ids and their existtance status
// traverse over them and any key is not exists in db, set them in a multi/exec
// request
func (s *Redis) sendMultiSetIfRequired(ids []string, existance []int) error {
	if len(ids) != len(existance) {
		return fmt.Errorf("length is not same Ids: %d Existance: %d", len(ids), len(existance))
	}

	// cache inactive duration
	seconds := s.inactiveDurationString()

	// get one connection from pool
	c := s.redis.Pool().Get()

	// item count for non-existent members
	notExistsCount := 0

	for i, exists := range existance {
		// `0` means, member doesnt exists in presence system
		if exists != 0 {
			continue
		}

		// init multi command lazily
		if notExistsCount == 0 {
			c.Send("MULTI")
		}

		notExistsCount++
		c.Send("SETEX", s.redis.AddPrefix(ids[i]), seconds, ids[i])
	}

	// execute multi command if only we flushed some to connection
	if notExistsCount != 0 {
		// ignore values
		if _, err := c.Do("EXEC"); err != nil {
			return err
		}
	}

	// do not forget to close the connection
	if err := c.Close(); err != nil {
		return err
	}

	return nil
}

// sendMultiExpire if the system tries to update more than one key at a time
// inorder to leverage rtt, send multi expire
func (s *Redis) sendMultiExpire(ids []string, duration string) ([]int, error) {
	// get one connection from pool
	c := s.redis.Pool().Get()

	// init multi command
	c.Send("MULTI")

	// send expire command for all members
	for _, id := range ids {
		c.Send("EXPIRE", s.redis.AddPrefix(id), duration)
	}

	// execute command
	r, err := c.Do("EXEC")
	if err != nil {
		return nil, err
	}

	// close connection
	if err := c.Close(); err != nil {
		return nil, err
	}

	values, err := s.redis.Values(r)
	if err != nil {
		return nil, err
	}

	res := make([]int, len(values))
	for i, value := range values {
		res[i], err = s.redis.Int(value)
		if err != nil {
			// what about returning half-generated slice?
			// instead of an empty one
			return nil, err
		}

	}

	return res, nil
}

// Status returns the current status multiple keys from system
func (s *Redis) Status(ids ...string) ([]Event, error) {
	if len(ids) == 1 {
		return s.singleStatus(ids[0])
	}

	// get one connection from pool
	c := s.redis.Pool().Get()

	// init multi command
	c.Send("MULTI")

	// send expire command for all members
	for _, id := range ids {
		c.Send("EXISTS", s.redis.AddPrefix(id))
	}

	// execute command
	r, err := c.Do("EXEC")
	if err != nil {
		return nil, err
	}

	// close connection
	if err := c.Close(); err != nil {
		return nil, err
	}

	values, err := s.redis.Values(r)
	if err != nil {
		return nil, err
	}

	res := make([]Event, len(values))
	for i, value := range values {
		status, err := s.redis.Int(value)
		if err != nil {
			return nil, err
		}

		res[i] = Event{
			ID: ids[i],
			// cast redis response to Status
			Status: Status(status),
		}
	}

	return res, nil
}

// Close closes the redis connection gracefully
func (s *Redis) Close() error {
	return s.close()
}

func (s *Redis) close() error {
	s.mu.Lock()
	defer s.mu.Unlock()

	if s.closed {
		return errors.New("closing of already closed connection")
	}

	s.closed = true

	if s.events != nil {
		close(s.events)
	}

	if s.psc != nil {
		s.psc.PUnsubscribe()
	}

	return s.redis.Close()
}

// Status returns the current status a key from system
func (s *Redis) singleStatus(id string) ([]Event, error) {
	res := make([]Event, 1)
	res[0] = Event{
		ID:     id,
		Status: Offline,
	}

	if s.redis.Exists(id) {
		res[0].Status = Online
	}

	return res, nil
}

// ListenStatusChanges subscribes with a pattern to the redis and
// gets online and offline status changes from it
func (s *Redis) ListenStatusChanges() chan Event {
	s.psc = s.redis.CreatePubSubConn()
	s.psc.PSubscribe(s.becameOnlinePattern, s.becameOfflinePattern)

	s.events = make(chan Event)
	go s.listenEvents()
	return s.events
}

// createEvent Creates the event with the required properties
func (s *Redis) listenEvents() {
	for {
		s.mu.Lock()
		if s.closed {
			s.mu.Unlock()
			return
		}
		s.mu.Unlock()

		switch n := s.psc.Receive().(type) {
		case gredis.PMessage:
			s.events <- s.createEvent(n)
		case error:
			s.errChan <- n
			return
		}
	}
}

// createEvent Creates the event with the required properties
func (s *Redis) createEvent(n gredis.PMessage) Event {
	e := Event{}

	// if incoming data len is smaller than our prefix, do not process the event
	if len(n.Data) < len(PresencePrefix) {
		s.errChan <- ErrInvalidID
		return e
	}

	e.ID = string(n.Data[len(PresencePrefix)+1:])

	switch n.Pattern {
	case s.becameOfflinePattern:
		e.Status = Offline
	case s.becameOnlinePattern:
		e.Status = Online
	default:
		//ignore other events, if we get any
	}

	return e
}

func (s *Redis) inactiveDurationString() string {
	return strconv.Itoa(int(s.inactiveDuration.Seconds()))
}

// Error returns error if it happens while listening  to status changes
func (s *Redis) Error() chan error {
	return s.errChan
}
<commit_msg>Redis: new exported error for invalid ids<commit_after>// Package presence provides simple user presence system
package presence

import (
	"errors"
	"fmt"
	"strconv"
	"sync"
	"time"

	gredis "github.com/garyburd/redigo/redis"
	"github.com/koding/redis"
)

// Prefix for presence package
var PresencePrefix = "presence"

// Error for stating that the event id is not valid
var ErrInvalidID = errors.New("invalid id")

// Redis holds the required connection data for redis
type Redis struct {
	// main redis connection
	redis *redis.RedisSession

	// inactiveDuration specifies no-probe allowance time
	inactiveDuration time.Duration

	// receiving offline events pattern
	becameOfflinePattern string

	// receiving online events pattern
	becameOnlinePattern string

	// errChan pipe all errors  the this channel
	errChan chan error

	// closed holds the status of connection
	closed bool

	//psc holds the pubsub channel if opened
	psc *gredis.PubSubConn

	// holds event channel
	events chan Event

	// lock for Redis struct
	mu sync.Mutex
}

// NewRedis creates a Redis for any broker system that is architected to use,
// communicate, forward events to the presence system
func NewRedis(server string, db int, inactiveDuration time.Duration) (Backend, error) {
	redis, err := redis.NewRedisSession(&redis.RedisConf{Server: server, DB: db})
	if err != nil {
		return nil, err
	}
	redis.SetPrefix(PresencePrefix)

	return &Redis{
		redis:                redis,
		becameOfflinePattern: fmt.Sprintf("__keyevent@%d__:expired", db),
		becameOnlinePattern:  fmt.Sprintf("__keyevent@%d__:set", db),
		inactiveDuration:     inactiveDuration,
		errChan:              make(chan error, 1),
	}, nil
}

// Online resets the expiration time for any given key
// if key doesnt exists, it means user is now online and should be set as online
// Whenever application gets any prob from a client
// should call this function
func (s *Redis) Online(ids ...string) error {
	if len(ids) == 1 {
		// if member exits increase ttl
		if s.redis.Expire(ids[0], s.inactiveDuration) == nil {
			return nil
		}

		// if member doesnt exist set it
		return s.redis.Setex(ids[0], s.inactiveDuration, ids[0])
	}

	existance, err := s.sendMultiExpire(ids, s.inactiveDurationString())
	if err != nil {
		return err
	}

	return s.sendMultiSetIfRequired(ids, existance)
}

// Offline sets given ids as offline, ignores any error
func (s *Redis) Offline(ids ...string) error {
	if len(ids) == 1 {
		if s.redis.Expire(ids[0], time.Second*0) == nil {
			return nil
		}
	}

	s.sendMultiExpire(ids, "0")
	return nil
}

// sendMultiSetIfRequired accepts set of ids and their existtance status
// traverse over them and any key is not exists in db, set them in a multi/exec
// request
func (s *Redis) sendMultiSetIfRequired(ids []string, existance []int) error {
	if len(ids) != len(existance) {
		return fmt.Errorf("length is not same Ids: %d Existance: %d", len(ids), len(existance))
	}

	// cache inactive duration
	seconds := s.inactiveDurationString()

	// get one connection from pool
	c := s.redis.Pool().Get()

	// item count for non-existent members
	notExistsCount := 0

	for i, exists := range existance {
		// `0` means, member doesnt exists in presence system
		if exists != 0 {
			continue
		}

		// init multi command lazily
		if notExistsCount == 0 {
			c.Send("MULTI")
		}

		notExistsCount++
		c.Send("SETEX", s.redis.AddPrefix(ids[i]), seconds, ids[i])
	}

	// execute multi command if only we flushed some to connection
	if notExistsCount != 0 {
		// ignore values
		if _, err := c.Do("EXEC"); err != nil {
			return err
		}
	}

	// do not forget to close the connection
	if err := c.Close(); err != nil {
		return err
	}

	return nil
}

// sendMultiExpire if the system tries to update more than one key at a time
// inorder to leverage rtt, send multi expire
func (s *Redis) sendMultiExpire(ids []string, duration string) ([]int, error) {
	// get one connection from pool
	c := s.redis.Pool().Get()

	// init multi command
	c.Send("MULTI")

	// send expire command for all members
	for _, id := range ids {
		c.Send("EXPIRE", s.redis.AddPrefix(id), duration)
	}

	// execute command
	r, err := c.Do("EXEC")
	if err != nil {
		return nil, err
	}

	// close connection
	if err := c.Close(); err != nil {
		return nil, err
	}

	values, err := s.redis.Values(r)
	if err != nil {
		return nil, err
	}

	res := make([]int, len(values))
	for i, value := range values {
		res[i], err = s.redis.Int(value)
		if err != nil {
			// what about returning half-generated slice?
			// instead of an empty one
			return nil, err
		}

	}

	return res, nil
}

// Status returns the current status multiple keys from system
func (s *Redis) Status(ids ...string) ([]Event, error) {
	if len(ids) == 1 {
		return s.singleStatus(ids[0])
	}

	// get one connection from pool
	c := s.redis.Pool().Get()

	// init multi command
	c.Send("MULTI")

	// send expire command for all members
	for _, id := range ids {
		c.Send("EXISTS", s.redis.AddPrefix(id))
	}

	// execute command
	r, err := c.Do("EXEC")
	if err != nil {
		return nil, err
	}

	// close connection
	if err := c.Close(); err != nil {
		return nil, err
	}

	values, err := s.redis.Values(r)
	if err != nil {
		return nil, err
	}

	res := make([]Event, len(values))
	for i, value := range values {
		status, err := s.redis.Int(value)
		if err != nil {
			return nil, err
		}

		res[i] = Event{
			ID: ids[i],
			// cast redis response to Status
			Status: Status(status),
		}
	}

	return res, nil
}

// Close closes the redis connection gracefully
func (s *Redis) Close() error {
	return s.close()
}

func (s *Redis) close() error {
	s.mu.Lock()
	defer s.mu.Unlock()

	if s.closed {
		return errors.New("closing of already closed connection")
	}

	s.closed = true

	if s.events != nil {
		close(s.events)
	}

	if s.psc != nil {
		s.psc.PUnsubscribe()
	}

	return s.redis.Close()
}

// Status returns the current status a key from system
func (s *Redis) singleStatus(id string) ([]Event, error) {
	res := make([]Event, 1)
	res[0] = Event{
		ID:     id,
		Status: Offline,
	}

	if s.redis.Exists(id) {
		res[0].Status = Online
	}

	return res, nil
}

// ListenStatusChanges subscribes with a pattern to the redis and
// gets online and offline status changes from it
func (s *Redis) ListenStatusChanges() chan Event {
	s.psc = s.redis.CreatePubSubConn()
	s.psc.PSubscribe(s.becameOnlinePattern, s.becameOfflinePattern)

	s.events = make(chan Event)
	go s.listenEvents()
	return s.events
}

// createEvent Creates the event with the required properties
func (s *Redis) listenEvents() {
	for {
		s.mu.Lock()
		if s.closed {
			s.mu.Unlock()
			return
		}
		s.mu.Unlock()

		switch n := s.psc.Receive().(type) {
		case gredis.PMessage:
			s.events <- s.createEvent(n)
		case error:
			s.errChan <- n
			return
		}
	}
}

// createEvent Creates the event with the required properties
func (s *Redis) createEvent(n gredis.PMessage) Event {
	e := Event{}

	// if incoming data len is smaller than our prefix, do not process the event
	if len(n.Data) < len(PresencePrefix) {
		s.errChan <- ErrInvalidID
		return e
	}

	e.ID = string(n.Data[len(PresencePrefix)+1:])

	switch n.Pattern {
	case s.becameOfflinePattern:
		e.Status = Offline
	case s.becameOnlinePattern:
		e.Status = Online
	default:
		//ignore other events, if we get any
	}

	return e
}

func (s *Redis) inactiveDurationString() string {
	return strconv.Itoa(int(s.inactiveDuration.Seconds()))
}

// Error returns error if it happens while listening  to status changes
func (s *Redis) Error() chan error {
	return s.errChan
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Plan 9 environment variables.

package syscall

import (
	"errors"
	"sync"
)

var (
	// envOnce guards copyenv, which populates env, envi and envs.
	envOnce sync.Once

	// envLock guards env, envi and envs.
	envLock sync.RWMutex

	// env maps from an environment variable to its value.
	// TODO: remove this? golang.org/issue/8849
	env = make(map[string]string)

	// envi maps from an environment variable to its index in envs.
	// TODO: remove this? golang.org/issue/8849
	envi = make(map[string]int)

	// envs contains elements of env in the form "key=value".
	// empty strings mean deleted.
	envs []string

	errZeroLengthKey = errors.New("zero length key")
	errShortWrite    = errors.New("i/o count too small")
)

func readenv(key string) (string, error) {
	fd, err := Open("/env/"+key, O_RDONLY)
	if err != nil {
		return "", err
	}
	defer Close(fd)
	l, _ := Seek(fd, 0, 2)
	Seek(fd, 0, 0)
	buf := make([]byte, l)
	n, err := Read(fd, buf)
	if err != nil {
		return "", err
	}
	if n > 0 && buf[n-1] == 0 {
		buf = buf[:n-1]
	}
	return string(buf), nil
}

func writeenv(key, value string) error {
	fd, err := Create("/env/"+key, O_RDWR, 0666)
	if err != nil {
		return err
	}
	defer Close(fd)
	b := []byte(value)
	n, err := Write(fd, b)
	if err != nil {
		return err
	}
	if n != len(b) {
		return errShortWrite
	}
	return nil
}

func copyenv() {
	fd, err := Open("/env", O_RDONLY)
	if err != nil {
		return
	}
	defer Close(fd)
	files, err := readdirnames(fd)
	if err != nil {
		return
	}
	envs = make([]string, len(files))
	i := 0
	for _, key := range files {
		v, err := readenv(key)
		if err != nil {
			continue
		}
		env[key] = v
		envs[i] = key + "=" + v
		envi[key] = i
		i++
	}
}

func Getenv(key string) (value string, found bool) {
	if len(key) == 0 {
		return "", false
	}

	envLock.RLock()
	defer envLock.RUnlock()

	if v, ok := env[key]; ok {
		return v, true
	}
	v, err := readenv(key)
	if err != nil {
		return "", false
	}
	env[key] = v
	envs = append(envs, key+"="+v)
	return v, true
}

func Setenv(key, value string) error {
	if len(key) == 0 {
		return errZeroLengthKey
	}

	envLock.Lock()
	defer envLock.Unlock()

	err := writeenv(key, value)
	if err != nil {
		return err
	}
	env[key] = value
	envs = append(envs, key+"="+value)
	envi[key] = len(envs) - 1
	return nil
}

func Clearenv() {
	envLock.Lock()
	defer envLock.Unlock()

	env = make(map[string]string)
	envi = make(map[string]int)
	envs = []string{}
	RawSyscall(SYS_RFORK, RFCENVG, 0, 0)
}

func Unsetenv(key string) error {
	if len(key) == 0 {
		return errZeroLengthKey
	}

	envLock.Lock()
	defer envLock.Unlock()

	Remove("/env/" + key)

	if i, ok := envi[key]; ok {
		delete(env, key)
		delete(envi, key)
		envs[i] = ""
	}
	return nil
}

func Environ() []string {
	envLock.RLock()
	defer envLock.RUnlock()

	envOnce.Do(copyenv)
	ret := make([]string, 0, len(envs))
	for _, pair := range envs {
		if pair != "" {
			ret = append(ret, pair)
		}
	}
	return ret
}
<commit_msg>syscall: don't cache environment variables on Plan 9<commit_after>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Plan 9 environment variables.

package syscall

import (
	"errors"
)

var (
	errZeroLengthKey = errors.New("zero length key")
	errShortWrite    = errors.New("i/o count too small")
)

func readenv(key string) (string, error) {
	fd, err := Open("/env/"+key, O_RDONLY)
	if err != nil {
		return "", err
	}
	defer Close(fd)
	l, _ := Seek(fd, 0, 2)
	Seek(fd, 0, 0)
	buf := make([]byte, l)
	n, err := Read(fd, buf)
	if err != nil {
		return "", err
	}
	if n > 0 && buf[n-1] == 0 {
		buf = buf[:n-1]
	}
	return string(buf), nil
}

func writeenv(key, value string) error {
	fd, err := Create("/env/"+key, O_RDWR, 0666)
	if err != nil {
		return err
	}
	defer Close(fd)
	b := []byte(value)
	n, err := Write(fd, b)
	if err != nil {
		return err
	}
	if n != len(b) {
		return errShortWrite
	}
	return nil
}

func Getenv(key string) (value string, found bool) {
	if len(key) == 0 {
		return "", false
	}
	v, err := readenv(key)
	if err != nil {
		return "", false
	}
	return v, true
}

func Setenv(key, value string) error {
	if len(key) == 0 {
		return errZeroLengthKey
	}
	err := writeenv(key, value)
	if err != nil {
		return err
	}
	return nil
}

func Clearenv() {
	RawSyscall(SYS_RFORK, RFCENVG, 0, 0)
}

func Unsetenv(key string) error {
	if len(key) == 0 {
		return errZeroLengthKey
	}
	Remove("/env/" + key)
	return nil
}

func Environ() []string {
	fd, err := Open("/env", O_RDONLY)
	if err != nil {
		return nil
	}
	defer Close(fd)
	files, err := readdirnames(fd)
	if err != nil {
		return nil
	}
	ret := make([]string, 0, len(files))

	for _, key := range files {
		v, err := readenv(key)
		if err != nil {
			continue
		}
		ret = append(ret, key+"="+v)
	}
	return ret
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package zanzibar

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"

	"time"

	"github.com/julienschmidt/httprouter"
	"github.com/uber-go/tally"
	"go.uber.org/zap"
)

// ServerHTTPRequest struct manages request
type ServerHTTPRequest struct {
	httpRequest *http.Request
	res         *ServerHTTPResponse
	gateway     *Gateway
	started     bool
	startTime   time.Time
	metrics     *EndpointMetrics

	Logger *zap.Logger
	Scope  tally.Scope

	EndpointName string
	HandlerName  string
	URL          *url.URL
	Method       string
	Params       httprouter.Params
	Header       Header
}

// NewServerHTTPRequest is helper function to alloc ServerHTTPRequest
func NewServerHTTPRequest(
	w http.ResponseWriter, r *http.Request,
	params httprouter.Params, endpoint *RouterEndpoint,
) *ServerHTTPRequest {
	req := &ServerHTTPRequest{
		gateway:     endpoint.gateway,
		httpRequest: r,

		Logger: endpoint.gateway.Logger,
		Scope:  endpoint.gateway.MetricScope,

		URL:     r.URL,
		Method:  r.Method,
		Params:  params,
		Header:  NewServerHTTPHeader(r.Header),
		metrics: &endpoint.metrics,
	}
	req.res = NewServerHTTPResponse(w, req)

	req.start(endpoint.EndpointName, endpoint.HandlerName)

	return req
}

// start the request, do some metrics etc
func (req *ServerHTTPRequest) start(endpoint string, handler string) {
	if req.started {
		/* coverage ignore next line */
		req.Logger.Error(
			"Cannot start ServerHTTPRequest twice",
			zap.String("path", req.URL.Path),
		)
		/* coverage ignore next line */
		return
	}

	req.EndpointName = endpoint
	req.HandlerName = handler
	req.started = true
	req.startTime = time.Now()

	req.metrics.requestRecvd.Inc(1)
}

// CheckHeaders verifies that request contains required headers.
func (req *ServerHTTPRequest) CheckHeaders(headers []string) bool {
	for _, headerName := range headers {
		headerValue := req.httpRequest.Header.Get(headerName)
		if headerValue == "" {
			req.res.SendErrorString(
				400, "Missing mandatory header: "+headerName,
			)
			req.Logger.Warn("Got request without mandatory header",
				zap.String("headerName", headerName),
			)
			return false
		}

	}
	return true
}

// ReadAndUnmarshalBody will try to unmarshal into struct or fail
func (req *ServerHTTPRequest) ReadAndUnmarshalBody(
	body json.Unmarshaler,
) bool {
	rawBody, success := req.ReadAll()
	if !success {
		return false
	}

	return req.UnmarshalBody(body, rawBody)
}

// ReadAll helper to read entire body
func (req *ServerHTTPRequest) ReadAll() ([]byte, bool) {
	rawBody, err := ioutil.ReadAll(req.httpRequest.Body)
	if err != nil {
		req.res.SendErrorString(500, "Could not ReadAll() body")
		req.Logger.Error("Could not ReadAll() body",
			zap.String("error", err.Error()),
		)
		return nil, false
	}

	return rawBody, true
}

// UnmarshalBody helper to unmarshal body into struct
func (req *ServerHTTPRequest) UnmarshalBody(
	body json.Unmarshaler, rawBody []byte,
) bool {
	err := body.UnmarshalJSON(rawBody)
	if err != nil {
		req.res.SendErrorString(400, "Could not parse json: "+err.Error())
		req.Logger.Warn("Could not parse json",
			zap.String("error", err.Error()),
		)
		return false
	}

	return true
}
<commit_msg>runtime: add query string support to server req<commit_after>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package zanzibar

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"

	"time"

	"github.com/julienschmidt/httprouter"
	"github.com/uber-go/tally"
	"go.uber.org/zap"
)

// ServerHTTPRequest struct manages request
type ServerHTTPRequest struct {
	httpRequest *http.Request
	res         *ServerHTTPResponse
	gateway     *Gateway
	started     bool
	startTime   time.Time
	metrics     *EndpointMetrics
	queryValues url.Values
	parseFailed bool

	Logger *zap.Logger
	Scope  tally.Scope

	EndpointName string
	HandlerName  string
	URL          *url.URL
	Method       string
	Params       httprouter.Params
	Header       Header
}

// NewServerHTTPRequest is helper function to alloc ServerHTTPRequest
func NewServerHTTPRequest(
	w http.ResponseWriter, r *http.Request,
	params httprouter.Params, endpoint *RouterEndpoint,
) *ServerHTTPRequest {
	req := &ServerHTTPRequest{
		gateway:     endpoint.gateway,
		httpRequest: r,
		queryValues: nil,

		Logger: endpoint.gateway.Logger,
		Scope:  endpoint.gateway.MetricScope,

		URL:     r.URL,
		Method:  r.Method,
		Params:  params,
		Header:  NewServerHTTPHeader(r.Header),
		metrics: &endpoint.metrics,
	}
	req.res = NewServerHTTPResponse(w, req)

	req.start(endpoint.EndpointName, endpoint.HandlerName)

	return req
}

// start the request, do some metrics etc
func (req *ServerHTTPRequest) start(endpoint string, handler string) {
	if req.started {
		/* coverage ignore next line */
		req.Logger.Error(
			"Cannot start ServerHTTPRequest twice",
			zap.String("path", req.URL.Path),
		)
		/* coverage ignore next line */
		return
	}

	req.EndpointName = endpoint
	req.HandlerName = handler
	req.started = true
	req.startTime = time.Now()

	req.metrics.requestRecvd.Inc(1)
}

// CheckHeaders verifies that request contains required headers.
func (req *ServerHTTPRequest) CheckHeaders(headers []string) bool {
	for _, headerName := range headers {
		headerValue := req.httpRequest.Header.Get(headerName)
		if headerValue == "" {
			req.Logger.Warn("Got request without mandatory header",
				zap.String("headerName", headerName),
			)

			if !req.parseFailed {
				req.res.SendErrorString(
					400, "Missing mandatory header: "+headerName,
				)
				req.parseFailed = true
			}

			return false
		}

	}
	return true
}

func (req *ServerHTTPRequest) parseQueryValues() bool {
	if req.parseFailed {
		return false
	}

	if req.queryValues != nil {
		return true
	}

	values, err := url.ParseQuery(req.httpRequest.URL.RawQuery)
	if err != nil {
		req.Logger.Warn("Got request with invalid query string",
			zap.String("error", err.Error()),
		)

		req.res.SendErrorString(
			400, "Could not parse query string",
		)
		req.parseFailed = true
		return false
	}

	req.queryValues = values
	return true
}

// GetQueryValue will return the first query parameter for key or empty string
func (req *ServerHTTPRequest) GetQueryValue(key string) (string, bool) {
	success := req.parseQueryValues()
	if !success {
		return "", false
	}

	return req.queryValues.Get(key), true
}

// GetQueryValues will return all query parameters for key.
func (req *ServerHTTPRequest) GetQueryValues(key string) ([]string, bool) {
	success := req.parseQueryValues()
	if !success {
		return nil, false
	}

	return req.queryValues[key], true
}

// ReadAndUnmarshalBody will try to unmarshal into struct or fail
func (req *ServerHTTPRequest) ReadAndUnmarshalBody(
	body json.Unmarshaler,
) bool {
	rawBody, success := req.ReadAll()
	if !success {
		return false
	}

	return req.UnmarshalBody(body, rawBody)
}

// ReadAll helper to read entire body
func (req *ServerHTTPRequest) ReadAll() ([]byte, bool) {
	rawBody, err := ioutil.ReadAll(req.httpRequest.Body)
	if err != nil {
		req.Logger.Error("Could not ReadAll() body",
			zap.String("error", err.Error()),
		)
		if !req.parseFailed {
			req.res.SendErrorString(500, "Could not ReadAll() body")
			req.parseFailed = true
		}

		return nil, false
	}

	return rawBody, true
}

// UnmarshalBody helper to unmarshal body into struct
func (req *ServerHTTPRequest) UnmarshalBody(
	body json.Unmarshaler, rawBody []byte,
) bool {
	err := body.UnmarshalJSON(rawBody)
	if err != nil {
		req.Logger.Warn("Could not parse json",
			zap.String("error", err.Error()),
		)
		if !req.parseFailed {
			req.res.SendErrorString(400, "Could not parse json: "+err.Error())
			req.parseFailed = true
		}
		return false
	}

	return true
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package auth

import (
	"github.com/jacobsa/aws/s3/http"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"testing"
)

func TestStringToSign(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type StringToSignTest struct {
}

func init() { RegisterTestSuite(&StringToSignTest{}) }

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *StringToSignTest) MissingDateHeader() {
	ExpectEq("TODO", "")
}

func (t *StringToSignTest) MinimalRequest() {
	// Request
	req := &http.Request {
		Verb: "PUT",
		Path: "/foo/bar/baz",
		Headers: map[string]string {
			"Date": "Tue, 27 Mar 2007 19:36:42 +0000",
		},
	}

	// Call
	s, err := stringToSign(req)
	AssertEq(nil, err)

	ExpectThat(
		s,
		Equals(
			"PUT\n" +
			"\n" +  // Content-MD5
			"\n" +  // Content-Type
			"Tue, 27 Mar 2007 19:36:42 +0000\n" +
			"/foo/bar/baz"))
}

func (t *StringToSignTest) IncludesContentMd5() {
	// Request
	req := &http.Request {
		Verb: "PUT",
		Path: "/foo/bar/baz",
		Headers: map[string]string {
			"Date": "some_date",
			"Content-MD5": "deadbeeffeedface",
		},
	}

	// Call
	s, err := stringToSign(req)
	AssertEq(nil, err)

	ExpectThat(
		s,
		Equals(
			"PUT\n" +
			"deadbeeffeedface\n" +
			"\n" +  // Content-Type
			"some_date" +
			"/foo/bar/baz"))
}

func (t *StringToSignTest) IncludesContentType() {
	// Request
	req := &http.Request {
		Verb: "PUT",
		Path: "/foo/bar/baz",
		Headers: map[string]string {
			"Date": "some_date",
			"Content-Type": "blah/foo",
		},
	}

	// Call
	s, err := stringToSign(req)
	AssertEq(nil, err)

	ExpectThat(
		s,
		Equals(
			"PUT\n" +
			"\n" +  // Content-MD5
			"blah/foo\n" +
			"some_date" +
			"/foo/bar/baz"))
}

func (t *StringToSignTest) IncludesDate() {
	ExpectEq("TODO", "")
}

func (t *StringToSignTest) ComplicatedRequest() {
	ExpectEq("TODO", "")
}
<commit_msg>Deleted a redundant test.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package auth

import (
	"github.com/jacobsa/aws/s3/http"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"testing"
)

func TestStringToSign(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type StringToSignTest struct {
}

func init() { RegisterTestSuite(&StringToSignTest{}) }

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *StringToSignTest) MissingDateHeader() {
	ExpectEq("TODO", "")
}

func (t *StringToSignTest) MinimalRequest() {
	// Request
	req := &http.Request {
		Verb: "PUT",
		Path: "/foo/bar/baz",
		Headers: map[string]string {
			"Date": "Tue, 27 Mar 2007 19:36:42 +0000",
		},
	}

	// Call
	s, err := stringToSign(req)
	AssertEq(nil, err)

	ExpectThat(
		s,
		Equals(
			"PUT\n" +
			"\n" +  // Content-MD5
			"\n" +  // Content-Type
			"Tue, 27 Mar 2007 19:36:42 +0000\n" +
			"/foo/bar/baz"))
}

func (t *StringToSignTest) IncludesContentMd5() {
	// Request
	req := &http.Request {
		Verb: "PUT",
		Path: "/foo/bar/baz",
		Headers: map[string]string {
			"Date": "some_date",
			"Content-MD5": "deadbeeffeedface",
		},
	}

	// Call
	s, err := stringToSign(req)
	AssertEq(nil, err)

	ExpectThat(
		s,
		Equals(
			"PUT\n" +
			"deadbeeffeedface\n" +
			"\n" +  // Content-Type
			"some_date" +
			"/foo/bar/baz"))
}

func (t *StringToSignTest) IncludesContentType() {
	// Request
	req := &http.Request {
		Verb: "PUT",
		Path: "/foo/bar/baz",
		Headers: map[string]string {
			"Date": "some_date",
			"Content-Type": "blah/foo",
		},
	}

	// Call
	s, err := stringToSign(req)
	AssertEq(nil, err)

	ExpectThat(
		s,
		Equals(
			"PUT\n" +
			"\n" +  // Content-MD5
			"blah/foo\n" +
			"some_date" +
			"/foo/bar/baz"))
}

func (t *StringToSignTest) ComplicatedRequest() {
	ExpectEq("TODO", "")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"github.com/gogf/gf/database/gredis"
	"github.com/gogf/gf/util/gconv"
)

// 使用原生gredis.New操作redis，但是注意需要自己调用Close方法关闭redis链接池
func main() {
	redis := gredis.New(gredis.Config{
		Host: "127.0.0.1",
		Port: 6379,
	})
	defer redis.Close()
	redis.Do("SET", "k", "v")
	v, _ := redis.Do("GET", "k")
	fmt.Println(gconv.String(v))
}
<commit_msg>bug:The parameter of the New method is a pointer<commit_after>package main

import (
	"fmt"

	"github.com/gogf/gf/database/gredis"
	"github.com/gogf/gf/util/gconv"
)

// 使用原生gredis.New操作redis，但是注意需要自己调用Close方法关闭redis链接池
func main() {
	config := &gredis.Config{
		Host: "127.0.0.1",
		Port: 6379,
	}
	redis := gredis.New(config)
	defer redis.Close()
	redis.Do("SET", "k", "v")
	v, _ := redis.Do("GET", "k")
	fmt.Println(gconv.String(v))
}
<|endoftext|>


<commit_before>package main

import (
	// "fmt"
	"github.com/cloudfoundry/gosteno"
	"github.com/codegangsta/cli"
	"github.com/pivotalservices/cfops/backup"
	. "github.com/pivotalservices/cfops/cli"
	"github.com/pivotalservices/cfops/install"
	"github.com/pivotalservices/cfops/start"
	"github.com/pivotalservices/cfops/system"
	"os"
	"strings"
)

var (
	// cli.StringFlag{
	// 	Name:   "iaas, i",
	// 	Value:  "aws, vsphere, vcloud, openstack",
	// 	Usage:  "set the IaaS type to target for deployment",
	// 	EnvVar: "CF_IAAS",
	// },
	// cli.StringFlag{
	// 	Name:   "debug, d",
	// 	Value:  "true, false",
	// 	Usage:  "enable/disable debug output",
	// 	EnvVar: "CF_TRACE",
	// },
	// cli.StringFlag{
	// 	Name:   "lang, l",
	// 	Value:  "en, es",
	// 	Usage:  "language for the cfops cli responses",
	// 	EnvVar: "CF_LANG",
	// },
	logFilePath    = NewFlag("logFile", "", "The CFOPS log file, defaults to STDOUT", "CFOPS_LOG")
	configFilePath = NewFlag("configFile", "config/cfops.json", "Location of the CFOPS config json file", "CFOPS_CONFIG")
	debug          = NewFlag("debug", false, "Debug logging", "CFOPS_TRACE")
)

type Config struct {
	Index uint
}

// To get the base foundation configuration see the Pivotal CF Data Collector @
// https://docs.google.com/a/pivotal.io/spreadsheets/d/1XHKSrJiQIu5MWGpPYWbMY8M09eqe-GV8MQsl_mqw1RM/edit#gid=0
func main() {

	commandFactory := NewCommandFactory()

	commandRunner := &system.OSCommandRunner{}

	start.New(commandFactory)
	install.New(commandFactory)
	backup.New(commandFactory, commandRunner)

	globalFlags := []*Flag{logFilePath, configFilePath, debug}

	app := NewApp(commandFactory, globalFlags)

	app.Before = func(c *cli.Context) error {
		for _, flag := range globalFlags {
			if flag.Type == "bool" {
				flag.Value = c.Bool(flag.Name)
				// fmt.Printf("BEFORE flag %s has value %v\n", flag.Name, flag.Value)
			} else {
				flag.Value = c.String(flag.Name)
				// fmt.Printf("BEFORE flag %s has value %s\n", flag.Name, flag.Value)
			}
		}
		_, logger := parseConfig(debug.ParseBool(), configFilePath.ParseString(), logFilePath.ParseString())

		commandRunner.SetLogger(logger)
		commandFactory.SetLogger(logger)
		return nil
	}

	app.RunAndExitOnError()
}

func parseConfig(debug bool, configFile, logFilePath string) (Config, *gosteno.Logger) {
	config := Config{}
	// err := config.LoadConfig(&config, configFile)
	// if err != nil {
	// 	panic(err)
	// }

	logger := NewLogger(debug, logFilePath, "cfops", config)
	logger.Info("Setting up CFOPS profiler")

	return config, logger
}

func NewLogger(verbose bool, logFilePath, name string, config Config) *gosteno.Logger {
	level := gosteno.LOG_INFO

	if verbose {
		level = gosteno.LOG_DEBUG
	}

	loggingConfig := &gosteno.Config{
		Sinks:     make([]gosteno.Sink, 1),
		Level:     level,
		Codec:     gosteno.NewJsonPrettifier(gosteno.EXCLUDE_DATA),
		EnableLOC: true}

	if strings.TrimSpace(logFilePath) == "" {
		loggingConfig.Sinks[0] = gosteno.NewIOSink(os.Stdout)
	} else {
		loggingConfig.Sinks[0] = gosteno.NewFileSink(logFilePath)
	}

	gosteno.Init(loggingConfig)
	logger := gosteno.NewLogger(name)
	logger.Debugf("Component %s in debug mode!", name)
	return logger
}
<commit_msg>Updated flags<commit_after>package main

import (
	// "fmt"
	"github.com/cloudfoundry/gosteno"
	"github.com/codegangsta/cli"
	"github.com/pivotalservices/cfops/backup"
	. "github.com/pivotalservices/cfops/cli"
	"github.com/pivotalservices/cfops/install"
	"github.com/pivotalservices/cfops/start"
	"github.com/pivotalservices/cfops/system"
	"os"
	"strings"
)

var (
	logFilePath    = NewFlag("logFile", "", "The CFOPS log file, defaults to STDOUT", "CFOPS_LOG")
	configFilePath = NewFlag("configFile", "config/cfops.json", "Location of the CFOPS config json file", "CFOPS_CONFIG")
	debug          = NewFlag("debug", false, "Debug logging", "CFOPS_TRACE")
	iaas           = NewFlag("iaas, i", "aws, vsphere, vcloud, openstack", "Sets the IaaS type to target for deployment", "CFOPS_IAAS")
	lang           = NewFlag("lang, l", "en, es", "Language for the cfops cli responses", "CFOPS_LANG")
)

type Config struct {
	Index uint
}

// To get the base foundation configuration see the Pivotal CF Data Collector @
// https://docs.google.com/a/pivotal.io/spreadsheets/d/1XHKSrJiQIu5MWGpPYWbMY8M09eqe-GV8MQsl_mqw1RM/edit#gid=0
func main() {

	commandFactory := NewCommandFactory()

	commandRunner := &system.OSCommandRunner{}

	start.New(commandFactory)
	install.New(commandFactory)
	backup.New(commandFactory, commandRunner)

	globalFlags := []*Flag{logFilePath, configFilePath, debug, iaas, lang}

	app := NewApp(commandFactory, globalFlags)

	app.Before = func(c *cli.Context) error {
		for _, flag := range globalFlags {
			if flag.Type == "bool" {
				flag.Value = c.Bool(flag.Name)
				// fmt.Printf("BEFORE flag %s has value %v\n", flag.Name, flag.Value)
			} else {
				flag.Value = c.String(flag.Name)
				// fmt.Printf("BEFORE flag %s has value %s\n", flag.Name, flag.Value)
			}
		}
		_, logger := parseConfig(debug.ParseBool(), configFilePath.ParseString(), logFilePath.ParseString())

		commandRunner.SetLogger(logger)
		commandFactory.SetLogger(logger)
		return nil
	}

	app.RunAndExitOnError()
}

func parseConfig(debug bool, configFile, logFilePath string) (Config, *gosteno.Logger) {
	config := Config{}
	// err := config.LoadConfig(&config, configFile)
	// if err != nil {
	// 	panic(err)
	// }

	logger := NewLogger(debug, logFilePath, "cfops", config)
	logger.Info("Setting up CFOPS profiler")

	return config, logger
}

func NewLogger(verbose bool, logFilePath, name string, config Config) *gosteno.Logger {
	level := gosteno.LOG_INFO

	if verbose {
		level = gosteno.LOG_DEBUG
	}

	loggingConfig := &gosteno.Config{
		Sinks:     make([]gosteno.Sink, 1),
		Level:     level,
		Codec:     gosteno.NewJsonPrettifier(gosteno.EXCLUDE_DATA),
		EnableLOC: true}

	if strings.TrimSpace(logFilePath) == "" {
		loggingConfig.Sinks[0] = gosteno.NewIOSink(os.Stdout)
	} else {
		loggingConfig.Sinks[0] = gosteno.NewFileSink(logFilePath)
	}

	gosteno.Init(loggingConfig)
	logger := gosteno.NewLogger(name)
	logger.Debugf("Component %s in debug mode!", name)
	return logger
}
<|endoftext|>