<commit_before>// Copyright 2015 Volker Dobler.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// latency.go contains checks against response time latency.

package ht

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"net/http/cookiejar"
	"os"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
)

func init() {
	RegisterCheck(&Latency{})
}

// ----------------------------------------------------------------------------
// Latency

// Latency provides checks against percentils of the response time latency.
type Latency struct {
	// N is the number if request to measure. It should be much larger
	// than Concurrent. Default is 50.
	N int `json:",omitempty"`

	// Concurrent is the number of concurrent requests in flight.
	// Defaults to 2.
	Concurrent int `json:",omitempty"`

	// Limits is a string of the following form:
	//    "50% ≤ 150; 80% ≤ 200; 95% ≤ 250; 0.9995 ≤ 900"
	// The limits above would require the median of the response
	// times to be <= 150 ms and would allow only 1 request in 2000 to
	// exced 900ms.
	// Note that it must be the ≤ sign (U+2264), a plain < or a <=
	// is not recognized.
	Limits string `json:",omitempty"`

	// IndividualSessions tries to run the concurrent requests in
	// individual sessions: A new one for each of the Concurrent many
	// requests (not N many sessions).
	// This is done by using a fresh cookiejar so it won't work if the
	// request requires prior login.
	IndividualSessions bool `json:",omitempty"`

	// If SkipChecks is true no checks are performed i.e. only the
	// requests are executed.
	SkipChecks bool `json:",omitempty"`

	// DumpTo is the filename where the latencies are reported.
	// The special values "stdout" and "stderr" are recognised.
	DumpTo string `json:",omitempty"`

	limits []latLimit
}

type latLimit struct {
	q   float64
	max time.Duration
}

// Execute implements Check's Execute method.
func (L *Latency) Execute(t *Test) error {
	var dumper io.Writer
	switch L.DumpTo {
	case "":
		dumper = ioutil.Discard
	case "stdout":
		dumper = os.Stdout
	case "stderr":
		dumper = os.Stderr
	default:
		file, err := os.OpenFile(L.DumpTo, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
		if err != nil {
			return err
		}
		defer file.Close()
		buffile := bufio.NewWriter(file)
		defer buffile.Flush()
		dumper = buffile
	}

	conc := L.Concurrent

	// Provide a set of Test instances to be executed.
	tests := []*Test{}
	for i := 0; i < conc; i++ {
		cpy, err := Merge(t)
		if err != nil {
			return err
		}
		cpy.Name = fmt.Sprintf("Latency-Test %d", i+1)
		checks := []Check{}
		for _, c := range t.Checks {
			if _, lt := c.(*Latency); L.SkipChecks || lt {
				continue
			}
			checks = append(checks, c)
		}
		cpy.Checks = checks
		cpy.Verbosity = 0

		if t.Jar != nil {
			if L.IndividualSessions {
				cpy.Jar, _ = cookiejar.New(nil)
			} else {
				cpy.Jar = t.Jar
			}
		}
		tests = append(tests, cpy)
	}

	results := make(chan latencyResult, 2*conc)

	// Synchronous warmup phase is much simpler.
	wg := &sync.WaitGroup{}
	started := time.Now()
	prewarmed := 0
	for prewarm := 0; prewarm < 2; prewarm++ {
		for i := 0; i < conc; i++ {
			prewarmed++
			wg.Add(1)
			go func(ex *Test) {
				ex.Run(t.Variables)
				wg.Done()
			}(tests[i])
		}
		wg.Wait()
	}
	offset := time.Since(started) / time.Duration(prewarmed*conc)

	// Conc requests generation workers execute their own test instance
	// until the channel running is closed.
	done := make(chan bool)
	wg2 := &sync.WaitGroup{}
	wg2.Add(1) // Add one sentinel value.
	for i := 0; i < conc; i++ {
		go func(ex *Test, id int) {
			for {
				wg2.Add(1)
				ex.Run(t.Variables)
				results <- latencyResult{
					status:   ex.Status,
					started:  ex.Started,
					duration: ex.Response.Duration,
					execBy:   id,
				}
				wg2.Done()
				select {
				case <-done:
					return
				default:
				}
			}
		}(tests[i], i)
		time.Sleep(offset)
	}

	// Collect results into data and signal end via done.
	data := make([]latencyResult, 2*L.N)
	counters := make([]int, Bogus)
	// TODO: clean t.Name from ',' and other stuff illegal in csv files.
	checkid := fmt.Sprintf("%s,%d", t.Name, L.Concurrent)
	seen := uint64(0) // Bitmap of testinstances who returned.
	all := (uint64(1) << uint(conc)) - 1
	measureFrom := 0
	n := 0
	started = time.Now()
	for n < len(data) {
		data[n] = <-results
		if seen == all {
			if measureFrom == 0 {
				measureFrom = n
			}
			counters[data[n].status]++
		} else {
			seen |= 1 << uint(data[n].execBy)
		}
		n++

		// TODO: Make limits configurable?
		if counters[Pass] == L.N ||
			counters[Fail] > L.N/5 ||
			counters[Error] > L.N/20 ||
			time.Since(started) > 2*time.Minute {
			break
		}
	}
	close(done)
	data = data[:n]

	// Drain rest; wait till requests currently in flight die.
	wg2.Done() // Done with sentinel value.
	wg2.Wait()

	completed := false
	if counters[Pass] == L.N {
		completed = true
	}
	for _, r := range data {
		fmt.Fprintf(dumper, "%s,%s,%s,%d,%t\n",
			checkid,
			r.started.Format(time.RFC3339Nano),
			r.status,
			r.duration/Duration(time.Millisecond),
			completed,
		)
	}
	if !completed {
		return fmt.Errorf("Got only %d PASS but %d FAIL and %d ERR",
			counters[Pass], counters[Fail], counters[Error])
	}

	latencies := []int{}
	for i, r := range data {
		if i < measureFrom || r.status != Pass {
			continue
		}
		latencies = append(latencies, int(r.duration)/int(time.Millisecond))
	}
	sort.Ints(latencies)

	errs := ""
	for _, lim := range L.limits {
		lat := time.Millisecond * time.Duration(quantile(latencies, lim.q))
		t.infof("Latency quantil (conc=%d) %0.2f%% ≤ %d ms",
			conc, lim.q*100, lat/time.Millisecond)
		if lat > lim.max {
			if errs != "" {
				errs += "; "
			}
			errs += fmt.Sprintf("%.2f%% = %s > limit %s",
				100*lim.q, lat, lim.max)
		}
	}
	if errs != "" {
		return fmt.Errorf("%s", errs)
	}

	return nil
}

type latencyResult struct {
	status   Status
	started  time.Time
	duration Duration
	execBy   int
}

// https://en.wikipedia.org/wiki/Quantile formula R-8
func quantile(x []int, p float64) int {
	N := float64(len(x))
	if p < 2.0/(3.0*(N+1.0/3.0)) {
		return x[0]
	}
	if p >= (N-1.0/3.0)/(N+1.0/3.0) {
		return x[len(x)-1]
	}

	h := (N+1.0/3.0)*p + 1.0/3.0
	fh := math.Floor(h)
	xl := x[int(fh)-1]
	xr := x[int(fh)]

	return xl + int((h-fh)*float64(xr-xl)+0.5)
}

// Prepare implements Check's Prepare method.
func (L *Latency) Prepare() error {
	if L.N == 0 {
		L.N = 50
	}
	if L.Concurrent == 0 {
		L.Concurrent = 2
	} else if L.Concurrent > 64 {
		return MalformedCheck{
			Err: fmt.Errorf("concurrency %d > allowed max of 64",
				L.Concurrent),
		}
	}

	if L.Limits == "" {
		L.Limits = "75% ≤ 500"
	}

	if err := L.parseLimit(); err != nil {
		fmt.Printf("err = %v\n", err)
		return MalformedCheck{Err: err}
	}

	return nil
}

func (L *Latency) parseLimit() error {
	parts := strings.Split(L.Limits, ";")
	for i := range parts {
		s := strings.TrimSpace(parts[i])
		q, t, err := parseQantileLimit(s)
		if err != nil {
			return err
		}

		L.limits = append(L.limits, latLimit{q: q, max: t})
	}

	return nil
}

func parseQantileLimit(s string) (float64, time.Duration, error) {
	parts := strings.SplitN(s, "≤", 2)
	if len(parts) != 2 {
		return 0, 0, fmt.Errorf("missing '≤'")
	}

	q := strings.TrimSpace(strings.TrimRight(parts[0], " %"))
	quantile, err := strconv.ParseFloat(q, 64)
	if err != nil {
		return 0, 0, err
	}
	if strings.Index(parts[0], "%") != -1 {
		quantile /= 100
	}
	if quantile < 0 || quantile > 1 {
		return 0, 0, fmt.Errorf("quantile %.3f out of range [0,1]", quantile)
	}

	b := strings.TrimSpace(strings.TrimLeft(parts[1], "="))

	m, err := time.ParseDuration(b)
	if err != nil {
		return 0, 0, err
	}
	if m <= 0 || m > 300*time.Second {
		return 0, 0, fmt.Errorf("limit %s out of range (0,300s]", m)
	}

	return quantile, m, nil
}

/*

For Conc==4 and RT_orig == 12  ==>  offset==3

 1 +---------+ +---------+ +---------+ +---------+ +---------+
 2    +---------+ +---------+ +---------+ +---------+ +---------+
 3       +---------+ +---------+ +---------+ +---------+ +---------+
 4          +---------+ +---------+ +---------+ +---------+ +---------+
         |  |           |           |
        -+--+-         -+-----------+-
        offset             RT_orig

*/
<commit_msg>ht: fix documentation of Latency limits<commit_after>// Copyright 2015 Volker Dobler.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// latency.go contains checks against response time latency.

package ht

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"net/http/cookiejar"
	"os"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
)

func init() {
	RegisterCheck(&Latency{})
}

// ----------------------------------------------------------------------------
// Latency

// Latency provides checks against percentils of the response time latency.
type Latency struct {
	// N is the number if request to measure. It should be much larger
	// than Concurrent. Default is 50.
	N int `json:",omitempty"`

	// Concurrent is the number of concurrent requests in flight.
	// Defaults to 2.
	Concurrent int `json:",omitempty"`

	// Limits is a string of the following form:
	//    "50% ≤ 150ms; 80% ≤ 200ms; 95% ≤ 250ms; 0.9995 ≤ 0.9s"
	// The limits above would require the median of the response
	// times to be <= 150 ms and would allow only 1 request in 2000 to
	// exced 900ms.
	// Note that it must be the ≤ sign (U+2264), a plain < or a <=
	// is not recognized.
	Limits string `json:",omitempty"`

	// IndividualSessions tries to run the concurrent requests in
	// individual sessions: A new one for each of the Concurrent many
	// requests (not N many sessions).
	// This is done by using a fresh cookiejar so it won't work if the
	// request requires prior login.
	IndividualSessions bool `json:",omitempty"`

	// If SkipChecks is true no checks are performed i.e. only the
	// requests are executed.
	SkipChecks bool `json:",omitempty"`

	// DumpTo is the filename where the latencies are reported.
	// The special values "stdout" and "stderr" are recognised.
	DumpTo string `json:",omitempty"`

	limits []latLimit
}

type latLimit struct {
	q   float64
	max time.Duration
}

// Execute implements Check's Execute method.
func (L *Latency) Execute(t *Test) error {
	var dumper io.Writer
	switch L.DumpTo {
	case "":
		dumper = ioutil.Discard
	case "stdout":
		dumper = os.Stdout
	case "stderr":
		dumper = os.Stderr
	default:
		file, err := os.OpenFile(L.DumpTo, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
		if err != nil {
			return err
		}
		defer file.Close()
		buffile := bufio.NewWriter(file)
		defer buffile.Flush()
		dumper = buffile
	}

	conc := L.Concurrent

	// Provide a set of Test instances to be executed.
	tests := []*Test{}
	for i := 0; i < conc; i++ {
		cpy, err := Merge(t)
		if err != nil {
			return err
		}
		cpy.Name = fmt.Sprintf("Latency-Test %d", i+1)
		checks := []Check{}
		for _, c := range t.Checks {
			if _, lt := c.(*Latency); L.SkipChecks || lt {
				continue
			}
			checks = append(checks, c)
		}
		cpy.Checks = checks
		cpy.Verbosity = 0

		if t.Jar != nil {
			if L.IndividualSessions {
				cpy.Jar, _ = cookiejar.New(nil)
			} else {
				cpy.Jar = t.Jar
			}
		}
		tests = append(tests, cpy)
	}

	results := make(chan latencyResult, 2*conc)

	// Synchronous warmup phase is much simpler.
	wg := &sync.WaitGroup{}
	started := time.Now()
	prewarmed := 0
	for prewarm := 0; prewarm < 2; prewarm++ {
		for i := 0; i < conc; i++ {
			prewarmed++
			wg.Add(1)
			go func(ex *Test) {
				ex.Run(t.Variables)
				wg.Done()
			}(tests[i])
		}
		wg.Wait()
	}
	offset := time.Since(started) / time.Duration(prewarmed*conc)

	// Conc requests generation workers execute their own test instance
	// until the channel running is closed.
	done := make(chan bool)
	wg2 := &sync.WaitGroup{}
	wg2.Add(1) // Add one sentinel value.
	for i := 0; i < conc; i++ {
		go func(ex *Test, id int) {
			for {
				wg2.Add(1)
				ex.Run(t.Variables)
				results <- latencyResult{
					status:   ex.Status,
					started:  ex.Started,
					duration: ex.Response.Duration,
					execBy:   id,
				}
				wg2.Done()
				select {
				case <-done:
					return
				default:
				}
			}
		}(tests[i], i)
		time.Sleep(offset)
	}

	// Collect results into data and signal end via done.
	data := make([]latencyResult, 2*L.N)
	counters := make([]int, Bogus)
	// TODO: clean t.Name from ',' and other stuff illegal in csv files.
	checkid := fmt.Sprintf("%s,%d", t.Name, L.Concurrent)
	seen := uint64(0) // Bitmap of testinstances who returned.
	all := (uint64(1) << uint(conc)) - 1
	measureFrom := 0
	n := 0
	started = time.Now()
	for n < len(data) {
		data[n] = <-results
		if seen == all {
			if measureFrom == 0 {
				measureFrom = n
			}
			counters[data[n].status]++
		} else {
			seen |= 1 << uint(data[n].execBy)
		}
		n++

		// TODO: Make limits configurable?
		if counters[Pass] == L.N ||
			counters[Fail] > L.N/5 ||
			counters[Error] > L.N/20 ||
			time.Since(started) > 2*time.Minute {
			break
		}
	}
	close(done)
	data = data[:n]

	// Drain rest; wait till requests currently in flight die.
	wg2.Done() // Done with sentinel value.
	wg2.Wait()

	completed := false
	if counters[Pass] == L.N {
		completed = true
	}
	for _, r := range data {
		fmt.Fprintf(dumper, "%s,%s,%s,%d,%t\n",
			checkid,
			r.started.Format(time.RFC3339Nano),
			r.status,
			r.duration/Duration(time.Millisecond),
			completed,
		)
	}
	if !completed {
		return fmt.Errorf("Got only %d PASS but %d FAIL and %d ERR",
			counters[Pass], counters[Fail], counters[Error])
	}

	latencies := []int{}
	for i, r := range data {
		if i < measureFrom || r.status != Pass {
			continue
		}
		latencies = append(latencies, int(r.duration)/int(time.Millisecond))
	}
	sort.Ints(latencies)

	errs := ""
	for _, lim := range L.limits {
		lat := time.Millisecond * time.Duration(quantile(latencies, lim.q))
		t.infof("Latency quantil (conc=%d) %0.2f%% ≤ %d ms",
			conc, lim.q*100, lat/time.Millisecond)
		if lat > lim.max {
			if errs != "" {
				errs += "; "
			}
			errs += fmt.Sprintf("%.2f%% = %s > limit %s",
				100*lim.q, lat, lim.max)
		}
	}
	if errs != "" {
		return fmt.Errorf("%s", errs)
	}

	return nil
}

type latencyResult struct {
	status   Status
	started  time.Time
	duration Duration
	execBy   int
}

// https://en.wikipedia.org/wiki/Quantile formula R-8
func quantile(x []int, p float64) int {
	N := float64(len(x))
	if p < 2.0/(3.0*(N+1.0/3.0)) {
		return x[0]
	}
	if p >= (N-1.0/3.0)/(N+1.0/3.0) {
		return x[len(x)-1]
	}

	h := (N+1.0/3.0)*p + 1.0/3.0
	fh := math.Floor(h)
	xl := x[int(fh)-1]
	xr := x[int(fh)]

	return xl + int((h-fh)*float64(xr-xl)+0.5)
}

// Prepare implements Check's Prepare method.
func (L *Latency) Prepare() error {
	if L.N == 0 {
		L.N = 50
	}
	if L.Concurrent == 0 {
		L.Concurrent = 2
	} else if L.Concurrent > 64 {
		return MalformedCheck{
			Err: fmt.Errorf("concurrency %d > allowed max of 64",
				L.Concurrent),
		}
	}

	if L.Limits == "" {
		L.Limits = "75% ≤ 500"
	}

	if err := L.parseLimit(); err != nil {
		fmt.Printf("err = %v\n", err)
		return MalformedCheck{Err: err}
	}

	return nil
}

func (L *Latency) parseLimit() error {
	parts := strings.Split(L.Limits, ";")
	for i := range parts {
		s := strings.TrimSpace(parts[i])
		q, t, err := parseQantileLimit(s)
		if err != nil {
			return err
		}

		L.limits = append(L.limits, latLimit{q: q, max: t})
	}

	return nil
}

func parseQantileLimit(s string) (float64, time.Duration, error) {
	parts := strings.SplitN(s, "≤", 2)
	if len(parts) != 2 {
		return 0, 0, fmt.Errorf("missing '≤'")
	}

	q := strings.TrimSpace(strings.TrimRight(parts[0], " %"))
	quantile, err := strconv.ParseFloat(q, 64)
	if err != nil {
		return 0, 0, err
	}
	if strings.Index(parts[0], "%") != -1 {
		quantile /= 100
	}
	if quantile < 0 || quantile > 1 {
		return 0, 0, fmt.Errorf("quantile %.3f out of range [0,1]", quantile)
	}

	b := strings.TrimSpace(strings.TrimLeft(parts[1], "="))

	m, err := time.ParseDuration(b)
	if err != nil {
		return 0, 0, err
	}
	if m <= 0 || m > 300*time.Second {
		return 0, 0, fmt.Errorf("limit %s out of range (0,300s]", m)
	}

	return quantile, m, nil
}

/*

For Conc==4 and RT_orig == 12  ==>  offset==3

 1 +---------+ +---------+ +---------+ +---------+ +---------+
 2    +---------+ +---------+ +---------+ +---------+ +---------+
 3       +---------+ +---------+ +---------+ +---------+ +---------+
 4          +---------+ +---------+ +---------+ +---------+ +---------+
         |  |           |           |
        -+--+-         -+-----------+-
        offset             RT_orig

*/
<|endoftext|>


<commit_before>// Copyright 2015 Volker Dobler.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// latency.go contains checks against response time latency.

package ht

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"os"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
)

func init() {
	RegisterCheck(&Latency{})
}

// ----------------------------------------------------------------------------
// Latency

type Latency struct {
	// N is the number if request to measure. It should be much larger
	// than Concurrent. Default is 50.
	N int `json:",omitempty"`

	// Concurrent is the number of concurrent requests in flight.
	// Defaults to 2.
	Concurrent int `json:",omitempty"`

	// Limits is a string of the following form:
	//    "50% ≤ 150; 80% ≤ 200; 95% ≤ 250; 0.9995 ≤ 900"
	// The limits above would require the median of the response
	// times to be <= 150 ms and would allow only 1 request in 2000 to
	// exced 900ms.
	// Note that it must be the ≤ signe (U+2264), a plain < or a <=
	// is not recognized.
	Limits string `json:",omitempty"`

	// If SkipChecks is true no checks are performed i.e. only the
	// requests are executed.
	SkipChecks bool `json:",omitempty"`

	// DumpTo is the filename where the latencies are reported.
	// The special values "stdout" and "stderr" are recognised.
	DumpTo string `json:",omitempty"`

	limits []latLimit
}

type latLimit struct {
	q   float64
	max time.Duration
}

// Execute implements Check's Execute method.
func (L *Latency) Execute(t *Test) error {
	var dumper io.Writer
	switch L.DumpTo {
	case "":
		dumper = ioutil.Discard
	case "stdout":
		dumper = os.Stdout
	case "stderr":
		dumper = os.Stderr
	default:
		file, err := os.OpenFile(L.DumpTo, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
		if err != nil {
			return err
		}
		defer file.Close()
		buffile := bufio.NewWriter(file)
		defer buffile.Flush()
		dumper = buffile
	}

	conc := L.Concurrent

	// Provide a set of Test instances to be executed.
	tests := []*Test{}
	for i := 0; i < conc; i++ {
		cpy, err := Merge(t)
		if err != nil {
			return err
		}
		cpy.Name = fmt.Sprintf("Latency-Test %d", i+1)
		checks := []Check{}
		for _, c := range t.Checks {
			if _, lt := c.(*Latency); L.SkipChecks || lt {
				continue
			}
			checks = append(checks, c)
		}
		cpy.Checks = checks
		cpy.Verbosity = 0
		tests = append(tests, cpy)
	}

	results := make(chan latencyResult, 2*conc)

	// Synchronous warmup phase is much simpler.
	wg := &sync.WaitGroup{}
	started := time.Now()
	prewarmed := 0
	for prewarm := 0; prewarm < 2; prewarm++ {
		for i := 0; i < conc; i++ {
			prewarmed++
			wg.Add(1)
			go func(ex *Test) {
				ex.Run(t.Variables)
				wg.Done()
			}(tests[i])
		}
		wg.Wait()
	}
	offset := time.Since(started) / time.Duration(prewarmed*conc)

	// Conc requests generation workers execute their own test instance
	// until the channel running is closed.
	done := make(chan bool)
	wg2 := &sync.WaitGroup{}
	wg2.Add(1) // Add one sentinel value.
	for i := 0; i < conc; i++ {
		go func(ex *Test, id int) {
			for {
				wg2.Add(1)
				ex.Run(nil) // TODO: get copy of variables from somewhere
				results <- latencyResult{
					status:   ex.Status,
					started:  ex.Started,
					duration: ex.Response.Duration,
					execBy:   id,
				}
				wg2.Done()
				select {
				case <-done:
					return
				default:
				}
			}
		}(tests[i], i)
		time.Sleep(offset)
	}

	// Collect results into data and signal end via done.
	data := make([]latencyResult, 2*L.N)
	counters := make([]int, Bogus)
	// TODO: clean t.Name from ',' and other stuff illegal in csv files.
	checkid := fmt.Sprintf("%s,%d", t.Name, L.Concurrent)
	seen := uint64(0) // Bitmap of testinstances who returned.
	all := (uint64(1) << uint(conc)) - 1
	measureFrom := 0
	n := 0
	started = time.Now()
	for n < len(data) {
		data[n] = <-results
		if seen == all {
			if measureFrom == 0 {
				measureFrom = n
			}
			counters[data[n].status]++
		} else {
			seen |= 1 << uint(data[n].execBy)
		}
		n++

		// TODO: Make limits configurable?
		if counters[Pass] == L.N ||
			counters[Fail] > L.N/5 ||
			counters[Error] > L.N/20 ||
			time.Since(started) > 2*time.Minute {
			break
		}
	}
	close(done)
	data = data[:n]

	// Drain rest; wait till requests currently in flight die.
	wg2.Done() // Done with sentinel value.
	wg2.Wait()

	completed := false
	if counters[Pass] == L.N {
		completed = true
	}
	for _, r := range data {
		fmt.Fprintf(dumper, "%s,%s,%s,%d,%t\n",
			checkid,
			r.started.Format(time.RFC3339Nano),
			r.status,
			r.duration/Duration(time.Millisecond),
			completed,
		)
	}
	if !completed {
		return fmt.Errorf("Got only %d PASS but %d FAIL and %d ERR",
			counters[Pass], counters[Fail], counters[Error])
	}

	latencies := []int{}
	for i, r := range data {
		if i < measureFrom || r.status != Pass {
			continue
		}
		latencies = append(latencies, int(r.duration)/int(time.Millisecond))
	}
	sort.Ints(latencies)

	errs := ""
	for _, lim := range L.limits {
		lat := time.Millisecond * time.Duration(quantile(latencies, lim.q))
		t.infof("Latency quantil (conc=%d) %0.4f = %d ms",
			conc, lim.q, lat/time.Millisecond)
		if lat > lim.max {
			if errs != "" {
				errs += "; "
			}
			errs += fmt.Sprintf("%.2f%% = %s > limit %s",
				100*lim.q, lat, lim.max)
		}
	}
	if errs != "" {
		return fmt.Errorf("%s", errs)
	}

	return nil
}

type latencyResult struct {
	status   Status
	started  time.Time
	duration Duration
	execBy   int
}

// https://en.wikipedia.org/wiki/Quantile formula R-8
func quantile(x []int, p float64) int {
	N := float64(len(x))
	if p < 2.0/(3.0*(N+1.0/3.0)) {
		return x[0]
	}
	if p >= (N-1.0/3.0)/(N+1.0/3.0) {
		return x[len(x)-1]
	}

	h := (N+1.0/3.0)*p + 1.0/3.0
	fh := math.Floor(h)
	xl := x[int(fh)-1]
	xr := x[int(fh)]

	return xl + int((h-fh)*float64(xr-xl)+0.5)
}

// Prepare implements Check's Prepare method.
func (L *Latency) Prepare() error {
	if L.N == 0 {
		L.N = 50
	}
	if L.Concurrent == 0 {
		L.Concurrent = 2
	} else if L.Concurrent > 64 {
		return MalformedCheck{
			Err: fmt.Errorf("concurrency %d > allowed max of 64",
				L.Concurrent),
		}
	}

	if L.Limits == "" {
		L.Limits = "75% ≤ 500"
	}

	if err := L.parseLimit(); err != nil {
		fmt.Printf("err = %v\n", err)
		return MalformedCheck{Err: err}
	}

	return nil
}

func (L *Latency) parseLimit() error {
	parts := strings.Split(L.Limits, ";")
	for i := range parts {
		s := strings.TrimSpace(parts[i])
		q, t, err := parseQantileLimit(s)
		if err != nil {
			return err
		}

		L.limits = append(L.limits, latLimit{q: q, max: t})
	}

	return nil
}

func parseQantileLimit(s string) (float64, time.Duration, error) {
	parts := strings.SplitN(s, "≤", 2)
	if len(parts) != 2 {
		return 0, 0, fmt.Errorf("missing '≤'")
	}

	q := strings.TrimSpace(strings.TrimRight(parts[0], " %"))
	quantile, err := strconv.ParseFloat(q, 64)
	if err != nil {
		return 0, 0, err
	}
	if strings.Index(parts[0], "%") != -1 {
		quantile /= 100
	}
	if quantile < 0 || quantile > 1 {
		return 0, 0, fmt.Errorf("quantile %.3f out of range [0,1]", quantile)
	}

	b := strings.TrimSpace(strings.TrimLeft(parts[1], "="))

	m, err := time.ParseDuration(b)
	if err != nil {
		return 0, 0, err
	}
	if m <= 0 || m > 300*time.Second {
		return 0, 0, fmt.Errorf("limit %s out of range (0,300s]", m)
	}

	return quantile, m, nil
}

/*

For Conc==4 and RT_orig == 12  ==>  offset==3

 1 +---------+ +---------+ +---------+ +---------+ +---------+
 2    +---------+ +---------+ +---------+ +---------+ +---------+
 3       +---------+ +---------+ +---------+ +---------+ +---------+
 4          +---------+ +---------+ +---------+ +---------+ +---------+
         |  |           |           |
        -+--+-         -+-----------+-
        offset             RT_orig

*/
<commit_msg>ht: realy keep variables during latency check<commit_after>// Copyright 2015 Volker Dobler.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// latency.go contains checks against response time latency.

package ht

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"os"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
)

func init() {
	RegisterCheck(&Latency{})
}

// ----------------------------------------------------------------------------
// Latency

type Latency struct {
	// N is the number if request to measure. It should be much larger
	// than Concurrent. Default is 50.
	N int `json:",omitempty"`

	// Concurrent is the number of concurrent requests in flight.
	// Defaults to 2.
	Concurrent int `json:",omitempty"`

	// Limits is a string of the following form:
	//    "50% ≤ 150; 80% ≤ 200; 95% ≤ 250; 0.9995 ≤ 900"
	// The limits above would require the median of the response
	// times to be <= 150 ms and would allow only 1 request in 2000 to
	// exced 900ms.
	// Note that it must be the ≤ signe (U+2264), a plain < or a <=
	// is not recognized.
	Limits string `json:",omitempty"`

	// If SkipChecks is true no checks are performed i.e. only the
	// requests are executed.
	SkipChecks bool `json:",omitempty"`

	// DumpTo is the filename where the latencies are reported.
	// The special values "stdout" and "stderr" are recognised.
	DumpTo string `json:",omitempty"`

	limits []latLimit
}

type latLimit struct {
	q   float64
	max time.Duration
}

// Execute implements Check's Execute method.
func (L *Latency) Execute(t *Test) error {
	var dumper io.Writer
	switch L.DumpTo {
	case "":
		dumper = ioutil.Discard
	case "stdout":
		dumper = os.Stdout
	case "stderr":
		dumper = os.Stderr
	default:
		file, err := os.OpenFile(L.DumpTo, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
		if err != nil {
			return err
		}
		defer file.Close()
		buffile := bufio.NewWriter(file)
		defer buffile.Flush()
		dumper = buffile
	}

	conc := L.Concurrent

	// Provide a set of Test instances to be executed.
	tests := []*Test{}
	for i := 0; i < conc; i++ {
		cpy, err := Merge(t)
		if err != nil {
			return err
		}
		cpy.Name = fmt.Sprintf("Latency-Test %d", i+1)
		checks := []Check{}
		for _, c := range t.Checks {
			if _, lt := c.(*Latency); L.SkipChecks || lt {
				continue
			}
			checks = append(checks, c)
		}
		cpy.Checks = checks
		cpy.Verbosity = 0
		tests = append(tests, cpy)
	}

	results := make(chan latencyResult, 2*conc)

	// Synchronous warmup phase is much simpler.
	wg := &sync.WaitGroup{}
	started := time.Now()
	prewarmed := 0
	for prewarm := 0; prewarm < 2; prewarm++ {
		for i := 0; i < conc; i++ {
			prewarmed++
			wg.Add(1)
			go func(ex *Test) {
				ex.Run(t.Variables)
				wg.Done()
			}(tests[i])
		}
		wg.Wait()
	}
	offset := time.Since(started) / time.Duration(prewarmed*conc)

	// Conc requests generation workers execute their own test instance
	// until the channel running is closed.
	done := make(chan bool)
	wg2 := &sync.WaitGroup{}
	wg2.Add(1) // Add one sentinel value.
	for i := 0; i < conc; i++ {
		go func(ex *Test, id int) {
			for {
				wg2.Add(1)
				ex.Run(t.Variables)
				results <- latencyResult{
					status:   ex.Status,
					started:  ex.Started,
					duration: ex.Response.Duration,
					execBy:   id,
				}
				wg2.Done()
				select {
				case <-done:
					return
				default:
				}
			}
		}(tests[i], i)
		time.Sleep(offset)
	}

	// Collect results into data and signal end via done.
	data := make([]latencyResult, 2*L.N)
	counters := make([]int, Bogus)
	// TODO: clean t.Name from ',' and other stuff illegal in csv files.
	checkid := fmt.Sprintf("%s,%d", t.Name, L.Concurrent)
	seen := uint64(0) // Bitmap of testinstances who returned.
	all := (uint64(1) << uint(conc)) - 1
	measureFrom := 0
	n := 0
	started = time.Now()
	for n < len(data) {
		data[n] = <-results
		if seen == all {
			if measureFrom == 0 {
				measureFrom = n
			}
			counters[data[n].status]++
		} else {
			seen |= 1 << uint(data[n].execBy)
		}
		n++

		// TODO: Make limits configurable?
		if counters[Pass] == L.N ||
			counters[Fail] > L.N/5 ||
			counters[Error] > L.N/20 ||
			time.Since(started) > 2*time.Minute {
			break
		}
	}
	close(done)
	data = data[:n]

	// Drain rest; wait till requests currently in flight die.
	wg2.Done() // Done with sentinel value.
	wg2.Wait()

	completed := false
	if counters[Pass] == L.N {
		completed = true
	}
	for _, r := range data {
		fmt.Fprintf(dumper, "%s,%s,%s,%d,%t\n",
			checkid,
			r.started.Format(time.RFC3339Nano),
			r.status,
			r.duration/Duration(time.Millisecond),
			completed,
		)
	}
	if !completed {
		return fmt.Errorf("Got only %d PASS but %d FAIL and %d ERR",
			counters[Pass], counters[Fail], counters[Error])
	}

	latencies := []int{}
	for i, r := range data {
		if i < measureFrom || r.status != Pass {
			continue
		}
		latencies = append(latencies, int(r.duration)/int(time.Millisecond))
	}
	sort.Ints(latencies)

	errs := ""
	for _, lim := range L.limits {
		lat := time.Millisecond * time.Duration(quantile(latencies, lim.q))
		t.infof("Latency quantil (conc=%d) %0.4f = %d ms",
			conc, lim.q, lat/time.Millisecond)
		if lat > lim.max {
			if errs != "" {
				errs += "; "
			}
			errs += fmt.Sprintf("%.2f%% = %s > limit %s",
				100*lim.q, lat, lim.max)
		}
	}
	if errs != "" {
		return fmt.Errorf("%s", errs)
	}

	return nil
}

type latencyResult struct {
	status   Status
	started  time.Time
	duration Duration
	execBy   int
}

// https://en.wikipedia.org/wiki/Quantile formula R-8
func quantile(x []int, p float64) int {
	N := float64(len(x))
	if p < 2.0/(3.0*(N+1.0/3.0)) {
		return x[0]
	}
	if p >= (N-1.0/3.0)/(N+1.0/3.0) {
		return x[len(x)-1]
	}

	h := (N+1.0/3.0)*p + 1.0/3.0
	fh := math.Floor(h)
	xl := x[int(fh)-1]
	xr := x[int(fh)]

	return xl + int((h-fh)*float64(xr-xl)+0.5)
}

// Prepare implements Check's Prepare method.
func (L *Latency) Prepare() error {
	if L.N == 0 {
		L.N = 50
	}
	if L.Concurrent == 0 {
		L.Concurrent = 2
	} else if L.Concurrent > 64 {
		return MalformedCheck{
			Err: fmt.Errorf("concurrency %d > allowed max of 64",
				L.Concurrent),
		}
	}

	if L.Limits == "" {
		L.Limits = "75% ≤ 500"
	}

	if err := L.parseLimit(); err != nil {
		fmt.Printf("err = %v\n", err)
		return MalformedCheck{Err: err}
	}

	return nil
}

func (L *Latency) parseLimit() error {
	parts := strings.Split(L.Limits, ";")
	for i := range parts {
		s := strings.TrimSpace(parts[i])
		q, t, err := parseQantileLimit(s)
		if err != nil {
			return err
		}

		L.limits = append(L.limits, latLimit{q: q, max: t})
	}

	return nil
}

func parseQantileLimit(s string) (float64, time.Duration, error) {
	parts := strings.SplitN(s, "≤", 2)
	if len(parts) != 2 {
		return 0, 0, fmt.Errorf("missing '≤'")
	}

	q := strings.TrimSpace(strings.TrimRight(parts[0], " %"))
	quantile, err := strconv.ParseFloat(q, 64)
	if err != nil {
		return 0, 0, err
	}
	if strings.Index(parts[0], "%") != -1 {
		quantile /= 100
	}
	if quantile < 0 || quantile > 1 {
		return 0, 0, fmt.Errorf("quantile %.3f out of range [0,1]", quantile)
	}

	b := strings.TrimSpace(strings.TrimLeft(parts[1], "="))

	m, err := time.ParseDuration(b)
	if err != nil {
		return 0, 0, err
	}
	if m <= 0 || m > 300*time.Second {
		return 0, 0, fmt.Errorf("limit %s out of range (0,300s]", m)
	}

	return quantile, m, nil
}

/*

For Conc==4 and RT_orig == 12  ==>  offset==3

 1 +---------+ +---------+ +---------+ +---------+ +---------+
 2    +---------+ +---------+ +---------+ +---------+ +---------+
 3       +---------+ +---------+ +---------+ +---------+ +---------+
 4          +---------+ +---------+ +---------+ +---------+ +---------+
         |  |           |           |
        -+--+-         -+-----------+-
        offset             RT_orig

*/
<|endoftext|>


<commit_before>// Package publicsuffix provides a domain name parser
// based on data from the public suffix list http://publicsuffix.org/.
// A public suffix is one under which Internet users can directly register names.
package publicsuffix

import (
	"bufio"
	"fmt"
	"io"
	"net/http/cookiejar"
	"os"
	"regexp"
	"strings"
)

const (
	NormalType    = 1
	WildcardType  = 2
	ExceptionType = 3

	listTokenPrivateDomains = "===BEGIN PRIVATE DOMAINS==="
	listTokenComment        = "//"
)

// defaultList is the default List and it is used by Parse and Domain.
var defaultList = NewList()

// DefaultRule is the default Rule that represents "*".
var DefaultRule = NewRule("*")

// DefaultParserOptions are the default options used to parse a Public Suffix list.
var DefaultParserOptions = &ParserOption{PrivateDomains: true}

// DefaultFindOptions are the default options used to perform the lookup of rules in the list.
var DefaultFindOptions = &FindOptions{IgnorePrivate: false}

// Rule represents a single rule in a Public Suffix List.
type Rule struct {
	Type    int
	Value   string
	Length  int
	Private bool
}

// ParserOption are the options you can use to customize the way a List
// is parsed from a file or a string.
type ParserOption struct {
	PrivateDomains bool
}

// FindOptions are the options you can use to customize the way a Rule
// is searched within the list.
type FindOptions struct {
	IgnorePrivate bool
}

// List represents a Public Suffix List.
type List struct {
	// rules is kept private because you should not access rules directly
	// for lookup optimization the list will not be guaranteed to be a simple slice forever
	rules []Rule
}

// NewList creates a new empty list.
func NewList() *List {
	return &List{}
}

// NewListFromString parses a string that represents a Public Suffix source
// and returns a List initialized with the rules in the source.
func NewListFromString(src string, options *ParserOption) (*List, error) {
	l := NewList()
	_, err := l.LoadString(src, options)
	return l, err
}

// NewListFromString parses a string that represents a Public Suffix source
// and returns a List initialized with the rules in the source.
func NewListFromFile(path string, options *ParserOption) (*List, error) {
	l := NewList()
	_, err := l.LoadFile(path, options)
	return l, err
}

// experimental
func (l *List) Load(r io.Reader, options *ParserOption) ([]Rule, error) {
	return l.parse(r, options)
}

// experimental
func (l *List) LoadString(src string, options *ParserOption) ([]Rule, error) {
	r := strings.NewReader(src)
	return l.parse(r, options)
}

// experimental
func (l *List) LoadFile(path string, options *ParserOption) ([]Rule, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	return l.parse(f, options)
}

// AddRule adds a new rule to the list.
//
// The exact position of the rule into the list is unpredictable.
// The list may be optimized internally for lookups, therefore the algorithm
// will decide the best position for the new rule.
func (l *List) AddRule(r *Rule) error {
	l.rules = append(l.rules, *r)
	return nil
}

// experimental
func (l *List) Size() int {
	return len(l.rules)
}

func (l *List) parse(r io.Reader, options *ParserOption) ([]Rule, error) {
	if options == nil {
		options = DefaultParserOptions
	}
	var rules []Rule

	scanner := bufio.NewScanner(r)
	var section int // 1 == ICANN, 2 == PRIVATE

Scanning:
	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		switch {

		// skip blank lines
		case line == "":
			break

		// include private domains or stop scanner
		case strings.Contains(line, listTokenPrivateDomains):
			if !options.PrivateDomains {
				break Scanning
			}
			section = 2

		// skip comments
		case strings.HasPrefix(line, listTokenComment):
			break

		default:
			rule := NewRule(line)
			rule.Private = (section == 2)
			l.AddRule(rule)
			rules = append(rules, *rule)
		}

	}

	return rules, scanner.Err()
}

// Finds and returns the most appropriate rule for the domain name.
func (l *List) Find(name string, options *FindOptions) Rule {
	var rule *Rule

	for _, r := range l.Select(name, options) {
		if r.Type == ExceptionType {
			return r
		}
		if rule == nil || rule.Length < r.Length {
			rule = &r
		}
	}

	if rule != nil {
		return *rule
	}

	return *DefaultRule
}

// experimental
func (l *List) Select(name string, options *FindOptions) []Rule {
	var found []Rule

	if options == nil {
		options = DefaultFindOptions
	}

	// In this phase the search is a simple sequential scan
	for _, rule := range l.rules {
		if !rule.Match(name) {
			continue
		}
		if options.IgnorePrivate && rule.Private {
			continue
		}
		found = append(found, rule)
	}

	return found
}

// NewRule parses the rule content, creates and returns a Rule.
func NewRule(content string) *Rule {
	var rule *Rule
	var value string

	switch content[0:1] {
	case "*": // wildcard
		if content == "*" {
			value = ""
		} else {
			value = content[2:]
		}
		rule = &Rule{Type: WildcardType, Value: value, Length: len(Labels(value)) + 1}
	case "!": // exception
		value = content[1:]
		rule = &Rule{Type: ExceptionType, Value: value, Length: len(Labels(value))}
	default: // normal
		value = content
		rule = &Rule{Type: NormalType, Value: value, Length: len(Labels(value))}
	}
	return rule
}

// Match checks if the rule matches the name.
//
// A domain name is said to match a rule if and only if all of the following conditions are met:
// - When the domain and rule are split into corresponding labels,
//   that the domain contains as many or more labels than the rule.
// - Beginning with the right-most labels of both the domain and the rule,
//   and continuing for all labels in the rule, one finds that for every pair,
//   either they are identical, or that the label from the rule is "*".
//
// See https://publicsuffix.org/list/
func (r *Rule) Match(name string) bool {
	left := strings.TrimSuffix(name, r.Value)

	// the name contains as many labels than the rule
	// this is a match, unless it's a wildcard
	// because the wildcard requires one more label
	if left == "" {
		return r.Type != WildcardType
	}

	// if there is one more label, the rule match
	// because either the rule is shorter than the domain
	// or the rule is a wildcard and there is one more label
	return left[len(left)-1:] == "."
}

// Decompose takes a name as input and decomposes it into a tuple of <TRD+SLD, TLD>,
// according to the rule definition and type.
func (r *Rule) Decompose(name string) [2]string {
	var parts []string

	switch r.Type {
	case WildcardType:
		parts = append([]string{`.*?`}, r.parts()...)
	default:
		parts = r.parts()
	}

	suffix := strings.Join(parts, `\.`)
	re := regexp.MustCompile(fmt.Sprintf(`^(.+)\.(%s)$`, suffix))

	matches := re.FindStringSubmatch(name)
	if len(matches) < 3 {
		return [2]string{"", ""}
	}

	return [2]string{matches[1], matches[2]}
}

func (r *Rule) parts() []string {
	labels := Labels(r.Value)
	if r.Type == ExceptionType {
		return labels[1:]
	}
	if r.Type == WildcardType && r.Value == "" {
		return []string{}
	}
	return labels
}

// Labels decomposes given domain name into labels,
// corresponding to the dot-separated tokens.
func Labels(name string) []string {
	return strings.Split(name, ".")
}

// DomainName represents a domain name.
type DomainName struct {
	TLD  string
	SLD  string
	TRD  string
	Rule *Rule
}

// String joins the components of the domain name into a single string.
// Empty labels are skipped.
//
// Examples:
//
// 	DomainName{"com", "example"}.String()
//	// example.com
// 	DomainName{"com", "example", "www"}.String()
//	// www.example.com
//
func (d *DomainName) String() string {
	switch {
	case d.TLD == "":
		return ""
	case d.SLD == "":
		return d.TLD
	case d.TRD == "":
		return d.SLD + "." + d.TLD
	default:
		return d.TRD + "." + d.SLD + "." + d.TLD
	}
}

// Domain extract and return the domain name from the input
// using the default (Public Suffix) List.
//
// Examples:
//
// 	publicsuffix.Domain("example.com")
//	// example.com
// 	publicsuffix.Domain("www.example.com)
//	// example.com
// 	publicsuffix.Domain("www.example.co.uk")
//	// example.co.uk
//
func Domain(name string) (string, error) {
	return DomainFromListWithOptions(DefaultList(), name, DefaultFindOptions)
}

// Parse decomposes the name into TLD, SLD, TRD
// using the default (Public Suffix) List,
// and returns the result as a DomainName
//
// Examples:
//
//	list := NewList()
//
// 	publicsuffix.Parse("example.com")
//	// &DomainName{"com", "example"}
// 	publicsuffix.Parse("www.example.com)
//	// &DomainName{"com", "example", "www"}
// 	publicsuffix.Parse("www.example.co.uk")
//	// &DomainName{"co.uk", "example"}
//
func Parse(name string) (*DomainName, error) {
	return ParseFromListWithOptions(DefaultList(), name, DefaultFindOptions)
}

// DomainFromListWithOptions extract and return the domain name from the input
// using the (Public Suffix) list passed as argument.
//
// Examples:
//
//	list := NewList()
//
// 	publicsuffix.DomainFromListWithOptions(list, "example.com")
//	// example.com
// 	publicsuffix.DomainFromListWithOptions(list, "www.example.com)
//	// example.com
// 	publicsuffix.DomainFromListWithOptions(list, "www.example.co.uk")
//	// example.co.uk
//
func DomainFromListWithOptions(l *List, name string, options *FindOptions) (string, error) {
	dn, err := ParseFromListWithOptions(l, name, options)
	if err != nil {
		return "", err
	}

	return dn.SLD + "." + dn.TLD, nil
}

// ParseFromListWithOptions decomposes the name into TLD, SLD, TRD
// using the (Public Suffix) list passed as argument,
// and returns the result as a DomainName
//
// Examples:
//
//	list := NewList()
//
// 	publicsuffix.ParseFromListWithOptions(list, "example.com")
//	// &DomainName{"com", "example"}
// 	publicsuffix.ParseFromListWithOptions(list, "www.example.com)
//	// &DomainName{"com", "example", "www"}
// 	publicsuffix.ParseFromListWithOptions(list, "www.example.co.uk")
//	// &DomainName{"co.uk", "example"}
//
func ParseFromListWithOptions(l *List, name string, options *FindOptions) (*DomainName, error) {
	n, err := normalize(name)
	if err != nil {
		return nil, err
	}

	r := l.Find(n, options)
	if tld := r.Decompose(n)[1]; tld == "" {
		return nil, fmt.Errorf("%s is a suffix", n)
	}

	dn := &DomainName{Rule: &r}
	dn.TLD, dn.SLD, dn.TRD = decompose(&r, n)
	return dn, nil
}

// DefaultList returns the default list, initialized with the rules stored in the list.
// The list is lazy-initialized the first time it is requested.
func DefaultList() *List {
	if defaultList.Size() == 0 {
		initDefaultList()
	}

	return defaultList
}

func normalize(name string) (string, error) {
	ret := strings.ToLower(name)

	if ret == "" {
		return "", fmt.Errorf("Name is blank")
	}
	if ret[0] == '.' {
		return "", fmt.Errorf("Name %s starts with a dot", ret)
	}

	return ret, nil
}

func decompose(r *Rule, name string) (tld, sld, trd string) {
	parts := r.Decompose(name)
	left, tld := parts[0], parts[1]

	dot := strings.LastIndex(left, ".")
	if dot == -1 {
		sld = left
		trd = ""
	} else {
		sld = left[dot+1:]
		trd = left[0:dot]
	}

	return
}

// CookieJarList implements the cookiejar.PublicSuffixList interface.
var CookieJarList cookiejar.PublicSuffixList = cookiejarList{}

type cookiejarList struct {
	List *List
}

// PublicSuffix implements cookiejar.PublicSuffixList.
func (l cookiejarList) PublicSuffix(domain string) string {
	if l.List == nil {
		l.List = DefaultList()
	}

	rule := l.List.Find(domain, nil)
	return rule.Decompose(domain)[1]
}

// PublicSuffix implements cookiejar.String.
func (cookiejarList) String() string {
	return defaultListVersion
}
<commit_msg>Fix doc mismatches<commit_after>// Package publicsuffix provides a domain name parser
// based on data from the public suffix list http://publicsuffix.org/.
// A public suffix is one under which Internet users can directly register names.
package publicsuffix

import (
	"bufio"
	"fmt"
	"io"
	"net/http/cookiejar"
	"os"
	"regexp"
	"strings"
)

const (
	NormalType    = 1
	WildcardType  = 2
	ExceptionType = 3

	listTokenPrivateDomains = "===BEGIN PRIVATE DOMAINS==="
	listTokenComment        = "//"
)

// defaultList is the default List and it is used by Parse and Domain.
var defaultList = NewList()

// DefaultRule is the default Rule that represents "*".
var DefaultRule = NewRule("*")

// DefaultParserOptions are the default options used to parse a Public Suffix list.
var DefaultParserOptions = &ParserOption{PrivateDomains: true}

// DefaultFindOptions are the default options used to perform the lookup of rules in the list.
var DefaultFindOptions = &FindOptions{IgnorePrivate: false}

// Rule represents a single rule in a Public Suffix List.
type Rule struct {
	Type    int
	Value   string
	Length  int
	Private bool
}

// ParserOption are the options you can use to customize the way a List
// is parsed from a file or a string.
type ParserOption struct {
	PrivateDomains bool
}

// FindOptions are the options you can use to customize the way a Rule
// is searched within the list.
type FindOptions struct {
	IgnorePrivate bool
}

// List represents a Public Suffix List.
type List struct {
	// rules is kept private because you should not access rules directly
	// for lookup optimization the list will not be guaranteed to be a simple slice forever
	rules []Rule
}

// NewList creates a new empty list.
func NewList() *List {
	return &List{}
}

// NewListFromString parses a string that represents a Public Suffix source
// and returns a List initialized with the rules in the source.
func NewListFromString(src string, options *ParserOption) (*List, error) {
	l := NewList()
	_, err := l.LoadString(src, options)
	return l, err
}

// NewListFromFile parses a string that represents a Public Suffix source
// and returns a List initialized with the rules in the source.
func NewListFromFile(path string, options *ParserOption) (*List, error) {
	l := NewList()
	_, err := l.LoadFile(path, options)
	return l, err
}

// experimental
func (l *List) Load(r io.Reader, options *ParserOption) ([]Rule, error) {
	return l.parse(r, options)
}

// experimental
func (l *List) LoadString(src string, options *ParserOption) ([]Rule, error) {
	r := strings.NewReader(src)
	return l.parse(r, options)
}

// experimental
func (l *List) LoadFile(path string, options *ParserOption) ([]Rule, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	return l.parse(f, options)
}

// AddRule adds a new rule to the list.
//
// The exact position of the rule into the list is unpredictable.
// The list may be optimized internally for lookups, therefore the algorithm
// will decide the best position for the new rule.
func (l *List) AddRule(r *Rule) error {
	l.rules = append(l.rules, *r)
	return nil
}

// experimental
func (l *List) Size() int {
	return len(l.rules)
}

func (l *List) parse(r io.Reader, options *ParserOption) ([]Rule, error) {
	if options == nil {
		options = DefaultParserOptions
	}
	var rules []Rule

	scanner := bufio.NewScanner(r)
	var section int // 1 == ICANN, 2 == PRIVATE

Scanning:
	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		switch {

		// skip blank lines
		case line == "":
			break

		// include private domains or stop scanner
		case strings.Contains(line, listTokenPrivateDomains):
			if !options.PrivateDomains {
				break Scanning
			}
			section = 2

		// skip comments
		case strings.HasPrefix(line, listTokenComment):
			break

		default:
			rule := NewRule(line)
			rule.Private = (section == 2)
			l.AddRule(rule)
			rules = append(rules, *rule)
		}

	}

	return rules, scanner.Err()
}

// Finds and returns the most appropriate rule for the domain name.
func (l *List) Find(name string, options *FindOptions) Rule {
	var rule *Rule

	for _, r := range l.Select(name, options) {
		if r.Type == ExceptionType {
			return r
		}
		if rule == nil || rule.Length < r.Length {
			rule = &r
		}
	}

	if rule != nil {
		return *rule
	}

	return *DefaultRule
}

// experimental
func (l *List) Select(name string, options *FindOptions) []Rule {
	var found []Rule

	if options == nil {
		options = DefaultFindOptions
	}

	// In this phase the search is a simple sequential scan
	for _, rule := range l.rules {
		if !rule.Match(name) {
			continue
		}
		if options.IgnorePrivate && rule.Private {
			continue
		}
		found = append(found, rule)
	}

	return found
}

// NewRule parses the rule content, creates and returns a Rule.
func NewRule(content string) *Rule {
	var rule *Rule
	var value string

	switch content[0:1] {
	case "*": // wildcard
		if content == "*" {
			value = ""
		} else {
			value = content[2:]
		}
		rule = &Rule{Type: WildcardType, Value: value, Length: len(Labels(value)) + 1}
	case "!": // exception
		value = content[1:]
		rule = &Rule{Type: ExceptionType, Value: value, Length: len(Labels(value))}
	default: // normal
		value = content
		rule = &Rule{Type: NormalType, Value: value, Length: len(Labels(value))}
	}
	return rule
}

// Match checks if the rule matches the name.
//
// A domain name is said to match a rule if and only if all of the following conditions are met:
// - When the domain and rule are split into corresponding labels,
//   that the domain contains as many or more labels than the rule.
// - Beginning with the right-most labels of both the domain and the rule,
//   and continuing for all labels in the rule, one finds that for every pair,
//   either they are identical, or that the label from the rule is "*".
//
// See https://publicsuffix.org/list/
func (r *Rule) Match(name string) bool {
	left := strings.TrimSuffix(name, r.Value)

	// the name contains as many labels than the rule
	// this is a match, unless it's a wildcard
	// because the wildcard requires one more label
	if left == "" {
		return r.Type != WildcardType
	}

	// if there is one more label, the rule match
	// because either the rule is shorter than the domain
	// or the rule is a wildcard and there is one more label
	return left[len(left)-1:] == "."
}

// Decompose takes a name as input and decomposes it into a tuple of <TRD+SLD, TLD>,
// according to the rule definition and type.
func (r *Rule) Decompose(name string) [2]string {
	var parts []string

	switch r.Type {
	case WildcardType:
		parts = append([]string{`.*?`}, r.parts()...)
	default:
		parts = r.parts()
	}

	suffix := strings.Join(parts, `\.`)
	re := regexp.MustCompile(fmt.Sprintf(`^(.+)\.(%s)$`, suffix))

	matches := re.FindStringSubmatch(name)
	if len(matches) < 3 {
		return [2]string{"", ""}
	}

	return [2]string{matches[1], matches[2]}
}

func (r *Rule) parts() []string {
	labels := Labels(r.Value)
	if r.Type == ExceptionType {
		return labels[1:]
	}
	if r.Type == WildcardType && r.Value == "" {
		return []string{}
	}
	return labels
}

// Labels decomposes given domain name into labels,
// corresponding to the dot-separated tokens.
func Labels(name string) []string {
	return strings.Split(name, ".")
}

// DomainName represents a domain name.
type DomainName struct {
	TLD  string
	SLD  string
	TRD  string
	Rule *Rule
}

// String joins the components of the domain name into a single string.
// Empty labels are skipped.
//
// Examples:
//
// 	DomainName{"com", "example"}.String()
//	// example.com
// 	DomainName{"com", "example", "www"}.String()
//	// www.example.com
//
func (d *DomainName) String() string {
	switch {
	case d.TLD == "":
		return ""
	case d.SLD == "":
		return d.TLD
	case d.TRD == "":
		return d.SLD + "." + d.TLD
	default:
		return d.TRD + "." + d.SLD + "." + d.TLD
	}
}

// Domain extract and return the domain name from the input
// using the default (Public Suffix) List.
//
// Examples:
//
// 	publicsuffix.Domain("example.com")
//	// example.com
// 	publicsuffix.Domain("www.example.com)
//	// example.com
// 	publicsuffix.Domain("www.example.co.uk")
//	// example.co.uk
//
func Domain(name string) (string, error) {
	return DomainFromListWithOptions(DefaultList(), name, DefaultFindOptions)
}

// Parse decomposes the name into TLD, SLD, TRD
// using the default (Public Suffix) List,
// and returns the result as a DomainName
//
// Examples:
//
//	list := NewList()
//
// 	publicsuffix.Parse("example.com")
//	// &DomainName{"com", "example"}
// 	publicsuffix.Parse("www.example.com)
//	// &DomainName{"com", "example", "www"}
// 	publicsuffix.Parse("www.example.co.uk")
//	// &DomainName{"co.uk", "example"}
//
func Parse(name string) (*DomainName, error) {
	return ParseFromListWithOptions(DefaultList(), name, DefaultFindOptions)
}

// DomainFromListWithOptions extract and return the domain name from the input
// using the (Public Suffix) list passed as argument.
//
// Examples:
//
//	list := NewList()
//
// 	publicsuffix.DomainFromListWithOptions(list, "example.com")
//	// example.com
// 	publicsuffix.DomainFromListWithOptions(list, "www.example.com)
//	// example.com
// 	publicsuffix.DomainFromListWithOptions(list, "www.example.co.uk")
//	// example.co.uk
//
func DomainFromListWithOptions(l *List, name string, options *FindOptions) (string, error) {
	dn, err := ParseFromListWithOptions(l, name, options)
	if err != nil {
		return "", err
	}

	return dn.SLD + "." + dn.TLD, nil
}

// ParseFromListWithOptions decomposes the name into TLD, SLD, TRD
// using the (Public Suffix) list passed as argument,
// and returns the result as a DomainName
//
// Examples:
//
//	list := NewList()
//
// 	publicsuffix.ParseFromListWithOptions(list, "example.com")
//	// &DomainName{"com", "example"}
// 	publicsuffix.ParseFromListWithOptions(list, "www.example.com)
//	// &DomainName{"com", "example", "www"}
// 	publicsuffix.ParseFromListWithOptions(list, "www.example.co.uk")
//	// &DomainName{"co.uk", "example"}
//
func ParseFromListWithOptions(l *List, name string, options *FindOptions) (*DomainName, error) {
	n, err := normalize(name)
	if err != nil {
		return nil, err
	}

	r := l.Find(n, options)
	if tld := r.Decompose(n)[1]; tld == "" {
		return nil, fmt.Errorf("%s is a suffix", n)
	}

	dn := &DomainName{Rule: &r}
	dn.TLD, dn.SLD, dn.TRD = decompose(&r, n)
	return dn, nil
}

// DefaultList returns the default list, initialized with the rules stored in the list.
// The list is lazy-initialized the first time it is requested.
func DefaultList() *List {
	if defaultList.Size() == 0 {
		initDefaultList()
	}

	return defaultList
}

func normalize(name string) (string, error) {
	ret := strings.ToLower(name)

	if ret == "" {
		return "", fmt.Errorf("Name is blank")
	}
	if ret[0] == '.' {
		return "", fmt.Errorf("Name %s starts with a dot", ret)
	}

	return ret, nil
}

func decompose(r *Rule, name string) (tld, sld, trd string) {
	parts := r.Decompose(name)
	left, tld := parts[0], parts[1]

	dot := strings.LastIndex(left, ".")
	if dot == -1 {
		sld = left
		trd = ""
	} else {
		sld = left[dot+1:]
		trd = left[0:dot]
	}

	return
}

// CookieJarList implements the cookiejar.PublicSuffixList interface.
var CookieJarList cookiejar.PublicSuffixList = cookiejarList{}

type cookiejarList struct {
	List *List
}

// PublicSuffix implements cookiejar.PublicSuffixList.
func (l cookiejarList) PublicSuffix(domain string) string {
	if l.List == nil {
		l.List = DefaultList()
	}

	rule := l.List.Find(domain, nil)
	return rule.Decompose(domain)[1]
}

// PublicSuffix implements cookiejar.String.
func (cookiejarList) String() string {
	return defaultListVersion
}
<|endoftext|>


<commit_before>package opts

import (
	"fmt"

	"github.com/docker/docker/pkg/blkiodev"
)

// WeightdeviceOpt defines a map of WeightDevices
type WeightdeviceOpt struct {
	values    []*blkiodev.WeightDevice
	validator ValidatorWeightFctType
}

// NewWeightdeviceOpt creates a new WeightdeviceOpt
func NewWeightdeviceOpt(validator ValidatorWeightFctType) WeightdeviceOpt {
	values := []*blkiodev.WeightDevice{}
	return WeightdeviceOpt{
		values:    values,
		validator: validator,
	}
}

// Set validates a WeightDevice and sets its name as a key in WeightdeviceOpt
func (opt *WeightdeviceOpt) Set(val string) error {
	var value *blkiodev.WeightDevice
	if opt.validator != nil {
		v, err := opt.validator(val)
		if err != nil {
			return err
		}
		value = v
	}
	(opt.values) = append((opt.values), value)
	return nil
}

// String returns Ulimit values as a string.
func (opt *WeightdeviceOpt) String() string {
	var out []string
	for _, v := range opt.values {
		out = append(out, v.String())
	}

	return fmt.Sprintf("%v", out)
}

// GetList returns a slice of pointers to WeightDevices.
func (opt *WeightdeviceOpt) GetList() []*blkiodev.WeightDevice {
	var weightdevice []*blkiodev.WeightDevice
	for _, v := range opt.values {
		weightdevice = append(weightdevice, v)
	}

	return weightdevice
}
<commit_msg>opts/weightdevice: fix typo<commit_after>package opts

import (
	"fmt"

	"github.com/docker/docker/pkg/blkiodev"
)

// WeightdeviceOpt defines a map of WeightDevices
type WeightdeviceOpt struct {
	values    []*blkiodev.WeightDevice
	validator ValidatorWeightFctType
}

// NewWeightdeviceOpt creates a new WeightdeviceOpt
func NewWeightdeviceOpt(validator ValidatorWeightFctType) WeightdeviceOpt {
	values := []*blkiodev.WeightDevice{}
	return WeightdeviceOpt{
		values:    values,
		validator: validator,
	}
}

// Set validates a WeightDevice and sets its name as a key in WeightdeviceOpt
func (opt *WeightdeviceOpt) Set(val string) error {
	var value *blkiodev.WeightDevice
	if opt.validator != nil {
		v, err := opt.validator(val)
		if err != nil {
			return err
		}
		value = v
	}
	(opt.values) = append((opt.values), value)
	return nil
}

// String returns WeightdeviceOpt values as a string.
func (opt *WeightdeviceOpt) String() string {
	var out []string
	for _, v := range opt.values {
		out = append(out, v.String())
	}

	return fmt.Sprintf("%v", out)
}

// GetList returns a slice of pointers to WeightDevices.
func (opt *WeightdeviceOpt) GetList() []*blkiodev.WeightDevice {
	var weightdevice []*blkiodev.WeightDevice
	for _, v := range opt.values {
		weightdevice = append(weightdevice, v)
	}

	return weightdevice
}
<|endoftext|>


<commit_before>package logging

import (
	"bytes"
	"io/ioutil"
	"time"

	"encoding/json"
	"github.com/gin-gonic/gin"
)

func GenerateLogInfo(c *gin.Context, start time.Time) LogInfo {
	return LogInfo{
		ClientIP:    c.ClientIP(),
		Date:        start.Format(time.RFC3339),
		Method:      c.Request.Method,
		RequestURI:  c.Request.URL.RequestURI(),
		Referer:     c.Request.Referer(),
		HTTPVersion: c.Request.Proto,
		Size:        c.Writer.Size(),
		Status:      c.Writer.Status(),
		UserAgent:   c.Request.UserAgent(),
		Latency:     time.Now().Sub(start),
	}
}

func ConvertToMapFromBody(c *gin.Context) (m map[string]interface{}, err error) {
	b, err := ioutil.ReadAll(c.Request.Body)
	if err != nil {
		return
	}
	c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(b))
	err = json.Unmarshal(b, &m)
	return
}
<commit_msg>Bugfix<commit_after>package logging

import (
	"bytes"
	"io/ioutil"
	"time"

	"encoding/json"
	"github.com/gin-gonic/gin"
)

func GenerateLogInfo(c *gin.Context, start time.Time) LogInfo {
	return LogInfo{
		ClientIP:    c.ClientIP(),
		Date:        start.Format(time.RFC3339),
		Method:      c.Request.Method,
		RequestURI:  c.Request.URL.RequestURI(),
		Referer:     c.Request.Referer(),
		HTTPVersion: c.Request.Proto,
		Size:        c.Writer.Size(),
		Status:      c.Writer.Status(),
		UserAgent:   c.Request.UserAgent(),
		Latency:     time.Now().Sub(start),
	}
}

func ConvertToMapFromBody(c *gin.Context) (m map[string]interface{}, err error) {
	b, err := ioutil.ReadAll(c.Request.Body)
	if err != nil {
		return
	}
	c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(b))

	if len(b) != 0 {
		err = json.Unmarshal(b, &m)
	}
	return
}
<|endoftext|>


<commit_before>// Copyright 2015 ThoughtWorks, Inc.

// This file is part of getgauge/html-report.

// getgauge/html-report is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// getgauge/html-report is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with getgauge/html-report.  If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"

	"encoding/json"

	"github.com/getgauge/common"
	"github.com/getgauge/html-report/gauge_messages"
	"github.com/getgauge/html-report/generator"
	"github.com/getgauge/html-report/listener"
)

const (
	defaultReportsDir           = "reports"
	gaugeReportsDirEnvName      = "gauge_reports_dir" // directory where reports are generated by plugins
	overwriteReportsEnvProperty = "overwrite_reports"
	resultJsFile                = "result.js"
	htmlReport                  = "html-report"
	setupAction                 = "setup"
	executionAction             = "execution"
	gaugeHost                   = "localhost"
	gaugePortEnv                = "plugin_connection_port"
	pluginActionEnv             = "html-report_action"
	timeFormat                  = "2006-01-02 15.04.05"
	defaultTheme                = "default"
	reportThemeProperty         = "GAUGE_HTML_REPORT_THEME_PATH"
	resultFile                  = "last_run_result.json"
)

var projectRoot string
var pluginDir string
var templateBasePath string

type nameGenerator interface {
	randomName() string
}

type timeStampedNameGenerator struct {
}

func (T timeStampedNameGenerator) randomName() string {
	return time.Now().Format(timeFormat)
}

func findPluginAndProjectRoot() {
	projectRoot = os.Getenv(common.GaugeProjectRootEnv)
	if projectRoot == "" {
		fmt.Printf("Environment variable '%s' is not set. \n", common.GaugeProjectRootEnv)
		os.Exit(1)
	}

	var err error
	pluginDir, err = os.Getwd()
	if err != nil {
		fmt.Printf("Error finding current working directory: %s \n", err)
		os.Exit(1)
	}
}

func createExecutionReport() {
	os.Chdir(projectRoot)
	listener, err := listener.NewGaugeListener(gaugeHost, os.Getenv(gaugePortEnv))
	if err != nil {
		fmt.Println("Could not create the gauge listener")
		os.Exit(1)
	}
	listener.OnSuiteResult(createReport)
	listener.Start()
}

func addDefaultPropertiesToProject() {
	defaultPropertiesFile := getDefaultPropertiesFile()

	reportsDirProperty := &(common.Property{
		Comment:      "The path to the gauge reports directory. Should be either relative to the project directory or an absolute path",
		Name:         gaugeReportsDirEnvName,
		DefaultValue: defaultReportsDir})

	overwriteReportProperty := &(common.Property{
		Comment:      "Set as false if gauge reports should not be overwritten on each execution. A new time-stamped directory will be created on each execution.",
		Name:         overwriteReportsEnvProperty,
		DefaultValue: "true"})

	if !common.FileExists(defaultPropertiesFile) {
		fmt.Printf("Failed to setup html report plugin in project. Default properties file does not exist at %s. \n", defaultPropertiesFile)
		return
	}
	if err := common.AppendProperties(defaultPropertiesFile, reportsDirProperty, overwriteReportProperty); err != nil {
		fmt.Printf("Failed to setup html report plugin in project: %s \n", err)
		return
	}
	fmt.Println("Succesfully added configurations for html-report to env/default/default.properties")
}

func getDefaultPropertiesFile() string {
	return filepath.Join(projectRoot, "env", "default", "default.properties")
}

func createReport(suiteResult *gauge_messages.SuiteExecutionResult) {
	projectRoot, err := common.GetProjectRoot()
	if err != nil {
		log.Fatalf("%s", err.Error())
	}
	reportsDir := getReportsDirectory(getNameGen())
	generator.ProjectRoot = projectRoot
	res := generator.ToSuiteResult(suiteResult.GetSuiteResult())
	go saveLastExecutionResult(res, reportsDir)
	generateReport(res, reportsDir, getThemePath())
}

func regenerateReport(inputFile, reportsDir, theme string) {
	b, err := ioutil.ReadFile(inputFile)
	if err != nil {
		log.Fatal(err.Error())
	}
	res := &generator.SuiteResult{}
	if err = json.Unmarshal(b, res); err != nil {
		log.Fatal(err.Error())
	}
	generator.CreateDirectory(reportsDir)
	if theme == "" {
		theme = getDefaultThemePath()
	}
	generateReport(res, reportsDir, theme)
}

func generateReport(res *generator.SuiteResult, reportsDir, themePath string) {
	err := generator.GenerateReports(res, reportsDir, themePath)
	if err != nil {
		log.Fatalf("Failed to generate reports: %s\n", err.Error())
	}
	err = copyReportTemplateFiles(reportsDir)
	if err != nil {
		log.Fatalf("Error copying template directory :%s\n", err.Error())
	}
	fmt.Printf("Successfully generated html-report to => %s\n", reportsDir)
}

func getNameGen() nameGenerator {
	var nameGen nameGenerator
	if shouldOverwriteReports() {
		nameGen = nil
	} else {
		nameGen = timeStampedNameGenerator{}
	}
	return nameGen
}

func getReportsDirectory(nameGen nameGenerator) string {
	reportsDir, err := filepath.Abs(os.Getenv(gaugeReportsDirEnvName))
	if reportsDir == "" || err != nil {
		reportsDir = defaultReportsDir
	}
	generator.CreateDirectory(reportsDir)
	var currentReportDir string
	if nameGen != nil {
		currentReportDir = filepath.Join(reportsDir, htmlReport, nameGen.randomName())
	} else {
		currentReportDir = filepath.Join(reportsDir, htmlReport)
	}
	generator.CreateDirectory(currentReportDir)
	return currentReportDir
}

func copyReportTemplateFiles(reportDir string) error {
	r := filepath.Join(getDefaultThemePath(), "assets")
	_, err := common.MirrorDir(r, reportDir)
	return err
}

func getDefaultThemePath() string {
	if templateBasePath == "" {
		dir, _ := getCurrentExecutableDir()
		templateBasePath = filepath.Join(dir, "..", "themes")
	}
	return filepath.Join(templateBasePath, "default")
}

func getThemePath() string {
	t := os.Getenv(reportThemeProperty)
	if t == "" {
		t = getDefaultThemePath()
	}
	return t
}

func getCurrentExecutableDir() (string, string) {
	ex, err := os.Executable()
	if err != nil {
		log.Fatalf(err.Error())
	}
	target, err := os.Readlink(ex)
	if err != nil {
		return path.Dir(ex), filepath.Base(ex)
	}
	return path.Dir(target), filepath.Base(ex)

}
func shouldOverwriteReports() bool {
	envValue := os.Getenv(overwriteReportsEnvProperty)
	if strings.ToLower(envValue) == "true" {
		return true
	}
	return false
}

func saveLastExecutionResult(r *generator.SuiteResult, reportsDir string) {
	o, err := json.Marshal(r)
	if err != nil {
		log.Printf("[Warning] Error saving Last Execution Run: %s\n", err.Error())
		return
	}
	outF := filepath.Join(reportsDir, resultFile)
	err = ioutil.WriteFile(outF, o, common.NewFilePermissions)
	if err != nil {
		log.Printf("[Warning] Failed to write to %s. Reason: %s\n", outF, err.Error())
		return
	}
	fmt.Printf("Result from current execution has been saved to %s\n", outF)
	dir, bName := getCurrentExecutableDir()
	exPath := filepath.Join(dir, bName)
	exTarget := filepath.Join(reportsDir, bName)
	if fileExists(exTarget) {
		os.Remove(exTarget)
	}
	err = os.Symlink(exPath, exTarget)
	instrFormat := "Run below command to regenerate the html-report\n\n  ./%s --input=%s --output=<OUT_DIR>\n  in working directory: %s\n\n"
	t := os.Getenv(reportThemeProperty)
	if t != "" {
		instrFormat = "Run below command to regenerate the html-report\n\n  ./%s --input=%s --output=<OUT_DIR> --theme=" + t +
			"\n  in working directory: %s\n\n"
	}

	if err != nil {
		log.Printf("[Warning] Unable to create symlink %s\n", exTarget)
		fmt.Printf(instrFormat, exPath, resultFile, reportsDir)
	}
	fmt.Printf(instrFormat, bName, resultFile, reportsDir)
}

func fileExists(path string) bool {
	_, err := os.Stat(path)
	if err == nil {
		return true
	}
	return !os.IsNotExist(err)
}
<commit_msg>remove verbose logging from html-report<commit_after>// Copyright 2015 ThoughtWorks, Inc.

// This file is part of getgauge/html-report.

// getgauge/html-report is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// getgauge/html-report is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with getgauge/html-report.  If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"

	"encoding/json"

	"github.com/getgauge/common"
	"github.com/getgauge/html-report/gauge_messages"
	"github.com/getgauge/html-report/generator"
	"github.com/getgauge/html-report/listener"
)

const (
	defaultReportsDir           = "reports"
	gaugeReportsDirEnvName      = "gauge_reports_dir" // directory where reports are generated by plugins
	overwriteReportsEnvProperty = "overwrite_reports"
	resultJsFile                = "result.js"
	htmlReport                  = "html-report"
	setupAction                 = "setup"
	executionAction             = "execution"
	gaugeHost                   = "localhost"
	gaugePortEnv                = "plugin_connection_port"
	pluginActionEnv             = "html-report_action"
	timeFormat                  = "2006-01-02 15.04.05"
	defaultTheme                = "default"
	reportThemeProperty         = "GAUGE_HTML_REPORT_THEME_PATH"
	resultFile                  = "last_run_result.json"
)

var projectRoot string
var pluginDir string
var templateBasePath string

type nameGenerator interface {
	randomName() string
}

type timeStampedNameGenerator struct {
}

func (T timeStampedNameGenerator) randomName() string {
	return time.Now().Format(timeFormat)
}

func findPluginAndProjectRoot() {
	projectRoot = os.Getenv(common.GaugeProjectRootEnv)
	if projectRoot == "" {
		fmt.Printf("Environment variable '%s' is not set. \n", common.GaugeProjectRootEnv)
		os.Exit(1)
	}

	var err error
	pluginDir, err = os.Getwd()
	if err != nil {
		fmt.Printf("Error finding current working directory: %s \n", err)
		os.Exit(1)
	}
}

func createExecutionReport() {
	os.Chdir(projectRoot)
	listener, err := listener.NewGaugeListener(gaugeHost, os.Getenv(gaugePortEnv))
	if err != nil {
		fmt.Println("Could not create the gauge listener")
		os.Exit(1)
	}
	listener.OnSuiteResult(createReport)
	listener.Start()
}

func addDefaultPropertiesToProject() {
	defaultPropertiesFile := getDefaultPropertiesFile()

	reportsDirProperty := &(common.Property{
		Comment:      "The path to the gauge reports directory. Should be either relative to the project directory or an absolute path",
		Name:         gaugeReportsDirEnvName,
		DefaultValue: defaultReportsDir})

	overwriteReportProperty := &(common.Property{
		Comment:      "Set as false if gauge reports should not be overwritten on each execution. A new time-stamped directory will be created on each execution.",
		Name:         overwriteReportsEnvProperty,
		DefaultValue: "true"})

	if !common.FileExists(defaultPropertiesFile) {
		fmt.Printf("Failed to setup html report plugin in project. Default properties file does not exist at %s. \n", defaultPropertiesFile)
		return
	}
	if err := common.AppendProperties(defaultPropertiesFile, reportsDirProperty, overwriteReportProperty); err != nil {
		fmt.Printf("Failed to setup html report plugin in project: %s \n", err)
		return
	}
	fmt.Println("Succesfully added configurations for html-report to env/default/default.properties")
}

func getDefaultPropertiesFile() string {
	return filepath.Join(projectRoot, "env", "default", "default.properties")
}

func createReport(suiteResult *gauge_messages.SuiteExecutionResult) {
	projectRoot, err := common.GetProjectRoot()
	if err != nil {
		log.Fatalf("%s", err.Error())
	}
	reportsDir := getReportsDirectory(getNameGen())
	generator.ProjectRoot = projectRoot
	res := generator.ToSuiteResult(suiteResult.GetSuiteResult())
	go saveLastExecutionResult(res, reportsDir)
	generateReport(res, reportsDir, getThemePath())
}

func regenerateReport(inputFile, reportsDir, theme string) {
	b, err := ioutil.ReadFile(inputFile)
	if err != nil {
		log.Fatal(err.Error())
	}
	res := &generator.SuiteResult{}
	if err = json.Unmarshal(b, res); err != nil {
		log.Fatal(err.Error())
	}
	generator.CreateDirectory(reportsDir)
	if theme == "" {
		theme = getDefaultThemePath()
	}
	generateReport(res, reportsDir, theme)
}

func generateReport(res *generator.SuiteResult, reportsDir, themePath string) {
	err := generator.GenerateReports(res, reportsDir, themePath)
	if err != nil {
		log.Fatalf("Failed to generate reports: %s\n", err.Error())
	}
	err = copyReportTemplateFiles(reportsDir)
	if err != nil {
		log.Fatalf("Error copying template directory :%s\n", err.Error())
	}
	fmt.Printf("Successfully generated html-report to => %s\n", reportsDir)
}

func getNameGen() nameGenerator {
	var nameGen nameGenerator
	if shouldOverwriteReports() {
		nameGen = nil
	} else {
		nameGen = timeStampedNameGenerator{}
	}
	return nameGen
}

func getReportsDirectory(nameGen nameGenerator) string {
	reportsDir, err := filepath.Abs(os.Getenv(gaugeReportsDirEnvName))
	if reportsDir == "" || err != nil {
		reportsDir = defaultReportsDir
	}
	generator.CreateDirectory(reportsDir)
	var currentReportDir string
	if nameGen != nil {
		currentReportDir = filepath.Join(reportsDir, htmlReport, nameGen.randomName())
	} else {
		currentReportDir = filepath.Join(reportsDir, htmlReport)
	}
	generator.CreateDirectory(currentReportDir)
	return currentReportDir
}

func copyReportTemplateFiles(reportDir string) error {
	r := filepath.Join(getDefaultThemePath(), "assets")
	_, err := common.MirrorDir(r, reportDir)
	return err
}

func getDefaultThemePath() string {
	if templateBasePath == "" {
		dir, _ := getCurrentExecutableDir()
		templateBasePath = filepath.Join(dir, "..", "themes")
	}
	return filepath.Join(templateBasePath, "default")
}

func getThemePath() string {
	t := os.Getenv(reportThemeProperty)
	if t == "" {
		t = getDefaultThemePath()
	}
	return t
}

func getCurrentExecutableDir() (string, string) {
	ex, err := os.Executable()
	if err != nil {
		log.Fatalf(err.Error())
	}
	target, err := os.Readlink(ex)
	if err != nil {
		return path.Dir(ex), filepath.Base(ex)
	}
	return path.Dir(target), filepath.Base(ex)

}
func shouldOverwriteReports() bool {
	envValue := os.Getenv(overwriteReportsEnvProperty)
	if strings.ToLower(envValue) == "true" {
		return true
	}
	return false
}

func saveLastExecutionResult(r *generator.SuiteResult, reportsDir string) {
	o, err := json.Marshal(r)
	if err != nil {
		log.Printf("[Warning] Error saving Last Execution Run: %s\n", err.Error())
		return
	}
	outF := filepath.Join(reportsDir, resultFile)
	err = ioutil.WriteFile(outF, o, common.NewFilePermissions)
	if err != nil {
		log.Printf("[Warning] Failed to write to %s. Reason: %s\n", outF, err.Error())
		return
	}
	fmt.Printf("Result from current execution has been saved to %s\n", outF)
	dir, bName := getCurrentExecutableDir()
	exPath := filepath.Join(dir, bName)
	exTarget := filepath.Join(reportsDir, bName)
	if fileExists(exTarget) {
		os.Remove(exTarget)
	}
	err = os.Symlink(exPath, exTarget)
	if err != nil {
		log.Printf("[Warning] Unable to create symlink %s\n", exTarget)
	}
}

func fileExists(path string) bool {
	_, err := os.Stat(path)
	if err == nil {
		return true
	}
	return !os.IsNotExist(err)
}
<|endoftext|>


<commit_before>/**
 * (C) Copyright 2013, Deft Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package dlshared

import (
	"net/url"
	"net/http"
	"io/ioutil"
	"time"
	"bytes"
	"strings"
	"crypto/tls"
	"labix.org/v2/mgo/bson"
	"encoding/json"
	"github.com/mreiferson/go-httpclient"
)

const (
	DefaultSocketTimeout = 4000
	HttpPostMethod = "POST"
	HttpGetMethod = "GET"

	ContentTypeHeader = "Content-Type"

	NoStatusCode = -1 // This is the value used if an error is generated before the status code is available

	ContentTypeTextPlain = "text/plain; charset=utf-8"
	ContentTypePostForm = "application/x-www-form-urlencoded"
	ContentTypeJson = "application/json"
)

// The http request client allows you to configure global values for timeout/connect etc. It also
// handles closing the transport and provides convenience methods for posting some types of data.
type HttpRequestClient struct {
	DisableKeepAlives bool
	DisableCompression bool
	SkipSslVerify bool
	MaxIdleConnsPerHost int
	ConnectTimeout time.Duration
	ResponseHeaderTimeout time.Duration
	RequestTimeout time.Duration
	ReadWriteTimeout time.Duration
}

func NewDefaultHttpRequestClient() *HttpRequestClient {
	return NewHttpRequestClient(true, false, false, 0, DefaultSocketTimeout, DefaultSocketTimeout, DefaultSocketTimeout, DefaultSocketTimeout)
}

func NewHttpRequestClient(	disableKeepAlives,
							disableCompression,
							skipSslVerify bool,
							maxIdleConnsPerHost,
							connectTimeoutInMs,
							responseHeaderTimeoutInMs,
							requestTimeoutInMs,
							readWriteTimeoutInMs int) *HttpRequestClient {

	return &HttpRequestClient{ 	DisableKeepAlives: disableKeepAlives,
								DisableCompression: disableCompression,
								SkipSslVerify: skipSslVerify,
								MaxIdleConnsPerHost: maxIdleConnsPerHost,
								ConnectTimeout: time.Duration(connectTimeoutInMs) * time.Millisecond,
								ResponseHeaderTimeout: time.Duration(responseHeaderTimeoutInMs) * time.Millisecond,
								RequestTimeout: time.Duration(requestTimeoutInMs) * time.Millisecond,
								ReadWriteTimeout: time.Duration(readWriteTimeoutInMs) * time.Millisecond,
	}
}

// Post the values to the url.
func (self *HttpRequestClient) Post(url string, values url.Values, headers map[string]string) (int, []byte, error) {

	client, transport := self.getClientAndTransport()
	defer transport.Close()

	request, err := http.NewRequest(HttpPostMethod, url,  strings.NewReader(values.Encode()))
	if err != nil { return NoStatusCode, nil, err }

	request.Header.Set(ContentTypeHeader, ContentTypePostForm)
	for key, value := range headers { request.Header.Set(key, value) }

	response, err := client.Do(request)
	if err != nil { return NoStatusCode, nil, err }

	statusCode := response.StatusCode

	defer response.Body.Close()

	if data, err := ioutil.ReadAll(response.Body); err != nil { return statusCode, nil, err
	} else { return statusCode, data, nil }
}

// Post the raw string to the url.
func (self *HttpRequestClient) PostStr(url string, value string, headers map[string]string) (int, []byte, error) {

	client, transport := self.getClientAndTransport()
	defer transport.Close()

	request, err := http.NewRequest(HttpPostMethod, url, bytes.NewReader([]byte(value)))
	if err != nil { return NoStatusCode, nil, err }

	request.Header.Set(ContentTypeHeader, ContentTypePostForm)
	for key, value := range headers { request.Header.Set(key, value) }

	response, err := client.Do(request)
	if err != nil { return NoStatusCode, nil, err }

	statusCode := response.StatusCode

	defer response.Body.Close()

	if data, err := ioutil.ReadAll(response.Body); err != nil { return statusCode, nil, err
	} else { return statusCode, data, nil }
}

// Post the json struct to the url.
func (self *HttpRequestClient) PostJson(url string, value interface{}, headers map[string]string) (int, []byte, error) {

	rawJson, err := json.Marshal(value)
	if err != nil { return NoStatusCode, nil, err }

	client, transport := self.getClientAndTransport()
	defer transport.Close()

	request, err := http.NewRequest(HttpPostMethod, url, bytes.NewReader(rawJson))
	if err != nil { return NoStatusCode, nil, err }

	request.Header.Set(ContentTypeHeader, ContentTypeJson)
	for key, value := range headers { request.Header.Set(key, value) }

	response, err := client.Do(request)
	if err != nil { return NoStatusCode, nil, err }

	statusCode := response.StatusCode

	defer response.Body.Close()

	if data, err := ioutil.ReadAll(response.Body); err != nil { return statusCode, nil, err
	} else { return statusCode, data, nil }
}

// Post the bson doc to the url.
func (self *HttpRequestClient) PostBson(url string, bsonDoc interface{}, headers map[string]string) (int, []byte, error) {

	rawBson, err := bson.Marshal(bsonDoc)
	if err != nil { return NoStatusCode, nil, err }

	client, transport := self.getClientAndTransport()
	defer transport.Close()

	request, err := http.NewRequest(HttpPostMethod, url, bytes.NewReader(rawBson))
	if err != nil { return NoStatusCode, nil, err }

	request.Header.Set(ContentTypeHeader, ContentTypePostForm)
	for key, value := range headers { request.Header.Set(key, value) }

	response, err := client.Do(request)
	if err != nil { return NoStatusCode, nil, err }

	statusCode := response.StatusCode

	defer response.Body.Close()

	// We do not return the response so don't report if there is an error.
	if data, err := ioutil.ReadAll(response.Body); err != nil { return statusCode, nil, err
	} else { return statusCode, data, nil }
}

// Issue a GET to retrieve a bson doc.
func (self *HttpRequestClient) GetBson(url string, headers map[string]string) (int, bson.M, error) {

	client, transport := self.getClientAndTransport()
	defer transport.Close()

	request, requestErr := http.NewRequest(HttpGetMethod, url, nil)
	if requestErr != nil { return NoStatusCode, nil, requestErr }

	for key, value := range headers { request.Header.Set(key, value) }

	response, err := client.Do(request)
	if err != nil { return NoStatusCode, nil, err }

	statusCode := response.StatusCode

	defer response.Body.Close()

	rawBson, err := ioutil.ReadAll(response.Body)
	if err != nil { return statusCode, nil, err }

	var bsonDoc bson.M
	if err := bson.Unmarshal(rawBson, &bsonDoc); err != nil { return statusCode, nil, err }

	return statusCode, bsonDoc, nil
}

// Use the clone method if you need to override some/all of the configured values.
func (self *HttpRequestClient) Clone() *HttpRequestClient {
	val := *self
	return &val
}

func (self *HttpRequestClient) getClientAndTransport() (*http.Client, *httpclient.Transport) {
	transport := self.getTransport()
	return &http.Client{ Transport: transport }, transport
}

func (self *HttpRequestClient) getTransport() *httpclient.Transport {
	return &httpclient.Transport {

		DisableKeepAlives: self.DisableKeepAlives,
		DisableCompression: self.DisableCompression,

		MaxIdleConnsPerHost: self.MaxIdleConnsPerHost,

		ConnectTimeout: self.ConnectTimeout,
		ResponseHeaderTimeout: self.ResponseHeaderTimeout,
		RequestTimeout: self.RequestTimeout,
		ReadWriteTimeout: self.ReadWriteTimeout,

		TLSClientConfig: &tls.Config{ InsecureSkipVerify: self.SkipSslVerify },
	}
}

func HttpPost(url string, values url.Values, headers map[string]string) (int, []byte, error) { return NewDefaultHttpRequestClient().Post(url, values, headers) }

func HttpPostStr(url string, value string, headers map[string]string) (int, []byte, error) { return NewDefaultHttpRequestClient().PostStr(url, value, headers) }

func HttpPostJson(url string, value interface{}, headers map[string]string) (int, []byte, error) { return NewDefaultHttpRequestClient().PostJson(url, value, headers) }

func HttpPostBson(url string, bsonDoc interface{}, headers map[string]string) (int, []byte, error) { return NewDefaultHttpRequestClient().PostBson(url, bsonDoc, headers) }

func HttpGetBson(url string, headers map[string]string) (int, bson.M, error) { return NewDefaultHttpRequestClient().GetBson(url, headers) }

<commit_msg>increased default timeout<commit_after>/**
 * (C) Copyright 2013, Deft Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package dlshared

import (
	"net/url"
	"net/http"
	"io/ioutil"
	"time"
	"bytes"
	"strings"
	"crypto/tls"
	"labix.org/v2/mgo/bson"
	"encoding/json"
	"github.com/mreiferson/go-httpclient"
)

const (
	DefaultSocketTimeout = 40000
	HttpPostMethod = "POST"
	HttpGetMethod = "GET"

	ContentTypeHeader = "Content-Type"

	NoStatusCode = -1 // This is the value used if an error is generated before the status code is available

	ContentTypeTextPlain = "text/plain; charset=utf-8"
	ContentTypePostForm = "application/x-www-form-urlencoded"
	ContentTypeJson = "application/json"
)

// The http request client allows you to configure global values for timeout/connect etc. It also
// handles closing the transport and provides convenience methods for posting some types of data.
type HttpRequestClient struct {
	DisableKeepAlives bool
	DisableCompression bool
	SkipSslVerify bool
	MaxIdleConnsPerHost int
	ConnectTimeout time.Duration
	ResponseHeaderTimeout time.Duration
	RequestTimeout time.Duration
	ReadWriteTimeout time.Duration
}

func NewDefaultHttpRequestClient() *HttpRequestClient {
	return NewHttpRequestClient(true, false, false, 0, DefaultSocketTimeout, DefaultSocketTimeout, DefaultSocketTimeout, DefaultSocketTimeout)
}

func NewHttpRequestClient(	disableKeepAlives,
							disableCompression,
							skipSslVerify bool,
							maxIdleConnsPerHost,
							connectTimeoutInMs,
							responseHeaderTimeoutInMs,
							requestTimeoutInMs,
							readWriteTimeoutInMs int) *HttpRequestClient {

	return &HttpRequestClient{ 	DisableKeepAlives: disableKeepAlives,
								DisableCompression: disableCompression,
								SkipSslVerify: skipSslVerify,
								MaxIdleConnsPerHost: maxIdleConnsPerHost,
								ConnectTimeout: time.Duration(connectTimeoutInMs) * time.Millisecond,
								ResponseHeaderTimeout: time.Duration(responseHeaderTimeoutInMs) * time.Millisecond,
								RequestTimeout: time.Duration(requestTimeoutInMs) * time.Millisecond,
								ReadWriteTimeout: time.Duration(readWriteTimeoutInMs) * time.Millisecond,
	}
}

// Post the values to the url.
func (self *HttpRequestClient) Post(url string, values url.Values, headers map[string]string) (int, []byte, error) {

	client, transport := self.getClientAndTransport()
	defer transport.Close()

	request, err := http.NewRequest(HttpPostMethod, url,  strings.NewReader(values.Encode()))
	if err != nil { return NoStatusCode, nil, err }

	request.Header.Set(ContentTypeHeader, ContentTypePostForm)
	for key, value := range headers { request.Header.Set(key, value) }

	response, err := client.Do(request)
	if err != nil { return NoStatusCode, nil, err }

	statusCode := response.StatusCode

	defer response.Body.Close()

	if data, err := ioutil.ReadAll(response.Body); err != nil { return statusCode, nil, err
	} else { return statusCode, data, nil }
}

// Post the raw string to the url.
func (self *HttpRequestClient) PostStr(url string, value string, headers map[string]string) (int, []byte, error) {

	client, transport := self.getClientAndTransport()
	defer transport.Close()

	request, err := http.NewRequest(HttpPostMethod, url, bytes.NewReader([]byte(value)))
	if err != nil { return NoStatusCode, nil, err }

	request.Header.Set(ContentTypeHeader, ContentTypePostForm)
	for key, value := range headers { request.Header.Set(key, value) }

	response, err := client.Do(request)
	if err != nil { return NoStatusCode, nil, err }

	statusCode := response.StatusCode

	defer response.Body.Close()

	if data, err := ioutil.ReadAll(response.Body); err != nil { return statusCode, nil, err
	} else { return statusCode, data, nil }
}

// Post the json struct to the url.
func (self *HttpRequestClient) PostJson(url string, value interface{}, headers map[string]string) (int, []byte, error) {

	rawJson, err := json.Marshal(value)
	if err != nil { return NoStatusCode, nil, err }

	client, transport := self.getClientAndTransport()
	defer transport.Close()

	request, err := http.NewRequest(HttpPostMethod, url, bytes.NewReader(rawJson))
	if err != nil { return NoStatusCode, nil, err }

	request.Header.Set(ContentTypeHeader, ContentTypeJson)
	for key, value := range headers { request.Header.Set(key, value) }

	response, err := client.Do(request)
	if err != nil { return NoStatusCode, nil, err }

	statusCode := response.StatusCode

	defer response.Body.Close()

	if data, err := ioutil.ReadAll(response.Body); err != nil { return statusCode, nil, err
	} else { return statusCode, data, nil }
}

// Post the bson doc to the url.
func (self *HttpRequestClient) PostBson(url string, bsonDoc interface{}, headers map[string]string) (int, []byte, error) {

	rawBson, err := bson.Marshal(bsonDoc)
	if err != nil { return NoStatusCode, nil, err }

	client, transport := self.getClientAndTransport()
	defer transport.Close()

	request, err := http.NewRequest(HttpPostMethod, url, bytes.NewReader(rawBson))
	if err != nil { return NoStatusCode, nil, err }

	request.Header.Set(ContentTypeHeader, ContentTypePostForm)
	for key, value := range headers { request.Header.Set(key, value) }

	response, err := client.Do(request)
	if err != nil { return NoStatusCode, nil, err }

	statusCode := response.StatusCode

	defer response.Body.Close()

	// We do not return the response so don't report if there is an error.
	if data, err := ioutil.ReadAll(response.Body); err != nil { return statusCode, nil, err
	} else { return statusCode, data, nil }
}

// Issue a GET to retrieve a bson doc.
func (self *HttpRequestClient) GetBson(url string, headers map[string]string) (int, bson.M, error) {

	client, transport := self.getClientAndTransport()
	defer transport.Close()

	request, requestErr := http.NewRequest(HttpGetMethod, url, nil)
	if requestErr != nil { return NoStatusCode, nil, requestErr }

	for key, value := range headers { request.Header.Set(key, value) }

	response, err := client.Do(request)
	if err != nil { return NoStatusCode, nil, err }

	statusCode := response.StatusCode

	defer response.Body.Close()

	rawBson, err := ioutil.ReadAll(response.Body)
	if err != nil { return statusCode, nil, err }

	var bsonDoc bson.M
	if err := bson.Unmarshal(rawBson, &bsonDoc); err != nil { return statusCode, nil, err }

	return statusCode, bsonDoc, nil
}

// Use the clone method if you need to override some/all of the configured values.
func (self *HttpRequestClient) Clone() *HttpRequestClient {
	val := *self
	return &val
}

func (self *HttpRequestClient) getClientAndTransport() (*http.Client, *httpclient.Transport) {
	transport := self.getTransport()
	return &http.Client{ Transport: transport }, transport
}

func (self *HttpRequestClient) getTransport() *httpclient.Transport {
	return &httpclient.Transport {

		DisableKeepAlives: self.DisableKeepAlives,
		DisableCompression: self.DisableCompression,

		MaxIdleConnsPerHost: self.MaxIdleConnsPerHost,

		ConnectTimeout: self.ConnectTimeout,
		ResponseHeaderTimeout: self.ResponseHeaderTimeout,
		RequestTimeout: self.RequestTimeout,
		ReadWriteTimeout: self.ReadWriteTimeout,

		TLSClientConfig: &tls.Config{ InsecureSkipVerify: self.SkipSslVerify },
	}
}

func HttpPost(url string, values url.Values, headers map[string]string) (int, []byte, error) { return NewDefaultHttpRequestClient().Post(url, values, headers) }

func HttpPostStr(url string, value string, headers map[string]string) (int, []byte, error) { return NewDefaultHttpRequestClient().PostStr(url, value, headers) }

func HttpPostJson(url string, value interface{}, headers map[string]string) (int, []byte, error) { return NewDefaultHttpRequestClient().PostJson(url, value, headers) }

func HttpPostBson(url string, bsonDoc interface{}, headers map[string]string) (int, []byte, error) { return NewDefaultHttpRequestClient().PostBson(url, bsonDoc, headers) }

func HttpGetBson(url string, headers map[string]string) (int, bson.M, error) { return NewDefaultHttpRequestClient().GetBson(url, headers) }

<|endoftext|>