<commit_before>/*
Tensile web stress test tool

Mike Hughes 2014
intermernet AT gmail DOT com

LICENSE BSD 3 Clause
*/
package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"runtime"
	"sync"
	"time"
)

const (
	app     = "\n\tTensile web stress test tool v%s\n\n"
	version = "0.1"
)

var (
	reqs     int
	max      int
	numCPU   int
	maxCPU   int
	maxErr   int
	errCount int

	urlStr string

	flagErr     string
	reqsError   string = "ERROR: -reqs must be greater than 0\n"
	maxError    string = "ERROR: -concurrent must be greater than 0\n"
	maxErrError string = "ERROR: -errorlimit must be greater than 0\n"
	urlError    string = "ERROR: -url cannot be blank\n"
	schemeError string = "ERROR: unsupported protocol scheme %s\n"

	cpuWarn       string = "NOTICE: -cpu=%d is greater than the number of CPUs on this system\n\tChanging -cpu to %d\n\n"
	maxGTreqsWarn string = "NOTICE: -concurrent=%d is greater than -requests\n\tChanging -concurrent to %d\n\n"

	wg sync.WaitGroup
)

func init() {
	flag.StringVar(&urlStr, "url", "http://localhost/", "Target URL")
	flag.StringVar(&urlStr, "u", "http://localhost/", "Target URL (short flag)")
	flag.IntVar(&reqs, "requests", 50, "Total requests")
	flag.IntVar(&reqs, "r", 50, "Total requests (short flag)")
	flag.IntVar(&max, "concurrent", 5, "Maximum concurrent requests")
	flag.IntVar(&max, "c", 5, "Maximum concurrent requests (short flag)")
	flag.IntVar(&maxErr, "errorlimit", 1, "Maximum errors")
	flag.IntVar(&maxErr, "e", 1, "Maximum errors (short flag)")
	maxCPU = runtime.NumCPU()
	flag.IntVar(&numCPU, "cpu", maxCPU, "Number of CPUs")
}

type response struct {
	*http.Response
	err error
}

func processing(waitChan chan bool) {
	wg.Wait()
	waitChan <- true
}

// Dispatcher
func dispatcher(reqChan chan *http.Request) {
	defer close(reqChan)
	for i := 0; i < reqs; i++ {
		req, err := http.NewRequest("GET", urlStr, nil)
		if err != nil {
			log.Println(err)
		}
		reqChan <- req
	}
}

// Worker Pool
func workerPool(reqChan chan *http.Request, respChan chan response, quit chan bool) {
	defer close(respChan)
	t := &http.Transport{}
	defer t.CloseIdleConnections()
	for i := 0; i < max; i++ {
		wg.Add(1)
		go worker(t, reqChan, respChan, quit)
	}
	waitChan := make(chan bool)
	go processing(waitChan)
	for {
		select {
		case <-waitChan:
			return
		case <-quit:
			return
		}
	}
}

// Worker
func worker(t *http.Transport, reqChan chan *http.Request, respChan chan response, quit chan bool) {
	defer wg.Done()
	for req := range reqChan {
		select {
		case <-quit:
			return
		default:
			resp, err := t.RoundTrip(req)
			r := response{resp, err}
			respChan <- r
		}
	}
}

// Consumer
func consumer(respChan chan response, quit chan bool) (int64, int64) {
	defer close(quit)
	var (
		conns int64
		size  int64
	)
	for r := range respChan {
		if r.err != nil || r.StatusCode >= 400 {
			if r.err != nil {
				log.Println(r.err)
			} else {
				log.Println(r.Status)
			}
			errCount++
		} else {
			size += r.ContentLength
		}
		if err := r.Body.Close(); err != nil {
			log.Println(r.err)
		}
		if errCount >= maxErr {
			for i := 0; i <= max; i++ {
				quit <- true
			}
			return conns, size
		}
		conns++
	}
	return conns, size
}

func main() {
	// Flag checks
	flag.Parse()
	fmt.Printf(app, version)
	if reqs <= 0 {
		flagErr += reqsError
	}
	if max <= 0 {
		flagErr += maxError
	}
	if maxErr <= 0 {
		flagErr += maxErrError
	}
	if urlStr == "" {
		flagErr += urlError
	}
	u, err := url.Parse(urlStr)
	if err != nil {
		flagErr += err.Error()
	}
	if u.Scheme != "http" && u.Scheme != "https" {
		flagErr += fmt.Sprintf(schemeError, u.Scheme)
	}
	if flagErr != "" {
		log.Fatal(fmt.Errorf("\n%s", flagErr))
	}
	if numCPU > maxCPU {
		fmt.Printf(cpuWarn, numCPU, maxCPU)
		numCPU = maxCPU
	}
	if max > reqs {
		fmt.Printf(maxGTreqsWarn, max, reqs)
		max = reqs
	}
	// Start
	runtime.GOMAXPROCS(numCPU)
	reqChan := make(chan *http.Request)
	respChan := make(chan response)
	quit := make(chan bool)
	fmt.Printf("Sending %d requests to %s with %d concurrent workers using %d CPUs.\n\n", reqs, urlStr, max, numCPU)
	start := time.Now()
	go dispatcher(reqChan)
	go workerPool(reqChan, respChan, quit)
	fmt.Println("Waiting for replies...\n")
	conns, size := consumer(respChan, quit)
	// Calculate stats
	took := time.Since(start)
	tookNS := took.Nanoseconds()
	var averageNS int64
	if conns != 0 {
		averageNS = tookNS / conns
	}
	average, err := time.ParseDuration(fmt.Sprintf("%d", averageNS) + "ns")
	if err != nil {
		log.Println(err)
	}
	fmt.Printf("Connections:\t%d\nConcurrent:\t%d\nTotal size:\t%d bytes\nTotal time:\t%s\nAverage time:\t%s\n", conns, max, size, took, average)
}
<commit_msg>Revert "development"<commit_after>/*
Tensile web stress test tool

Mike Hughes 2014
intermernet AT gmail DOT com

LICENSE BSD 3 Clause
*/
package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"runtime"
	"sync"
	"time"
)

const (
	app     = "\n\tTensile web stress test tool v%s\n\n"
	version = "0.1"
)

var (
	reqs   int
	max    int
	numCPU int
	maxCPU int

	urlStr string

	flagErr     string
	reqsError   string = "ERROR: -reqs must be greater than 0\n"
	maxError    string = "ERROR: -concurrent must be greater than 0\n"
	urlError    string = "ERROR: -url cannot be blank\n"
	schemeError string = "ERROR: unsupported protocol scheme %s\n"

	cpuWarn       string = "NOTICE: -cpu=%d is greater than the number of CPUs on this system\n\tChanging -cpu to %d\n\n"
	maxGTreqsWarn string = "NOTICE: -concurrent=%d is greater than -requests\n\tChanging -concurrent to %d\n\n"

	wg sync.WaitGroup
)

func init() {
	flag.StringVar(&urlStr, "url", "http://localhost/", "Target URL")
	flag.StringVar(&urlStr, "u", "http://localhost/", "Target URL (short flag)")
	flag.IntVar(&reqs, "requests", 50, "Total requests")
	flag.IntVar(&reqs, "r", 50, "Total requests (short flag)")
	flag.IntVar(&max, "concurrent", 5, "Maximum concurrent requests")
	flag.IntVar(&max, "c", 5, "Maximum concurrent requests (short flag)")
	maxCPU = runtime.NumCPU()
	flag.IntVar(&numCPU, "cpu", maxCPU, "Number of CPUs")
}

type Response struct {
	*http.Response
	err error
}

// Dispatcher
func dispatcher(reqChan chan *http.Request) {
	defer close(reqChan)
	for i := 0; i < reqs; i++ {
		req, err := http.NewRequest("GET", urlStr, nil)
		if err != nil {
			log.Println(err)
		}
		reqChan <- req
	}
}

// Worker Pool
func workerPool(reqChan chan *http.Request, respChan chan Response) {
	defer close(respChan)
	t := &http.Transport{}
	defer t.CloseIdleConnections()
	for i := 0; i < max; i++ {
		wg.Add(1)
		go worker(t, reqChan, respChan)
	}
	wg.Wait()
}

// Worker
func worker(t *http.Transport, reqChan chan *http.Request, respChan chan Response) {
	defer wg.Done()
	for req := range reqChan {
		resp, err := t.RoundTrip(req)
		r := Response{resp, err}
		respChan <- r
	}
}

// Consumer
func consumer(respChan chan Response) (int64, int64) {
	var (
		conns int64
		size  int64
	)
	for r := range respChan {
		if r.err != nil {
			log.Println(r.err)
		} else {
			size += r.ContentLength
			if err := r.Body.Close(); err != nil {
				log.Println(r.err)
			}
		}
		conns++
	}
	return conns, size
}

func main() {
	// Flag checks
	flag.Parse()
	fmt.Printf(app, version)
	if reqs <= 0 {
		flagErr += reqsError
	}
	if max <= 0 {
		flagErr += maxError
	}
	if urlStr == "" {
		flagErr += urlError
	}
	u, err := url.Parse(urlStr)
	if err != nil {
		flagErr += err.Error()
	}
	if u.Scheme != "http" && u.Scheme != "https" {
		flagErr += fmt.Sprintf(schemeError, u.Scheme)
	}
	if flagErr != "" {
		log.Fatal(fmt.Errorf("\n%s", flagErr))
	}
	if numCPU > maxCPU {
		fmt.Printf(cpuWarn, numCPU, maxCPU)
		numCPU = maxCPU
	}
	if max > reqs {
		fmt.Printf(maxGTreqsWarn, max, reqs)
		max = reqs
	}
	// Start
	runtime.GOMAXPROCS(numCPU)
	reqChan := make(chan *http.Request)
	respChan := make(chan Response)
	fmt.Printf("Sending %d requests to %s with %d concurrent workers using %d CPUs.\n\n", reqs, urlStr, max, numCPU)
	start := time.Now()
	go dispatcher(reqChan)
	go workerPool(reqChan, respChan)
	fmt.Println("Waiting for replies...\n")
	conns, size := consumer(respChan)
	// Calculate stats
	took := time.Since(start)
	tookNS := took.Nanoseconds()
	var averageNS int64
	if conns != 0 {
		averageNS = tookNS / conns
	}
	average, err := time.ParseDuration(fmt.Sprintf("%d", averageNS) + "ns")
	if err != nil {
		log.Println(err)
	}
	fmt.Printf("Connections:\t%d\nConcurrent:\t%d\nTotal size:\t%d bytes\nTotal time:\t%s\nAverage time:\t%s\n", conns, max, size, took, average)
}
<|endoftext|>


<commit_before>package http

import (
	"context"
	"database/sql/driver"
	"fmt"
	"net/url"
	"strconv"
	"strings"

	"github.com/pkg/errors"

	"github.com/amsokol/go-ignite-client/http"
)

type conn struct {
	client *http.Client
}

// See https://golang.org/pkg/database/sql/driver/#Conn for more details
func (c *conn) Prepare(query string) (driver.Stmt, error) {
	return c.PrepareContext(context.Background(), query)
}

// See https://golang.org/pkg/database/sql/driver/#ConnPrepareContext for more details
func (c *conn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) {
	if c.client == nil {
		return nil, driver.ErrBadConn
	}
	return &stmt{connection: c, query: query}, nil
}

// See https://golang.org/pkg/database/sql/driver/#Conn for more details
func (c *conn) Close() error {
	c.client = nil

	return nil
}

// See https://golang.org/pkg/database/sql/driver/#ConnBeginTx for more details
func (c *conn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) {
	return nil, nil
}

// See https://golang.org/pkg/database/sql/driver/#Conn
func (c *conn) Begin() (driver.Tx, error) {
	return c.BeginTx(nil, driver.TxOptions{})
}

func (c *conn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) {
	v, err := c.namedValues2UrlValues(args)
	if err != nil {
		return nil, errors.Wrap(err, "Failed convert parameters for REST API")
	}

	_, _, err = c.client.QryFldExe(query, v)
	if err != nil {
		return nil, errors.Wrap(err, "Failed to invoke 'qryfldexe' command")
	}

	return &result{}, nil
}

func (c *conn) Ping(ctx context.Context) error {
	_, _, err := c.client.Version()
	return err
}

func (c *conn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) {
	if c.client == nil {
		return nil, driver.ErrBadConn
	}

	v, err := c.namedValues2UrlValues(args)
	if err != nil {
		return nil, errors.Wrap(err, "Failed convert parameters for REST API 'qryfldexe' command")
	}

	r, _, err := c.client.QryFldExe(query, v)
	if err != nil {
		return nil, errors.Wrap(err, "Failed to invoke 'qryfldexe' command")
	}

	cl := len(r.FieldsMetadata)
	rows := rows{connection: c, queryId: fmt.Sprintf("%d", r.QueryID), last: r.Last, columns: make([]column, cl, cl)}

	// columns
	for i, c := range r.FieldsMetadata {
		rows.columns[i] = column{name: c.FieldName, igniteType: c.FieldTypeName}
	}

	// data
	err = rows.setResultSet(r.Items)
	if err != nil {
		return nil, errors.Wrap(err, "Failed extract ResultSet from 'qryfldexe' response")
	}

	return &rows, nil
}

func (c *conn) fetchContext(ctx context.Context, queryId string) ([][]interface{}, bool, error) {
	if c.client == nil {
		return nil, false, driver.ErrBadConn
	}

	r, _, err := c.client.QryFetch(queryId)
	if err != nil {
		return nil, false, errors.Wrap(err, "Failed to invoke 'qryfldexe' command")
	}

	return r.Items, r.Last, nil
}

func (c *conn) closeQueryContext(ctx context.Context, queryId string) error {
	if c.client == nil {
		return driver.ErrBadConn
	}

	_, _, err := c.client.QryCls(queryId)

	return err
}

func (c *conn) namedValues2UrlValues(nvs []driver.NamedValue) (*url.Values, error) {
	vs := url.Values{}

	l := len(nvs)
	for i := 1; i <= l; i++ {
		for _, nv := range nvs {
			if nv.Ordinal == i {
				if nv.Value == nil {
					return nil, errors.WithStack(errors.New("Ignite HTTP REST API does not support NULL as parameter"))
				}
				var av string
				switch v := nv.Value.(type) {
				case int8:
					av = fmt.Sprintf("%d", v)
				case int16:
					av = fmt.Sprintf("%d", v)
				case int32:
					av = fmt.Sprintf("%d", v)
				case int64:
					av = fmt.Sprintf("%d", v)
				case float64:
					av = fmt.Sprintf("%f", v)
				case float32:
					av = fmt.Sprintf("%f", v)
				case bool:
					av = fmt.Sprintf("%t", v)
				case string:
					av = v
				// TODO: add binary support
				// TODO: add time.Time support
				default:
					return nil, errors.WithStack(errors.New(strings.Join([]string{"Unsupported parameter type with index", strconv.Itoa(i)}, " ")))
				}
				vs.Add(strings.Join([]string{"arg", strconv.Itoa(i)}, ""), av)
				break
			}
		}
	}
	return &vs, nil
}
<commit_msg>ignite REST API does not support transactions<commit_after>package http

import (
	"context"
	"database/sql/driver"
	"fmt"
	"net/url"
	"strconv"
	"strings"

	"github.com/pkg/errors"

	"github.com/amsokol/go-ignite-client/http"
)

type conn struct {
	client *http.Client
}

// See https://golang.org/pkg/database/sql/driver/#Conn for more details
func (c *conn) Prepare(query string) (driver.Stmt, error) {
	return c.PrepareContext(context.Background(), query)
}

// See https://golang.org/pkg/database/sql/driver/#ConnPrepareContext for more details
func (c *conn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) {
	if c.client == nil {
		return nil, driver.ErrBadConn
	}
	return &stmt{connection: c, query: query}, nil
}

// See https://golang.org/pkg/database/sql/driver/#Conn for more details
func (c *conn) Close() error {
	c.client = nil

	return nil
}

// See https://golang.org/pkg/database/sql/driver/#ConnBeginTx for more details
func (c *conn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) {
	return nil, errors.New("Ignite REST API does not support transactions")
}

// See https://golang.org/pkg/database/sql/driver/#Conn
func (c *conn) Begin() (driver.Tx, error) {
	return c.BeginTx(nil, driver.TxOptions{})
}

func (c *conn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) {
	v, err := c.namedValues2UrlValues(args)
	if err != nil {
		return nil, errors.Wrap(err, "Failed convert parameters for REST API")
	}

	_, _, err = c.client.QryFldExe(query, v)
	if err != nil {
		return nil, errors.Wrap(err, "Failed to invoke 'qryfldexe' command")
	}

	return &result{}, nil
}

func (c *conn) Ping(ctx context.Context) error {
	_, _, err := c.client.Version()
	return err
}

func (c *conn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) {
	if c.client == nil {
		return nil, driver.ErrBadConn
	}

	v, err := c.namedValues2UrlValues(args)
	if err != nil {
		return nil, errors.Wrap(err, "Failed convert parameters for REST API 'qryfldexe' command")
	}

	r, _, err := c.client.QryFldExe(query, v)
	if err != nil {
		return nil, errors.Wrap(err, "Failed to invoke 'qryfldexe' command")
	}

	cl := len(r.FieldsMetadata)
	rows := rows{connection: c, queryId: fmt.Sprintf("%d", r.QueryID), last: r.Last, columns: make([]column, cl, cl)}

	// columns
	for i, c := range r.FieldsMetadata {
		rows.columns[i] = column{name: c.FieldName, igniteType: c.FieldTypeName}
	}

	// data
	err = rows.setResultSet(r.Items)
	if err != nil {
		return nil, errors.Wrap(err, "Failed extract ResultSet from 'qryfldexe' response")
	}

	return &rows, nil
}

func (c *conn) fetchContext(ctx context.Context, queryId string) ([][]interface{}, bool, error) {
	if c.client == nil {
		return nil, false, driver.ErrBadConn
	}

	r, _, err := c.client.QryFetch(queryId)
	if err != nil {
		return nil, false, errors.Wrap(err, "Failed to invoke 'qryfldexe' command")
	}

	return r.Items, r.Last, nil
}

func (c *conn) closeQueryContext(ctx context.Context, queryId string) error {
	if c.client == nil {
		return driver.ErrBadConn
	}

	_, _, err := c.client.QryCls(queryId)

	return err
}

func (c *conn) namedValues2UrlValues(nvs []driver.NamedValue) (*url.Values, error) {
	vs := url.Values{}

	l := len(nvs)
	for i := 1; i <= l; i++ {
		for _, nv := range nvs {
			if nv.Ordinal == i {
				if nv.Value == nil {
					return nil, errors.WithStack(errors.New("Ignite HTTP REST API does not support NULL as parameter"))
				}
				var av string
				switch v := nv.Value.(type) {
				case int8:
					av = fmt.Sprintf("%d", v)
				case int16:
					av = fmt.Sprintf("%d", v)
				case int32:
					av = fmt.Sprintf("%d", v)
				case int64:
					av = fmt.Sprintf("%d", v)
				case float64:
					av = fmt.Sprintf("%f", v)
				case float32:
					av = fmt.Sprintf("%f", v)
				case bool:
					av = fmt.Sprintf("%t", v)
				case string:
					av = v
				// TODO: add binary support
				// TODO: add time.Time support
				default:
					return nil, errors.WithStack(errors.New(strings.Join([]string{"Unsupported parameter type with index", strconv.Itoa(i)}, " ")))
				}
				vs.Add(strings.Join([]string{"arg", strconv.Itoa(i)}, ""), av)
				break
			}
		}
	}
	return &vs, nil
}
<|endoftext|>


<commit_before>package aspect

import (
	"fmt"
	"runtime"
	"strings"
	"testing"
)

// callerInfo returns a string containing the file and line number of the
// assert call that failed.
// https://github.com/stretchr/testify/blob/master/assert/assertions.go
// Copyright (c) 2012 - 2013 Mat Ryer and Tyler Bunnell
func callerInfo() string {
	file := ""
	line := 0
	ok := false

	for i := 0; ; i++ {
		_, file, line, ok = runtime.Caller(i)
		if !ok {
			return ""
		}
		parts := strings.Split(file, "/")
		file = parts[len(parts)-1]

		// dir := parts[len(parts)-2]
		if file == "testing.go" {
			continue
		}
		break
	}
	return fmt.Sprintf("%s:%d", file, line)
}

type sqlTest struct {
	t       *testing.T
	dialect Dialect
}

// SQL tests that the given Compiles instance matches the expected string for
// the current dialect.
func (t *sqlTest) SQL(expect string, stmt Compiles, ps ...interface{}) {
	// Get caller information in case of failure
	caller := callerInfo()

	// Start a new parameters instance
	params := Params()

	// Compile the given stmt with the tester's dialect
	actual, err := stmt.Compile(t.dialect, params)
	if err != nil {
		t.t.Error("%s: unexpected error from compile: %s", caller, err)
		return
	}

	if expect != actual {
		t.t.Errorf(
			"%s: unexpected SQL: expect %s, got %s",
			caller,
			expect,
			actual,
		)
	}
	// Test that the parameters are equal
	if params.Len() != len(ps) {
		t.t.Errorf(
			"%s: unexpected number of parameters for %s: expect %d, got %d",
			caller,
			actual,
			params.Len(),
			len(ps),
		)
		return
	}

	// TODO Examine individual parameters for equality
}

func NewTester(t *testing.T, d Dialect) *sqlTest {
	return &sqlTest{t: t, dialect: d}
}
<commit_msg>Added Error method to sqlTest to test for expected errors<commit_after>package aspect

import (
	"fmt"
	"runtime"
	"strings"
	"testing"
)

// callerInfo returns a string containing the file and line number of the
// assert call that failed.
// https://github.com/stretchr/testify/blob/master/assert/assertions.go
// Copyright (c) 2012 - 2013 Mat Ryer and Tyler Bunnell
func callerInfo() string {
	file := ""
	line := 0
	ok := false

	for i := 0; ; i++ {
		_, file, line, ok = runtime.Caller(i)
		if !ok {
			return ""
		}
		parts := strings.Split(file, "/")
		file = parts[len(parts)-1]

		// dir := parts[len(parts)-2]
		if file == "testing.go" {
			continue
		}
		break
	}
	return fmt.Sprintf("%s:%d", file, line)
}

type sqlTest struct {
	t       *testing.T
	dialect Dialect
}

// Error tests that the given Compiles instances generates an error for the
// current dialect.
func (t *sqlTest) Error(stmt Compiles) {
	if _, err := stmt.Compile(t.dialect, Params()); err == nil {
		t.t.Errorf("%s: expected error, received nil", callerInfo())
	}
}

// SQL tests that the given Compiles instance matches the expected string for
// the current dialect.
func (t *sqlTest) SQL(expect string, stmt Compiles, ps ...interface{}) {
	// Get caller information in case of failure
	caller := callerInfo()

	// Start a new parameters instance
	params := Params()

	// Compile the given stmt with the tester's dialect
	actual, err := stmt.Compile(t.dialect, params)
	if err != nil {
		t.t.Errorf("%s: unexpected error from compile: %s", caller, err)
		return
	}

	if expect != actual {
		t.t.Errorf(
			"%s: unexpected SQL: expect %s, got %s",
			caller,
			expect,
			actual,
		)
	}
	// Test that the parameters are equal
	if params.Len() != len(ps) {
		t.t.Errorf(
			"%s: unexpected number of parameters for %s: expect %d, got %d",
			caller,
			actual,
			params.Len(),
			len(ps),
		)
		return
	}

	// TODO Examine individual parameters for equality
}

func NewTester(t *testing.T, d Dialect) *sqlTest {
	return &sqlTest{t: t, dialect: d}
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"runtime"
	"strconv"

	"github.com/AdRoll/goamz/aws"
	"github.com/AdRoll/goamz/ec2"
	"github.com/AdRoll/goamz/sns"
	"github.com/AdRoll/goamz/sqs"
	_ "github.com/lib/pq"
)

var SQS *sqs.SQS
var bufferCount = 100
var sem = make(chan bool, bufferCount)

var lambdaIP string

func getlambdaIP() (string, error) {
	resp, err := http.Get("https://api.ipify.org/")
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	contents, _ := ioutil.ReadAll(resp.Body)
	ip := string(contents)
	return ip, nil
}

func AddIPToGroup(p string, s string, secGroup string) error {
	auth := aws.Auth{AccessKey: p, SecretKey: s}
	region := aws.USEast
	ec2item := ec2.New(auth, region)

	g := ec2.SecurityGroup{Id: secGroup}
	ipperm := ec2.IPPerm{}
	ipperm.Protocol = "tcp"
	ipperm.FromPort = 5432
	ipperm.ToPort = 5432
	ipperm.SourceIPs = []string{fmt.Sprintf("%v/24", lambdaIP)}
	perms := []ec2.IPPerm{ipperm}
	_, errAdd := ec2item.AuthorizeSecurityGroup(g, perms)
	if errAdd != nil {
		log.Println("ERROR:", errAdd)
	} else {
		log.Println("Complete! Added! for:", lambdaIP)
	}
	return errAdd
}
func RemoveIPFromGroup(p string, s string, secGroup string) error {
	auth := aws.Auth{AccessKey: p, SecretKey: s}
	region := aws.USEast
	ec2item := ec2.New(auth, region)

	g := ec2.SecurityGroup{Id: secGroup}
	ipperm := ec2.IPPerm{}
	ipperm.Protocol = "tcp"
	ipperm.FromPort = 5432
	ipperm.ToPort = 5432
	ipperm.SourceIPs = []string{fmt.Sprintf("%v/24", lambdaIP)}
	perms := []ec2.IPPerm{ipperm}
	_, errRevoke := ec2item.RevokeSecurityGroup(g, perms)
	if errRevoke != nil {
		log.Println("ERROR:", errRevoke)
	} else {
		log.Println("Complete! Revoked! for:", lambdaIP)
	}
	return errRevoke
}

func main() {
	var errIP error
	if lambdaIP, errIP = getlambdaIP(); errIP != nil {
		log.Println(errIP)
		return
	}
	pub, sec, sg, _ := getSettings()
	AddIPToGroup(pub, sec, sg)
	bufferCount = GetBufferCountFromDB()
	log.Println("Buffer Count:", bufferCount)
	n := runtime.NumCPU()
	log.Println("Num CPUS:", n)
	runtime.GOMAXPROCS(n)
	sqsQ, err := getQueue("sns-prox", pub, sec)
	if err != nil {
		panic(err)
	}
	for k, v := range os.Args {
		log.Println(k, v)
	}
	log.Println("-----")
	if len(os.Args) > 1 {
		rawData := os.Args[1]

		var kpayload KinesisPayload
		err := json.Unmarshal([]byte(rawData), &kpayload)
		if err != nil {
			log.Println("Error Kinesisis Payload:", err)
		}
		for _, v := range kpayload.Records {
			log.Println("Record:", v)
			var tpm topicPageMessage
			sDec, errDec := base64.StdEncoding.DecodeString(v.Kinesis.Data)
			if errDec != nil {
				log.Println("Error:", errDec)
			} else {
				log.Println(string(sDec))
				errJSON := json.Unmarshal(sDec, &tpm)
				if errJSON != nil {
					log.Println("Error:", errJSON)
					return
				}
				msgstart := fmt.Sprintf("Started page %v\n", tpm.PageNum)
				WriteLogMessage(msgstart)
				arns := getDevicesArnsByTopicIDPage(tpm.TopicID, tpm.PageNum, 10000)
				//This should return the arn & the lang for the user
				//we'd then pull the correct iten out of the message map
				msgSlice := make([]sqs.Message, 0, 10)
				msgAll := [][]sqs.Message{}
				log.Printf(msgstart)
				for _, v := range arns {
					tempData := fmt.Sprintf("arn:%v|%v", v, tpm.Message)
					msg := sqs.Message{Body: base64.StdEncoding.EncodeToString([]byte(tempData))}
					msgSlice = append(msgSlice, msg)
					if len(msgSlice) == 10 {
						msgAll = append(msgAll, msgSlice)
						msgSlice = []sqs.Message{}
					}
				}
				for _, s := range msgAll {
					s := s //It's idomatic go I swear! http://golang.org/doc/effective_go.html#channels

					//Using the Semaphore
					sem <- true
					go func(sl10 []sqs.Message) {
						proxySNS(sqsQ, sl10)
						defer func() { <-sem }()
					}(s)

				}
				for i := 0; i < cap(sem); i++ {
					sem <- true
				}
				log.Println("All done!")
			}
			if tpm.PageNum == 1 {
				publishPageComplete(1)
			}
			if tpm.LastPage {
				publishPageComplete(tpm.PageNum)
			}
			msgcomplete := fmt.Sprintf("Completed page %v\n", tpm.PageNum)
			WriteLogMessage(msgcomplete)
			log.Printf(msgcomplete)

		}
		return

	}
	log.Println("Error: os.Args was 1 length.")
}
func publishPageComplete(pagenum int) error {
	topicarn, topicErr := getTopicArn()
	if topicErr != nil {
		log.Println(topicErr)
		return topicErr
	}
	p, s, _, _ := getSettings()
	auth := aws.Auth{AccessKey: p, SecretKey: s}
	region := aws.Region{}
	region.Name = "us-east-1"
	region.SNSEndpoint = "http://sns.us-east-1.amazonaws.com"
	awssns, _ := sns.New(auth, region)
	if awssns == nil {
		return fmt.Errorf("Can't get sns reference for %v %v", auth, region)
	}
	msg := fmt.Sprintf("Page %v complete.", pagenum)
	opt := sns.PublishOptions{}
	opt.TopicArn = topicarn
	opt.Message = msg
	opt.Subject = msg
	_, pubErr := awssns.Publish(&opt)
	if pubErr != nil {
		return pubErr
	}
	return nil
}

func getDevicesArnsByTopicIDPage(topicID, pagenum, pagesize int) []string {
	WriteLogMessage(fmt.Sprintf("DB Start: %v %v %v", topicID, pagenum, pagesize))
	var arns []string
	info := getDBSettings()
	db, errCon := sql.Open("postgres", fmt.Sprintf("host=%v user=%v password=%v dbname=%v sslmode=require", info.Host, info.Username, info.Password, info.Database))
	defer db.Close()
	if errCon != nil {
		log.Fatal(errCon)
	}
	rows, err := db.Query(`
		select
			u.endpointarn
		from
			subscription s ,userdevices u
		where
			s.topicid= $1 and
			s.userID=u.userid
			order by u.userid
			limit $2 offset $3
;`, topicID, pagesize, (pagenum-1)*pagesize)
	if err != nil {
		panic(err)
	}
	for rows.Next() {
		var arn string
		errScan := rows.Scan(&arn)
		if errScan != nil {
			panic(errScan)
		}
		arns = append(arns, arn)
	}
	WriteLogMessage(fmt.Sprintf("DB End: %v %v %v", topicID, pagenum, pagesize))

	return arns
}

func WriteLogMessage(msg string) {
	info := getDBSettings()
	db, errCon := sql.Open("postgres", fmt.Sprintf("host=%v user=%v password=%v dbname=%v sslmode=require", info.Host, info.Username, info.Password, info.Database))
	defer db.Close()
	if errCon != nil {
		log.Fatal(errCon)
	}
	_, err := db.Query(`
		insert into logtimes (message) values ($1);
	;`, msg)
	if err != nil {
		panic(err)
	}

}

func GetBufferCountFromDB() int {
	info := getDBSettings()
	db, errCon := sql.Open("postgres", fmt.Sprintf("host=%v user=%v password=%v dbname=%v sslmode=require", info.Host, info.Username, info.Password, info.Database))
	defer db.Close()
	if errCon != nil {
		log.Fatal(errCon)
	}
	rows, err := db.Query(`
		select
			s.value
		from
			lambdasettings s
		where
			s.name= 'buffercount'
	;`)
	if err != nil {
		panic(err)
	}
	for rows.Next() {
		var strval string
		errScan := rows.Scan(&strval)
		if errScan != nil {
			panic(errScan)
		}
		bc, cErr := strconv.Atoi(strval)
		if cErr != nil {
			panic(cErr)
		}
		return bc
	}
	return 100
}

type topicPageMessage struct {
	TopicID  int    `json:"topic_id"`
	Message  string `json:"message"`
	PageNum  int    `json:"page_num"`
	LastPage bool   `json:"last_page"`
}

type KinesisPayload struct {
	Records []struct {
		AwsRegion         string `json:"awsRegion"`
		EventID           string `json:"eventID"`
		EventName         string `json:"eventName"`
		EventSource       string `json:"eventSource"`
		EventSourceARN    string `json:"eventSourceARN"`
		EventVersion      string `json:"eventVersion"`
		InvokeIdentityArn string `json:"invokeIdentityArn"`
		Kinesis           struct {
			Data                 string `json:"data"`
			KinesisSchemaVersion string `json:"kinesisSchemaVersion"`
			PartitionKey         string `json:"partitionKey"`
			SequenceNumber       string `json:"sequenceNumber"`
		} `json:"kinesis"`
	} `json:"Records"`
}

func getTopicArn() (string, error) {
	file, err := ioutil.ReadFile("./settings.json")
	if err != nil {
		return "", err
	}
	settingsMap := make(map[string]string)
	json.Unmarshal(file, &settingsMap)
	return settingsMap["Topicarn"], nil
}

func getSettings() (string, string, string, error) {
	file, err := ioutil.ReadFile("./settings.json")
	if err != nil {
		return "", "", "", nil
	}
	settingsMap := make(map[string]string)
	json.Unmarshal(file, &settingsMap)
	return settingsMap["Access"], settingsMap["Secret"], settingsMap["SecGroup"], nil
}

func getDBSettings() *dbInfo {
	file, err := ioutil.ReadFile("./settings.json")
	if err != nil {
		log.Println("Error:", err)
		return nil
	}
	db := dbInfo{}
	err2 := json.Unmarshal(file, &db)
	if err2 != nil {
		log.Println("Error:", err2)
		return nil
	}
	return &db
}

type dbInfo struct {
	Host     string
	Database string
	Username string
	Password string
}

func proxySNS(q *sqs.Queue, msgs []sqs.Message) {

	_, respErr := q.SendMessageBatch(msgs)
	if respErr != nil {
		log.Println("ERROR:", respErr)
	}
}

func getQueue(name, public, secret string) (*sqs.Queue, error) {
	auth := aws.Auth{AccessKey: public, SecretKey: secret}
	region := aws.Region{}
	region.Name = "us-east-1"
	region.SQSEndpoint = "http://sqs.us-east-1.amazonaws.com"
	awssqs := sqs.New(auth, region)
	if awssqs == nil {
		return nil, fmt.Errorf("Can't get sqs reference for %v %v", auth, region)
	}
	return awssqs.GetQueue(name)
}
<commit_msg>might need to remove things<commit_after>package main

import (
	"database/sql"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"runtime"
	"strconv"

	"github.com/AdRoll/goamz/aws"
	"github.com/AdRoll/goamz/ec2"
	"github.com/AdRoll/goamz/sns"
	"github.com/AdRoll/goamz/sqs"
	_ "github.com/lib/pq"
)

var SQS *sqs.SQS
var bufferCount = 100
var sem = make(chan bool, bufferCount)

var lambdaIP string

func getlambdaIP() (string, error) {
	resp, err := http.Get("https://api.ipify.org/")
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	contents, _ := ioutil.ReadAll(resp.Body)
	ip := string(contents)
	return ip, nil
}

func AddIPToGroup(p string, s string, secGroup string) error {
	auth := aws.Auth{AccessKey: p, SecretKey: s}
	region := aws.USEast
	ec2item := ec2.New(auth, region)

	g := ec2.SecurityGroup{Id: secGroup}
	ipperm := ec2.IPPerm{}
	ipperm.Protocol = "tcp"
	ipperm.FromPort = 5432
	ipperm.ToPort = 5432
	ipperm.SourceIPs = []string{fmt.Sprintf("%v/24", lambdaIP)}
	perms := []ec2.IPPerm{ipperm}
	_, errAdd := ec2item.AuthorizeSecurityGroup(g, perms)
	if errAdd != nil {
		log.Println("ERROR:", errAdd)
	} else {
		log.Println("Complete! Added! for:", lambdaIP)
	}
	return errAdd
}
func RemoveIPFromGroup(p string, s string, secGroup string) error {
	auth := aws.Auth{AccessKey: p, SecretKey: s}
	region := aws.USEast
	ec2item := ec2.New(auth, region)

	g := ec2.SecurityGroup{Id: secGroup}
	ipperm := ec2.IPPerm{}
	ipperm.Protocol = "tcp"
	ipperm.FromPort = 5432
	ipperm.ToPort = 5432
	ipperm.SourceIPs = []string{fmt.Sprintf("%v/24", lambdaIP)}
	perms := []ec2.IPPerm{ipperm}
	_, errRevoke := ec2item.RevokeSecurityGroup(g, perms)
	if errRevoke != nil {
		log.Println("ERROR:", errRevoke)
	} else {
		log.Println("Complete! Revoked! for:", lambdaIP)
	}
	return errRevoke
}

func main() {
	var errIP error
	if lambdaIP, errIP = getlambdaIP(); errIP != nil {
		log.Println(errIP)
		return
	}
	pub, sec, sg, _ := getSettings()
	defer RemoveIPFromGroup(pub, sec, sg)
	AddIPToGroup(pub, sec, sg)
	bufferCount = GetBufferCountFromDB()
	log.Println("Buffer Count:", bufferCount)
	n := runtime.NumCPU()
	log.Println("Num CPUS:", n)
	runtime.GOMAXPROCS(n)
	sqsQ, err := getQueue("sns-prox", pub, sec)
	if err != nil {
		panic(err)
	}
	for k, v := range os.Args {
		log.Println(k, v)
	}
	log.Println("-----")
	if len(os.Args) > 1 {
		rawData := os.Args[1]

		var kpayload KinesisPayload
		err := json.Unmarshal([]byte(rawData), &kpayload)
		if err != nil {
			log.Println("Error Kinesisis Payload:", err)
		}
		for _, v := range kpayload.Records {
			log.Println("Record:", v)
			var tpm topicPageMessage
			sDec, errDec := base64.StdEncoding.DecodeString(v.Kinesis.Data)
			if errDec != nil {
				log.Println("Error:", errDec)
			} else {
				log.Println(string(sDec))
				errJSON := json.Unmarshal(sDec, &tpm)
				if errJSON != nil {
					log.Println("Error:", errJSON)
					return
				}
				msgstart := fmt.Sprintf("Started page %v\n", tpm.PageNum)
				WriteLogMessage(msgstart)
				arns := getDevicesArnsByTopicIDPage(tpm.TopicID, tpm.PageNum, 10000)
				//This should return the arn & the lang for the user
				//we'd then pull the correct iten out of the message map
				msgSlice := make([]sqs.Message, 0, 10)
				msgAll := [][]sqs.Message{}
				log.Printf(msgstart)
				for _, v := range arns {
					tempData := fmt.Sprintf("arn:%v|%v", v, tpm.Message)
					msg := sqs.Message{Body: base64.StdEncoding.EncodeToString([]byte(tempData))}
					msgSlice = append(msgSlice, msg)
					if len(msgSlice) == 10 {
						msgAll = append(msgAll, msgSlice)
						msgSlice = []sqs.Message{}
					}
				}
				for _, s := range msgAll {
					s := s //It's idomatic go I swear! http://golang.org/doc/effective_go.html#channels

					//Using the Semaphore
					sem <- true
					go func(sl10 []sqs.Message) {
						proxySNS(sqsQ, sl10)
						defer func() { <-sem }()
					}(s)

				}
				for i := 0; i < cap(sem); i++ {
					sem <- true
				}
				log.Println("All done!")
			}
			if tpm.PageNum == 1 {
				publishPageComplete(1)
			}
			if tpm.LastPage {
				publishPageComplete(tpm.PageNum)
			}
			msgcomplete := fmt.Sprintf("Completed page %v\n", tpm.PageNum)
			WriteLogMessage(msgcomplete)
			log.Printf(msgcomplete)

		}
		return

	}
	log.Println("Error: os.Args was 1 length.")
}
func publishPageComplete(pagenum int) error {
	topicarn, topicErr := getTopicArn()
	if topicErr != nil {
		log.Println(topicErr)
		return topicErr
	}
	p, s, _, _ := getSettings()
	auth := aws.Auth{AccessKey: p, SecretKey: s}
	region := aws.Region{}
	region.Name = "us-east-1"
	region.SNSEndpoint = "http://sns.us-east-1.amazonaws.com"
	awssns, _ := sns.New(auth, region)
	if awssns == nil {
		return fmt.Errorf("Can't get sns reference for %v %v", auth, region)
	}
	msg := fmt.Sprintf("Page %v complete.", pagenum)
	opt := sns.PublishOptions{}
	opt.TopicArn = topicarn
	opt.Message = msg
	opt.Subject = msg
	_, pubErr := awssns.Publish(&opt)
	if pubErr != nil {
		return pubErr
	}
	return nil
}

func getDevicesArnsByTopicIDPage(topicID, pagenum, pagesize int) []string {
	WriteLogMessage(fmt.Sprintf("DB Start: %v %v %v", topicID, pagenum, pagesize))
	var arns []string
	info := getDBSettings()
	db, errCon := sql.Open("postgres", fmt.Sprintf("host=%v user=%v password=%v dbname=%v sslmode=require", info.Host, info.Username, info.Password, info.Database))
	defer db.Close()
	if errCon != nil {
		log.Fatal(errCon)
	}
	rows, err := db.Query(`
		select
			u.endpointarn
		from
			subscription s ,userdevices u
		where
			s.topicid= $1 and
			s.userID=u.userid
			order by u.userid
			limit $2 offset $3
;`, topicID, pagesize, (pagenum-1)*pagesize)
	if err != nil {
		panic(err)
	}
	for rows.Next() {
		var arn string
		errScan := rows.Scan(&arn)
		if errScan != nil {
			panic(errScan)
		}
		arns = append(arns, arn)
	}
	WriteLogMessage(fmt.Sprintf("DB End: %v %v %v", topicID, pagenum, pagesize))

	return arns
}

func WriteLogMessage(msg string) {
	info := getDBSettings()
	db, errCon := sql.Open("postgres", fmt.Sprintf("host=%v user=%v password=%v dbname=%v sslmode=require", info.Host, info.Username, info.Password, info.Database))
	defer db.Close()
	if errCon != nil {
		log.Fatal(errCon)
	}
	_, err := db.Query(`
		insert into logtimes (message) values ($1);
	;`, msg)
	if err != nil {
		panic(err)
	}

}

func GetBufferCountFromDB() int {
	info := getDBSettings()
	db, errCon := sql.Open("postgres", fmt.Sprintf("host=%v user=%v password=%v dbname=%v sslmode=require", info.Host, info.Username, info.Password, info.Database))
	defer db.Close()
	if errCon != nil {
		log.Fatal(errCon)
	}
	rows, err := db.Query(`
		select
			s.value
		from
			lambdasettings s
		where
			s.name= 'buffercount'
	;`)
	if err != nil {
		panic(err)
	}
	for rows.Next() {
		var strval string
		errScan := rows.Scan(&strval)
		if errScan != nil {
			panic(errScan)
		}
		bc, cErr := strconv.Atoi(strval)
		if cErr != nil {
			panic(cErr)
		}
		return bc
	}
	return 100
}

type topicPageMessage struct {
	TopicID  int    `json:"topic_id"`
	Message  string `json:"message"`
	PageNum  int    `json:"page_num"`
	LastPage bool   `json:"last_page"`
}

type KinesisPayload struct {
	Records []struct {
		AwsRegion         string `json:"awsRegion"`
		EventID           string `json:"eventID"`
		EventName         string `json:"eventName"`
		EventSource       string `json:"eventSource"`
		EventSourceARN    string `json:"eventSourceARN"`
		EventVersion      string `json:"eventVersion"`
		InvokeIdentityArn string `json:"invokeIdentityArn"`
		Kinesis           struct {
			Data                 string `json:"data"`
			KinesisSchemaVersion string `json:"kinesisSchemaVersion"`
			PartitionKey         string `json:"partitionKey"`
			SequenceNumber       string `json:"sequenceNumber"`
		} `json:"kinesis"`
	} `json:"Records"`
}

func getTopicArn() (string, error) {
	file, err := ioutil.ReadFile("./settings.json")
	if err != nil {
		return "", err
	}
	settingsMap := make(map[string]string)
	json.Unmarshal(file, &settingsMap)
	return settingsMap["Topicarn"], nil
}

func getSettings() (string, string, string, error) {
	file, err := ioutil.ReadFile("./settings.json")
	if err != nil {
		return "", "", "", nil
	}
	settingsMap := make(map[string]string)
	json.Unmarshal(file, &settingsMap)
	return settingsMap["Access"], settingsMap["Secret"], settingsMap["SecGroup"], nil
}

func getDBSettings() *dbInfo {
	file, err := ioutil.ReadFile("./settings.json")
	if err != nil {
		log.Println("Error:", err)
		return nil
	}
	db := dbInfo{}
	err2 := json.Unmarshal(file, &db)
	if err2 != nil {
		log.Println("Error:", err2)
		return nil
	}
	return &db
}

type dbInfo struct {
	Host     string
	Database string
	Username string
	Password string
}

func proxySNS(q *sqs.Queue, msgs []sqs.Message) {

	_, respErr := q.SendMessageBatch(msgs)
	if respErr != nil {
		log.Println("ERROR:", respErr)
	}
}

func getQueue(name, public, secret string) (*sqs.Queue, error) {
	auth := aws.Auth{AccessKey: public, SecretKey: secret}
	region := aws.Region{}
	region.Name = "us-east-1"
	region.SQSEndpoint = "http://sqs.us-east-1.amazonaws.com"
	awssqs := sqs.New(auth, region)
	if awssqs == nil {
		return nil, fmt.Errorf("Can't get sqs reference for %v %v", auth, region)
	}
	return awssqs.GetQueue(name)
}
<|endoftext|>


<commit_before>// Package proxy is middleware that proxies HTTP requests.
package proxy

import (
	"errors"
	"net"
	"net/http"
	"net/url"
	"strings"
	"sync/atomic"
	"time"

	"github.com/mholt/caddy/caddyhttp/httpserver"
)

// Proxy represents a middleware instance that can proxy requests.
type Proxy struct {
	Next      httpserver.Handler
	Upstreams []Upstream
}

// Upstream manages a pool of proxy upstream hosts.
type Upstream interface {
	// The path this upstream host should be routed on
	From() string

	// Selects an upstream host to be routed to. It
	// should return a suitable upstream host, or nil
	// if no such hosts are available.
	Select(*http.Request) *UpstreamHost

	// Checks if subpath is not an ignored path
	AllowedPath(string) bool

	// Gets how long to try selecting upstream hosts
	// in the case of cascading failures.
	GetTryDuration() time.Duration

	// Gets how long to wait between selecting upstream
	// hosts in the case of cascading failures.
	GetTryInterval() time.Duration
}

// UpstreamHostDownFunc can be used to customize how Down behaves.
type UpstreamHostDownFunc func(*UpstreamHost) bool

// UpstreamHost represents a single proxy upstream
type UpstreamHost struct {
	Conns             int64  // must be first field to be 64-bit aligned on 32-bit systems
	Name              string // hostname of this upstream host
	ReverseProxy      *ReverseProxy
	Fails             int32
	FailTimeout       time.Duration
	Unhealthy         bool
	UpstreamHeaders   http.Header
	DownstreamHeaders http.Header
	CheckDown         UpstreamHostDownFunc
	WithoutPathPrefix string
	MaxConns          int64
}

// Down checks whether the upstream host is down or not.
// Down will try to use uh.CheckDown first, and will fall
// back to some default criteria if necessary.
func (uh *UpstreamHost) Down() bool {
	if uh.CheckDown == nil {
		// Default settings
		return uh.Unhealthy || uh.Fails > 0
	}
	return uh.CheckDown(uh)
}

// Full checks whether the upstream host has reached its maximum connections
func (uh *UpstreamHost) Full() bool {
	return uh.MaxConns > 0 && uh.Conns >= uh.MaxConns
}

// Available checks whether the upstream host is available for proxying to
func (uh *UpstreamHost) Available() bool {
	return !uh.Down() && !uh.Full()
}

// tryDuration is how long to try upstream hosts; failures result in
// immediate retries until this duration ends or we get a nil host.
var tryDuration = 60 * time.Second

// ServeHTTP satisfies the httpserver.Handler interface.
func (p Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) {
	// start by selecting most specific matching upstream config
	upstream := p.match(r)
	if upstream == nil {
		return p.Next.ServeHTTP(w, r)
	}

	// this replacer is used to fill in header field values
	replacer := httpserver.NewReplacer(r, nil, "")

	// outreq is the request that makes a roundtrip to the backend
	outreq := createUpstreamRequest(r)

	// since Select() should give us "up" hosts, keep retrying
	// hosts until timeout (or until we get a nil host).
	start := time.Now()
	var backendErr error
	for {
		host := upstream.Select(r)
		if host == nil {
			if backendErr == nil {
				backendErr = errors.New("no hosts available upstream")
			}
			return http.StatusBadGateway, backendErr
		}
		if rr, ok := w.(*httpserver.ResponseRecorder); ok && rr.Replacer != nil {
			rr.Replacer.Set("upstream", host.Name)
		}

		proxy := host.ReverseProxy

		// a backend's name may contain more than just the host,
		// so we parse it as a URL to try to isolate the host.
		if nameURL, err := url.Parse(host.Name); err == nil {
			outreq.Host = nameURL.Host
			if proxy == nil {
				proxy = NewSingleHostReverseProxy(nameURL, host.WithoutPathPrefix, http.DefaultMaxIdleConnsPerHost)
			}

			// use upstream credentials by default
			if outreq.Header.Get("Authorization") == "" && nameURL.User != nil {
				pwd, _ := nameURL.User.Password()
				outreq.SetBasicAuth(nameURL.User.Username(), pwd)
			}
		} else {
			outreq.Host = host.Name
		}
		if proxy == nil {
			return http.StatusInternalServerError, errors.New("proxy for host '" + host.Name + "' is nil")
		}

		// set headers for request going upstream
		if host.UpstreamHeaders != nil {
			// modify headers for request that will be sent to the upstream host
			mutateHeadersByRules(outreq.Header, host.UpstreamHeaders, replacer)
			if hostHeaders, ok := outreq.Header["Host"]; ok && len(hostHeaders) > 0 {
				outreq.Host = hostHeaders[len(hostHeaders)-1]
			}
		}

		// prepare a function that will update response
		// headers coming back downstream
		var downHeaderUpdateFn respUpdateFn
		if host.DownstreamHeaders != nil {
			downHeaderUpdateFn = createRespHeaderUpdateFn(host.DownstreamHeaders, replacer)
		}

		// tell the proxy to serve the request
		atomic.AddInt64(&host.Conns, 1)
		backendErr = proxy.ServeHTTP(w, outreq, downHeaderUpdateFn)
		atomic.AddInt64(&host.Conns, -1)

		// if no errors, we're done here
		if backendErr == nil {
			return 0, nil
		}

		// failover; remember this failure for some time if
		// request failure counting is enabled
		timeout := host.FailTimeout
		if timeout > 0 {
			atomic.AddInt32(&host.Fails, 1)
			go func(host *UpstreamHost, timeout time.Duration) {
				time.Sleep(timeout)
				atomic.AddInt32(&host.Fails, -1)
			}(host, timeout)
		}

		// if we've tried long enough, break
		if time.Now().Sub(start) >= upstream.GetTryDuration() {
			break
		}

		// otherwise, wait and try the next available host
		time.Sleep(upstream.GetTryInterval())
	}

	return http.StatusBadGateway, backendErr
}

// match finds the best match for a proxy config based on r.
func (p Proxy) match(r *http.Request) Upstream {
	var u Upstream
	var longestMatch int
	for _, upstream := range p.Upstreams {
		basePath := upstream.From()
		if !httpserver.Path(r.URL.Path).Matches(basePath) || !upstream.AllowedPath(r.URL.Path) {
			continue
		}
		if len(basePath) > longestMatch {
			longestMatch = len(basePath)
			u = upstream
		}
	}
	return u
}

// createUpstremRequest shallow-copies r into a new request
// that can be sent upstream.
//
// Derived from reverseproxy.go in the standard Go httputil package.
func createUpstreamRequest(r *http.Request) *http.Request {
	outreq := new(http.Request)
	*outreq = *r // includes shallow copies of maps, but okay

	// Restore URL Path if it has been modified
	if outreq.URL.RawPath != "" {
		outreq.URL.Opaque = outreq.URL.RawPath
	}

	// Remove hop-by-hop headers to the backend. Especially
	// important is "Connection" because we want a persistent
	// connection, regardless of what the client sent to us. This
	// is modifying the same underlying map from r (shallow
	// copied above) so we only copy it if necessary.
	var copiedHeaders bool
	for _, h := range hopHeaders {
		if outreq.Header.Get(h) != "" {
			if !copiedHeaders {
				outreq.Header = make(http.Header)
				copyHeader(outreq.Header, r.Header)
				copiedHeaders = true
			}
			outreq.Header.Del(h)
		}
	}

	if clientIP, _, err := net.SplitHostPort(r.RemoteAddr); err == nil {
		// If we aren't the first proxy, retain prior
		// X-Forwarded-For information as a comma+space
		// separated list and fold multiple headers into one.
		if prior, ok := outreq.Header["X-Forwarded-For"]; ok {
			clientIP = strings.Join(prior, ", ") + ", " + clientIP
		}
		outreq.Header.Set("X-Forwarded-For", clientIP)
	}

	return outreq
}

func createRespHeaderUpdateFn(rules http.Header, replacer httpserver.Replacer) respUpdateFn {
	return func(resp *http.Response) {
		mutateHeadersByRules(resp.Header, rules, replacer)
	}
}

func mutateHeadersByRules(headers, rules http.Header, repl httpserver.Replacer) {
	for ruleField, ruleValues := range rules {
		if strings.HasPrefix(ruleField, "+") {
			for _, ruleValue := range ruleValues {
				headers.Add(strings.TrimPrefix(ruleField, "+"), repl.Replace(ruleValue))
			}
		} else if strings.HasPrefix(ruleField, "-") {
			headers.Del(strings.TrimPrefix(ruleField, "-"))
		} else if len(ruleValues) > 0 {
			headers.Set(ruleField, repl.Replace(ruleValues[len(ruleValues)-1]))
		}
	}
}
<commit_msg>Delete tryDuration, now unused<commit_after>// Package proxy is middleware that proxies HTTP requests.
package proxy

import (
	"errors"
	"net"
	"net/http"
	"net/url"
	"strings"
	"sync/atomic"
	"time"

	"github.com/mholt/caddy/caddyhttp/httpserver"
)

// Proxy represents a middleware instance that can proxy requests.
type Proxy struct {
	Next      httpserver.Handler
	Upstreams []Upstream
}

// Upstream manages a pool of proxy upstream hosts.
type Upstream interface {
	// The path this upstream host should be routed on
	From() string

	// Selects an upstream host to be routed to. It
	// should return a suitable upstream host, or nil
	// if no such hosts are available.
	Select(*http.Request) *UpstreamHost

	// Checks if subpath is not an ignored path
	AllowedPath(string) bool

	// Gets how long to try selecting upstream hosts
	// in the case of cascading failures.
	GetTryDuration() time.Duration

	// Gets how long to wait between selecting upstream
	// hosts in the case of cascading failures.
	GetTryInterval() time.Duration
}

// UpstreamHostDownFunc can be used to customize how Down behaves.
type UpstreamHostDownFunc func(*UpstreamHost) bool

// UpstreamHost represents a single proxy upstream
type UpstreamHost struct {
	Conns             int64  // must be first field to be 64-bit aligned on 32-bit systems
	Name              string // hostname of this upstream host
	ReverseProxy      *ReverseProxy
	Fails             int32
	FailTimeout       time.Duration
	Unhealthy         bool
	UpstreamHeaders   http.Header
	DownstreamHeaders http.Header
	CheckDown         UpstreamHostDownFunc
	WithoutPathPrefix string
	MaxConns          int64
}

// Down checks whether the upstream host is down or not.
// Down will try to use uh.CheckDown first, and will fall
// back to some default criteria if necessary.
func (uh *UpstreamHost) Down() bool {
	if uh.CheckDown == nil {
		// Default settings
		return uh.Unhealthy || uh.Fails > 0
	}
	return uh.CheckDown(uh)
}

// Full checks whether the upstream host has reached its maximum connections
func (uh *UpstreamHost) Full() bool {
	return uh.MaxConns > 0 && uh.Conns >= uh.MaxConns
}

// Available checks whether the upstream host is available for proxying to
func (uh *UpstreamHost) Available() bool {
	return !uh.Down() && !uh.Full()
}

// ServeHTTP satisfies the httpserver.Handler interface.
func (p Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) {
	// start by selecting most specific matching upstream config
	upstream := p.match(r)
	if upstream == nil {
		return p.Next.ServeHTTP(w, r)
	}

	// this replacer is used to fill in header field values
	replacer := httpserver.NewReplacer(r, nil, "")

	// outreq is the request that makes a roundtrip to the backend
	outreq := createUpstreamRequest(r)

	// since Select() should give us "up" hosts, keep retrying
	// hosts until timeout (or until we get a nil host).
	start := time.Now()
	var backendErr error
	for {
		host := upstream.Select(r)
		if host == nil {
			if backendErr == nil {
				backendErr = errors.New("no hosts available upstream")
			}
			return http.StatusBadGateway, backendErr
		}
		if rr, ok := w.(*httpserver.ResponseRecorder); ok && rr.Replacer != nil {
			rr.Replacer.Set("upstream", host.Name)
		}

		proxy := host.ReverseProxy

		// a backend's name may contain more than just the host,
		// so we parse it as a URL to try to isolate the host.
		if nameURL, err := url.Parse(host.Name); err == nil {
			outreq.Host = nameURL.Host
			if proxy == nil {
				proxy = NewSingleHostReverseProxy(nameURL, host.WithoutPathPrefix, http.DefaultMaxIdleConnsPerHost)
			}

			// use upstream credentials by default
			if outreq.Header.Get("Authorization") == "" && nameURL.User != nil {
				pwd, _ := nameURL.User.Password()
				outreq.SetBasicAuth(nameURL.User.Username(), pwd)
			}
		} else {
			outreq.Host = host.Name
		}
		if proxy == nil {
			return http.StatusInternalServerError, errors.New("proxy for host '" + host.Name + "' is nil")
		}

		// set headers for request going upstream
		if host.UpstreamHeaders != nil {
			// modify headers for request that will be sent to the upstream host
			mutateHeadersByRules(outreq.Header, host.UpstreamHeaders, replacer)
			if hostHeaders, ok := outreq.Header["Host"]; ok && len(hostHeaders) > 0 {
				outreq.Host = hostHeaders[len(hostHeaders)-1]
			}
		}

		// prepare a function that will update response
		// headers coming back downstream
		var downHeaderUpdateFn respUpdateFn
		if host.DownstreamHeaders != nil {
			downHeaderUpdateFn = createRespHeaderUpdateFn(host.DownstreamHeaders, replacer)
		}

		// tell the proxy to serve the request
		atomic.AddInt64(&host.Conns, 1)
		backendErr = proxy.ServeHTTP(w, outreq, downHeaderUpdateFn)
		atomic.AddInt64(&host.Conns, -1)

		// if no errors, we're done here
		if backendErr == nil {
			return 0, nil
		}

		// failover; remember this failure for some time if
		// request failure counting is enabled
		timeout := host.FailTimeout
		if timeout > 0 {
			atomic.AddInt32(&host.Fails, 1)
			go func(host *UpstreamHost, timeout time.Duration) {
				time.Sleep(timeout)
				atomic.AddInt32(&host.Fails, -1)
			}(host, timeout)
		}

		// if we've tried long enough, break
		if time.Now().Sub(start) >= upstream.GetTryDuration() {
			break
		}

		// otherwise, wait and try the next available host
		time.Sleep(upstream.GetTryInterval())
	}

	return http.StatusBadGateway, backendErr
}

// match finds the best match for a proxy config based on r.
func (p Proxy) match(r *http.Request) Upstream {
	var u Upstream
	var longestMatch int
	for _, upstream := range p.Upstreams {
		basePath := upstream.From()
		if !httpserver.Path(r.URL.Path).Matches(basePath) || !upstream.AllowedPath(r.URL.Path) {
			continue
		}
		if len(basePath) > longestMatch {
			longestMatch = len(basePath)
			u = upstream
		}
	}
	return u
}

// createUpstremRequest shallow-copies r into a new request
// that can be sent upstream.
//
// Derived from reverseproxy.go in the standard Go httputil package.
func createUpstreamRequest(r *http.Request) *http.Request {
	outreq := new(http.Request)
	*outreq = *r // includes shallow copies of maps, but okay

	// Restore URL Path if it has been modified
	if outreq.URL.RawPath != "" {
		outreq.URL.Opaque = outreq.URL.RawPath
	}

	// Remove hop-by-hop headers to the backend. Especially
	// important is "Connection" because we want a persistent
	// connection, regardless of what the client sent to us. This
	// is modifying the same underlying map from r (shallow
	// copied above) so we only copy it if necessary.
	var copiedHeaders bool
	for _, h := range hopHeaders {
		if outreq.Header.Get(h) != "" {
			if !copiedHeaders {
				outreq.Header = make(http.Header)
				copyHeader(outreq.Header, r.Header)
				copiedHeaders = true
			}
			outreq.Header.Del(h)
		}
	}

	if clientIP, _, err := net.SplitHostPort(r.RemoteAddr); err == nil {
		// If we aren't the first proxy, retain prior
		// X-Forwarded-For information as a comma+space
		// separated list and fold multiple headers into one.
		if prior, ok := outreq.Header["X-Forwarded-For"]; ok {
			clientIP = strings.Join(prior, ", ") + ", " + clientIP
		}
		outreq.Header.Set("X-Forwarded-For", clientIP)
	}

	return outreq
}

func createRespHeaderUpdateFn(rules http.Header, replacer httpserver.Replacer) respUpdateFn {
	return func(resp *http.Response) {
		mutateHeadersByRules(resp.Header, rules, replacer)
	}
}

func mutateHeadersByRules(headers, rules http.Header, repl httpserver.Replacer) {
	for ruleField, ruleValues := range rules {
		if strings.HasPrefix(ruleField, "+") {
			for _, ruleValue := range ruleValues {
				headers.Add(strings.TrimPrefix(ruleField, "+"), repl.Replace(ruleValue))
			}
		} else if strings.HasPrefix(ruleField, "-") {
			headers.Del(strings.TrimPrefix(ruleField, "-"))
		} else if len(ruleValues) > 0 {
			headers.Set(ruleField, repl.Replace(ruleValues[len(ruleValues)-1]))
		}
	}
}
<|endoftext|>


<commit_before>package tls

import (
	"crypto/tls"
	"crypto/x509"

	"github.com/spiffe/go-spiffe/spiffe"
)

type TLSPeer struct {
	// Slice of permitted SPIFFE IDs
	SpiffeIDs []string

	TrustRoots *x509.CertPool
}

// NewTLSConfig creates a SPIFFE-compatible TLS configuration.
// We are opinionated towards mutual TLS. If you don't want
// mutual TLS, you'll need to update the returned config.
//
// `certs` contains one or more certificates to present to the
// other side of the connection, leaf first.
func (t *TLSPeer) NewTLSConfig(certs []tls.Certificate) *tls.Config {
	config := &tls.Config{
		// Disable validation/verification because we perform
		// this step with custom logic in `verifyPeerCertificate`
		ClientAuth:            tls.RequireAnyClientCert,
		InsecureSkipVerify:    true,
		VerifyPeerCertificate: t.verifyPeerCertificate,
		Certificates:          certs,
	}

	return config
}

// verifyPeerCertificate serves callbacks from TLS listeners/dialers. It performs
// SPIFFE-specific validation steps on behalf of the golang TLS library
func (t *TLSPeer) verifyPeerCertificate(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) (err error) {
	// First, parse all received certs
	var certs []*x509.Certificate
	for _, rawCert := range rawCerts {
		cert, err := x509.ParseCertificate(rawCert)
		if err != nil {
			return err
		}

		certs = append(certs, cert)
	}

	// Perform path validation
	// Leaf is the first off the wire:
	// https://tools.ietf.org/html/rfc5246#section-7.4.2
	intermediates := x509.NewCertPool()
	for _, intermediate := range certs[1:] {
		intermediates.AddCert(intermediate)
	}
	err = spiffe.VerifyCertificate(certs[0], intermediates, t.TrustRoots)
	if err != nil {
		return err
	}

	// Look for a known SPIFFE ID in the leaf
	err = spiffe.MatchID(t.SpiffeIDs, certs[0])
	if err != nil {
		return err
	}

	// If we are here, then all is well
	return nil
}
<commit_msg>gofmt<commit_after>package tls

import (
	"crypto/tls"
	"crypto/x509"

	"github.com/spiffe/go-spiffe/spiffe"
)

type TLSPeer struct {
	// Slice of permitted SPIFFE IDs
	SpiffeIDs []string

	TrustRoots *x509.CertPool
}

// NewTLSConfig creates a SPIFFE-compatible TLS configuration.
// We are opinionated towards mutual TLS. If you don't want
// mutual TLS, you'll need to update the returned config.
//
// `certs` contains one or more certificates to present to the
// other side of the connection, leaf first.
func (t *TLSPeer) NewTLSConfig(certs []tls.Certificate) *tls.Config {
	config := &tls.Config{
		// Disable validation/verification because we perform
		// this step with custom logic in `verifyPeerCertificate`
		ClientAuth:            tls.RequireAnyClientCert,
		InsecureSkipVerify:    true,
		VerifyPeerCertificate: t.verifyPeerCertificate,
		Certificates:          certs,
	}

	return config
}

// verifyPeerCertificate serves callbacks from TLS listeners/dialers. It performs
// SPIFFE-specific validation steps on behalf of the golang TLS library
func (t *TLSPeer) verifyPeerCertificate(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) (err error) {
	// First, parse all received certs
	var certs []*x509.Certificate
	for _, rawCert := range rawCerts {
		cert, err := x509.ParseCertificate(rawCert)
		if err != nil {
			return err
		}

		certs = append(certs, cert)
	}

	// Perform path validation
	// Leaf is the first off the wire:
	// https://tools.ietf.org/html/rfc5246#section-7.4.2
	intermediates := x509.NewCertPool()
	for _, intermediate := range certs[1:] {
		intermediates.AddCert(intermediate)
	}
	err = spiffe.VerifyCertificate(certs[0], intermediates, t.TrustRoots)
	if err != nil {
		return err
	}

	// Look for a known SPIFFE ID in the leaf
	err = spiffe.MatchID(t.SpiffeIDs, certs[0])
	if err != nil {
		return err
	}

	// If we are here, then all is well
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package types declares the data types and implements
// the algorithms for type-checking of Go packages.
// Use Check and Config.Check to invoke the type-checker.
//
// Type-checking consists of several interdependent phases:
//
// Name resolution maps each identifier (ast.Ident) in the program to the
// language object (Object) it denotes.
// Use Info.Objects, Info.Implicits for the results of name resolution.
//
// Constant folding computes the exact constant value (exact.Value) for
// every expression (ast.Expr) that is a compile-time constant.
// Use Info.Values for the results of constant folding.
//
// Type inference computes the type (Type) of every expression (ast.Expr)
// and checks for compliance with the language specification.
// Use Info.Types for the results of type evaluation.
//
package types

import (
	"go/ast"
	"go/token"

	"code.google.com/p/go.tools/go/exact"
)

// Check type-checks a package and returns the resulting complete package
// object, or a nil package and the first error. The package is specified
// by a list of *ast.Files and corresponding file set, and the import path
// the package is identified with. The clean path must not be empty or dot (".").
//
// For more control over type-checking and results, use Config.Check.
func Check(path string, fset *token.FileSet, files []*ast.File) (*Package, error) {
	var conf Config
	pkg, err := conf.Check(path, fset, files, nil)
	if err != nil {
		return nil, err
	}
	return pkg, nil
}

// A Config specifies the configuration for type checking.
// The zero value for Config is a ready-to-use default configuration.
type Config struct {
	// If IgnoreFuncBodies is set, function bodies are not
	// type-checked.
	IgnoreFuncBodies bool

	// If FakeImportC is set, `import "C"` (for packages requiring Cgo)
	// and expressions with qualified identifiers referring to package
	// C are silently ignored.
	// Caution: Effects may be unpredictable - do not use casually!
	FakeImportC bool

	// If Error != nil, it is called with each error found
	// during type checking. The error strings of errors with
	// detailed position information are formatted as follows:
	// filename:line:column: message
	Error func(err error)

	// If Import != nil, it is called for each imported package.
	// Otherwise, GcImporter is called.
	// An importer resolves import paths to Package objects.
	// The imports map records the packages already imported,
	// indexed by package id (canonical import path).
	// An importer must determine the canonical import path and
	// check the map to see if it is already present in the imports map.
	// If so, the Importer can return the map entry.  Otherwise, the
	// importer should load the package data for the given path into
	// a new *Package, record pkg in the imports map, and then
	// return pkg.
	Import func(imports map[string]*Package, path string) (pkg *Package, err error)

	// If Alignof != nil, it is called to determine the alignment
	// of the given type. Otherwise DefaultAlignmentof is called.
	// Alignof must implement the alignment guarantees required by
	// the spec.
	Alignof func(Type) int64

	// If Offsetsof != nil, it is called to determine the offsets
	// of the given struct fields, in bytes. Otherwise DefaultOffsetsof
	// is called. Offsetsof must implement the offset guarantees
	// required by the spec.
	Offsetsof func(fields []*Var) []int64

	// If Sizeof != nil, it is called to determine the size of the
	// given type. Otherwise, DefaultSizeof is called. Sizeof must
	// implement the size guarantees required by the spec.
	Sizeof func(Type) int64
}

// Info holds result type information for a type-checked package.
// Only the information for which a map is provided is collected.
// If the package has type errors, the collected information may
// be incomplete.
type Info struct {
	// Types maps expressions to their types. Identifiers on the
	// lhs of declarations are collected in Objects, not Types.
	Types map[ast.Expr]Type

	// Values maps constant expressions to their values.
	Values map[ast.Expr]exact.Value

	// Objects maps identifiers to their corresponding objects (including
	// package names, dots "." of dot-imports, and blank "_" identifiers).
	// For identifiers that do not denote objects (e.g., blank identifiers
	// on the lhs of assignments, or symbolic variables t in t := x.(type)
	// of type switch headers), the corresponding objects are nil.
	// BUG(gri) Label identifiers in break, continue, or goto statements
	// are not yet mapped.
	Objects map[*ast.Ident]Object

	// Implicits maps nodes to their implicitly declared objects, if any.
	// The following node and object types may appear:
	//
	//	node               declared object
	//
	//	*ast.ImportSpec    *Package (imports w/o renames), or imported objects (dot-imports)
	//	*ast.CaseClause    type-specific *Var for each type switch case clause (incl. default)
	//      *ast.Field         anonymous struct field or parameter *Var
	//
	Implicits map[ast.Node]Object

	// Selections maps selector expressions to their corresponding selections.
	Selections map[*ast.SelectorExpr]*Selection

	// Scopes maps ast.Nodes to the scopes they define. Note that package scopes
	// are not associated with a specific node but with all files belonging to a
	// package. Thus, the package scope can be found in the type-checked package
	// object.
	//
	// The following node types may appear in Scopes:
	//
	//	*ast.File
	//	*ast.FuncType
	//	*ast.BlockStmt
	//	*ast.IfStmt
	//	*ast.SwitchStmt
	//	*ast.TypeSwitchStmt
	//	*ast.CaseClause
	//	*ast.CommClause
	//	*ast.ForStmt
	//	*ast.RangeStmt
	//
	Scopes map[ast.Node]*Scope
}

// Check type-checks a package and returns the resulting package object,
// the first error if any, and if info != nil, additional type information.
// The package is marked as complete if no errors occurred, otherwise it is
// incomplete.
//
// The package is specified by a list of *ast.Files and corresponding file
// set, and the import path the package is identified with. The clean path
// must not be empty or dot (".").
func (conf *Config) Check(path string, fset *token.FileSet, files []*ast.File, info *Info) (*Package, error) {
	pkg, err := conf.check(path, fset, files, info)
	if err == nil {
		pkg.complete = true
	}
	return pkg, err
}

// IsAssignableTo reports whether a value of type V
// is assignable to a variable of type T.
func IsAssignableTo(V, T Type) bool {
	x := operand{mode: value, typ: V}
	return x.isAssignableTo(nil, T) // config not needed for non-constant x
}

// BUG(gri): Some built-ins don't check parameters fully, yet (e.g. append).
// BUG(gri): Use of labels is only partially checked.
// BUG(gri): Unused variables and imports are not reported.
// BUG(gri): Interface vs non-interface comparisons are not correctly implemented.
// BUG(gri): Switch statements don't check duplicate cases for all types for which it is required.
// BUG(gri): Some built-ins may not be callable if in statement-context.
<commit_msg>go.tools/go/types: clearer documentation<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package types declares the data types and implements
// the algorithms for type-checking of Go packages.
// Use Check and Config.Check to invoke the type-checker.
//
// Type-checking consists of several interdependent phases:
//
// Name resolution maps each identifier (ast.Ident) in the program to the
// language object (Object) it denotes.
// Use Info.Objects, Info.Implicits for the results of name resolution.
//
// Constant folding computes the exact constant value (exact.Value) for
// every expression (ast.Expr) that is a compile-time constant.
// Use Info.Values for the results of constant folding.
//
// Type inference computes the type (Type) of every expression (ast.Expr)
// and checks for compliance with the language specification.
// Use Info.Types for the results of type evaluation.
//
package types

import (
	"go/ast"
	"go/token"

	"code.google.com/p/go.tools/go/exact"
)

// Check type-checks a package and returns the resulting complete package
// object, or a nil package and the first error. The package is specified
// by a list of *ast.Files and corresponding file set, and the import path
// the package is identified with. The clean path must not be empty or dot (".").
//
// For more control over type-checking and results, use Config.Check.
func Check(path string, fset *token.FileSet, files []*ast.File) (*Package, error) {
	var conf Config
	pkg, err := conf.Check(path, fset, files, nil)
	if err != nil {
		return nil, err
	}
	return pkg, nil
}

// A Config specifies the configuration for type checking.
// The zero value for Config is a ready-to-use default configuration.
type Config struct {
	// If IgnoreFuncBodies is set, function bodies are not
	// type-checked.
	IgnoreFuncBodies bool

	// If FakeImportC is set, `import "C"` (for packages requiring Cgo)
	// declares an empty "C" package and errors are omitted for qualified
	// identifiers referring to package C (which won't find an object).
	// Caution: Effects may be unpredictable due to unpredictable follow-
	// up errors - do not use casually! This feature is mainly intended
	// for the standard library cmd/api tool.
	FakeImportC bool

	// If Error != nil, it is called with each error found
	// during type checking. The error strings of errors with
	// detailed position information are formatted as follows:
	// filename:line:column: message
	Error func(err error)

	// If Import != nil, it is called for each imported package.
	// Otherwise, GcImporter is called.
	// An importer resolves import paths to Package objects.
	// The imports map records the packages already imported,
	// indexed by package id (canonical import path).
	// An importer must determine the canonical import path and
	// check the map to see if it is already present in the imports map.
	// If so, the Importer can return the map entry.  Otherwise, the
	// importer should load the package data for the given path into
	// a new *Package, record pkg in the imports map, and then
	// return pkg.
	Import func(imports map[string]*Package, path string) (pkg *Package, err error)

	// If Alignof != nil, it is called to determine the alignment
	// of the given type. Otherwise DefaultAlignmentof is called.
	// Alignof must implement the alignment guarantees required by
	// the spec.
	Alignof func(Type) int64

	// If Offsetsof != nil, it is called to determine the offsets
	// of the given struct fields, in bytes. Otherwise DefaultOffsetsof
	// is called. Offsetsof must implement the offset guarantees
	// required by the spec.
	Offsetsof func(fields []*Var) []int64

	// If Sizeof != nil, it is called to determine the size of the
	// given type. Otherwise, DefaultSizeof is called. Sizeof must
	// implement the size guarantees required by the spec.
	Sizeof func(Type) int64
}

// Info holds result type information for a type-checked package.
// Only the information for which a map is provided is collected.
// If the package has type errors, the collected information may
// be incomplete.
type Info struct {
	// Types maps expressions to their types. Identifiers on the
	// lhs of declarations are collected in Objects, not Types.
	Types map[ast.Expr]Type

	// Values maps constant expressions to their values.
	Values map[ast.Expr]exact.Value

	// Objects maps identifiers to their corresponding objects (including
	// package names, dots "." of dot-imports, and blank "_" identifiers).
	// For identifiers that do not denote objects (e.g., blank identifiers
	// on the lhs of assignments, or symbolic variables t in t := x.(type)
	// of type switch headers), the corresponding objects are nil.
	// BUG(gri) Label identifiers in break, continue, or goto statements
	// are not yet mapped.
	Objects map[*ast.Ident]Object

	// Implicits maps nodes to their implicitly declared objects, if any.
	// The following node and object types may appear:
	//
	//	node               declared object
	//
	//	*ast.ImportSpec    *Package (imports w/o renames), or imported objects (dot-imports)
	//	*ast.CaseClause    type-specific *Var for each type switch case clause (incl. default)
	//      *ast.Field         anonymous struct field or parameter *Var
	//
	Implicits map[ast.Node]Object

	// Selections maps selector expressions to their corresponding selections.
	Selections map[*ast.SelectorExpr]*Selection

	// Scopes maps ast.Nodes to the scopes they define. Note that package scopes
	// are not associated with a specific node but with all files belonging to a
	// package. Thus, the package scope can be found in the type-checked package
	// object.
	//
	// The following node types may appear in Scopes:
	//
	//	*ast.File
	//	*ast.FuncType
	//	*ast.BlockStmt
	//	*ast.IfStmt
	//	*ast.SwitchStmt
	//	*ast.TypeSwitchStmt
	//	*ast.CaseClause
	//	*ast.CommClause
	//	*ast.ForStmt
	//	*ast.RangeStmt
	//
	Scopes map[ast.Node]*Scope
}

// Check type-checks a package and returns the resulting package object,
// the first error if any, and if info != nil, additional type information.
// The package is marked as complete if no errors occurred, otherwise it is
// incomplete.
//
// The package is specified by a list of *ast.Files and corresponding file
// set, and the import path the package is identified with. The clean path
// must not be empty or dot (".").
func (conf *Config) Check(path string, fset *token.FileSet, files []*ast.File, info *Info) (*Package, error) {
	pkg, err := conf.check(path, fset, files, info)
	if err == nil {
		pkg.complete = true
	}
	return pkg, err
}

// IsAssignableTo reports whether a value of type V
// is assignable to a variable of type T.
func IsAssignableTo(V, T Type) bool {
	x := operand{mode: value, typ: V}
	return x.isAssignableTo(nil, T) // config not needed for non-constant x
}

// BUG(gri): Some built-ins don't check parameters fully, yet (e.g. append).
// BUG(gri): Use of labels is only partially checked.
// BUG(gri): Unused variables and imports are not reported.
// BUG(gri): Interface vs non-interface comparisons are not correctly implemented.
// BUG(gri): Switch statements don't check duplicate cases for all types for which it is required.
// BUG(gri): Some built-ins may not be callable if in statement-context.
<|endoftext|>


<commit_before>// Needs correcting on timing code
// Docs here: https://golang.org/pkg/time/#Time.UnixNano
// Testpad here: http://tour.golang.org/basics/13

package main

import (
	"fmt"
	"time"
)

func main() {
	var i int
	var x float64
	num_steps := 1000000000
	fmt.Printf("Calculating PI using:\n  %v slices\n  1 process\n", num_steps)

	var sum float64 = 0
	step := 1.0 / float64(num_steps)

	var start time.Time = time.Now()

	for i = 0; i < num_steps; i++ {
		x = (float64(i) + 0.5) * step
		sum += 4.0 / (1.0 + x*x)
	}

	var stop time.Time = time.Now()

	var pi = sum * step

	timeTaken := (float64(stop.Sub(start)) * 0.000000001) // Durations come in ns

	fmt.Printf("Obtained value for PI: %.16g\n Time taken: %v s\n", pi, timeTaken)
}

//  const f = "%T(%v)\n"
<commit_msg>Adds command-line argument<commit_after>// Needs correcting on timing code
// Docs here: https://golang.org/pkg/time/#Time.UnixNano
// Testpad here: http://tour.golang.org/basics/13

package main

import (
	"fmt"
	"os"
	"strconv"
	"time"
)

func main() {
	var i int
	var x float64

	var num_steps int
	var err error

	if len(os.Args) > 1 {
		fmt.Println("beep")
		num_steps, err = strconv.Atoi(os.Args[1])
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error parsing argument as step count: %s\n", err)
			os.Exit(1)
		}
	} else {
		fmt.Println("boop")
		num_steps = 1000000000
	}

	fmt.Printf("Calculating PI using:\n  %v slices\n  1 process\n", num_steps)

	var sum float64 = 0
	step := 1.0 / float64(num_steps)

	var start time.Time = time.Now()

	for i = 0; i < num_steps; i++ {
		x = (float64(i) + 0.5) * step
		sum += 4.0 / (1.0 + x*x)
	}

	var stop time.Time = time.Now()

	var pi = sum * step

	timeTaken := (float64(stop.Sub(start)) * 0.000000001) // Durations come in ns

	fmt.Printf("Obtained value for PI: %.16g\n Time taken: %v s\n", pi, timeTaken)
}

//  const f = "%T(%v)\n"
<|endoftext|>


<commit_before>// Copyright 2013 Eric Myhre
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gosh

import (
	"os/exec"
	"sync"
	"sync/atomic"
	"syscall"
	"time"
)

const (
	/**
	 * 'Unstarted' is the state of a command that has been constructed, but execution has not yet begun.
	 */
	UNSTARTED int32 = iota

	/**
	 * 'Running' is the state of a command that has begun execution, but not yet finished.
	 */
	RUNNING

	/**
	 * 'Finished' is the state of a command that has finished normally.
	 *
	 * The exit code may or may not have been success, but at the very least we
	 * successfully observed that exit code.
	 */
	FINISHED

	/**
	 * 'Panicked' is the state of a command that at some point began execution, but has encountered
	 * serious problems.
	 *
	 * It may not be clear whether or not the command is still running, since a panic implies we no
	 * longer have completely clear visibility to the command on the underlying system.  The exit
	 * code may not be reliably known.
	 */
	PANICKED
)

func NewRunningCommand(cmd *exec.Cmd) *RunningCommand {
	return &RunningCommand{
		cmd:      cmd,
		state:    UNSTARTED,
		exitCh:   make(chan bool),
		exitCode: -1,
	}
}

type RunningCommand struct {
	mutex sync.Mutex

	/** Always access this with functions from the atomic package, and when
	* transitioning states set the status after all other fields are mutated,
	* so that checks of State() serve as a memory barrier for all. */
	state int32

	cmd *exec.Cmd
	//TODO: or: callback func() int

	/** If this is set, game over. */
	err error

	/** Wait for this to close in order to wait for the process to return. */
	exitCh chan bool

	/** Exit code if we're state==FINISHED and exit codes are possible on this platform, or
	 * -1 if we're not there yet.  Will not change after exitCh has closed. */
	exitCode int

	/** Functions to call back when the command has exited. */
	exitListeners []func(*RunningCommand)
}

func (cmd *RunningCommand) State() int32 {
	return atomic.LoadInt32(&cmd.state)
}

/** Returns true if the command is current running. */
func (cmd *RunningCommand) IsRunning() bool {
	state := cmd.State()
	return state == RUNNING
}

/** Returns true if the command has ever been started (including if the command is already finished). */
func (cmd *RunningCommand) IsStarted() bool {
	state := cmd.State()
	return state == RUNNING || state == FINISHED || state == PANICKED
}

/** Returns true if the command is finished (either gracefully, or with internal errors). */
func (cmd *RunningCommand) IsDone() bool {
	state := cmd.State()
	return state == FINISHED || state == PANICKED
}

/** Returns true if the command is finished gracefully.  (A nonzero exit code may still be set.) */
func (cmd *RunningCommand) IsFinishedGracefully() bool {
	state := cmd.State()
	return state == FINISHED
}

func (cmd *RunningCommand) Start() {
	if err := cmd.startCalmly(); err != nil {
		panic(err)
	}
}

func (cmd *RunningCommand) startCalmly() error {
	cmd.mutex.Lock()
	defer cmd.mutex.Unlock()

	if cmd.IsStarted() {
		return nil
	}

	atomic.StoreInt32(&cmd.state, RUNNING)
	if err := cmd.cmd.Start(); err != nil {
		cmd.finalState(CommandStartError{cause: err})
		return cmd.err
	}

	go cmd.waitAndHandleExit()
	return nil
}

func (cmd *RunningCommand) waitAndHandleExit() {
	err := cmd.cmd.Wait()

	cmd.mutex.Lock()
	defer cmd.mutex.Unlock()

	if err == nil {
		cmd.exitCode = 0
	} else if exitError, ok := err.(*exec.ExitError); ok {
		if waitStatus, ok := exitError.Sys().(syscall.WaitStatus); ok {
			cmd.exitCode = waitStatus.ExitStatus()
		} else {
			panic(exitError) //TODO: damage control better.  consider setting some kind of CommandMonitorError.
		}
	} else {
		panic(err) //TODO: damage control better.  consider setting some kind of CommandMonitorError.
	}
	cmd.finalState(nil)
}

func (cmd *RunningCommand) finalState(err error) {
	// must hold cmd.mutex before calling this
	// golang is an epic troll: claims to be best buddy for concurrent code, SYNC PACKAGE DOES NOT HAVE REENTRANT LOCKS
	if cmd.IsRunning() {
		if err == nil {
			atomic.StoreInt32(&cmd.state, FINISHED)
		} else {
			cmd.err = err
			atomic.StoreInt32(&cmd.state, PANICKED)
		}
		//TODO iterate over exit listeners
		for _, cb := range cmd.exitListeners {
			func() {
				defer recover()
				cb(cmd)
			}()
		}
	}
	close(cmd.exitCh)
}

/**
 * Add a function to be called when this command completes.
 *
 * These listener functions will be invoked after the exit code and other command
 * state is final, but before other Wait() methods unblock.
 * (This means if you want for example to log a message that a process exited, and
 * your main function is Wait()'ing for that process... if you use AddExitListener()
 * to invoke your log function then you will always get the log.)
 *
 * The listener function should complete quickly and not try to perform other blocking
 * operations or locks, since other actions are waiting until the listeners have all
 * been called.  Panics that escape the function will be silently discarded; do not
 * panic in a listener.
 *
 * If the command is already in the state FINISHED or PANICKED, the callback function
 * will be invoked immediately in the current goroutine.
 */
func (cmd *RunningCommand) AddExitListener(callback func(*RunningCommand)) {
	cmd.mutex.Lock()
	defer cmd.mutex.Unlock()

	if cmd.IsDone() {
		func() {
			defer recover()
			callback(cmd)
		}()
	} else {
		cmd.exitListeners = append(cmd.exitListeners, callback)
	}
}

/**
 * Returns a channel that will be open until the command is complete.
 * This is suitable for use in a select block.
 */
func (cmd *RunningCommand) GetExitChannel() <-chan bool {
	return cmd.exitCh
}

/**
 * Waits for the command to exit before returning.
 *
 * There are no consequences to waiting on a single command repeatedly;
 * all wait calls will return normally when the command completes.  The order
 * in which multiple wait calls will return is undefined.  Similarly, there
 * are no consequences to waiting on a command that has not yet started;
 * the function will still wait without error until the command finishes.
 * (Much friendlier than os.exec.Cmd.Wait(), neh?)
 */
func (cmd *RunningCommand) Wait() {
	<-cmd.GetExitChannel()
}

/**
 * Waits for the command to exit before returning, or for the specified duration.
 * Returns true if the return was due to the command finishing, or false if the
 * return was due to timeout.
 */
func (cmd *RunningCommand) WaitSoon(d time.Duration) bool {
	select {
	case <-time.After(d):
		return false
	case <-cmd.GetExitChannel():
		return true
	}
}

/**
 * Waits for the command to exit if it has not already, then returns the exit code.
 */
func (cmd *RunningCommand) GetExitCode() int {
	if !cmd.IsDone() {
		cmd.Wait()
	}
	return cmd.exitCode
}

/**
 * Waits for the command to exit if it has not already, or for the specified duration,
 * then either returns the exit code, or -1 if the duration expired and the command
 * still hasn't returned.
 */
func (cmd *RunningCommand) GetExitCodeSoon(d time.Duration) int {
	if cmd.WaitSoon(d) {
		return cmd.exitCode
	} else {
		return -1
	}
}
<commit_msg>RunningCommand.Start() now returns the same object for chaining convenience.<commit_after>// Copyright 2013 Eric Myhre
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gosh

import (
	"os/exec"
	"sync"
	"sync/atomic"
	"syscall"
	"time"
)

const (
	/**
	 * 'Unstarted' is the state of a command that has been constructed, but execution has not yet begun.
	 */
	UNSTARTED int32 = iota

	/**
	 * 'Running' is the state of a command that has begun execution, but not yet finished.
	 */
	RUNNING

	/**
	 * 'Finished' is the state of a command that has finished normally.
	 *
	 * The exit code may or may not have been success, but at the very least we
	 * successfully observed that exit code.
	 */
	FINISHED

	/**
	 * 'Panicked' is the state of a command that at some point began execution, but has encountered
	 * serious problems.
	 *
	 * It may not be clear whether or not the command is still running, since a panic implies we no
	 * longer have completely clear visibility to the command on the underlying system.  The exit
	 * code may not be reliably known.
	 */
	PANICKED
)

func NewRunningCommand(cmd *exec.Cmd) *RunningCommand {
	return &RunningCommand{
		cmd:      cmd,
		state:    UNSTARTED,
		exitCh:   make(chan bool),
		exitCode: -1,
	}
}

type RunningCommand struct {
	mutex sync.Mutex

	/** Always access this with functions from the atomic package, and when
	* transitioning states set the status after all other fields are mutated,
	* so that checks of State() serve as a memory barrier for all. */
	state int32

	cmd *exec.Cmd
	//TODO: or: callback func() int

	/** If this is set, game over. */
	err error

	/** Wait for this to close in order to wait for the process to return. */
	exitCh chan bool

	/** Exit code if we're state==FINISHED and exit codes are possible on this platform, or
	 * -1 if we're not there yet.  Will not change after exitCh has closed. */
	exitCode int

	/** Functions to call back when the command has exited. */
	exitListeners []func(*RunningCommand)
}

func (cmd *RunningCommand) State() int32 {
	return atomic.LoadInt32(&cmd.state)
}

/** Returns true if the command is current running. */
func (cmd *RunningCommand) IsRunning() bool {
	state := cmd.State()
	return state == RUNNING
}

/** Returns true if the command has ever been started (including if the command is already finished). */
func (cmd *RunningCommand) IsStarted() bool {
	state := cmd.State()
	return state == RUNNING || state == FINISHED || state == PANICKED
}

/** Returns true if the command is finished (either gracefully, or with internal errors). */
func (cmd *RunningCommand) IsDone() bool {
	state := cmd.State()
	return state == FINISHED || state == PANICKED
}

/** Returns true if the command is finished gracefully.  (A nonzero exit code may still be set.) */
func (cmd *RunningCommand) IsFinishedGracefully() bool {
	state := cmd.State()
	return state == FINISHED
}

func (cmd *RunningCommand) Start() *RunningCommand {
	if err := cmd.startCalmly(); err != nil {
		panic(err)
	}
	return cmd
}

func (cmd *RunningCommand) startCalmly() error {
	cmd.mutex.Lock()
	defer cmd.mutex.Unlock()

	if cmd.IsStarted() {
		return nil
	}

	atomic.StoreInt32(&cmd.state, RUNNING)
	if err := cmd.cmd.Start(); err != nil {
		cmd.finalState(CommandStartError{cause: err})
		return cmd.err
	}

	go cmd.waitAndHandleExit()
	return nil
}

func (cmd *RunningCommand) waitAndHandleExit() {
	err := cmd.cmd.Wait()

	cmd.mutex.Lock()
	defer cmd.mutex.Unlock()

	if err == nil {
		cmd.exitCode = 0
	} else if exitError, ok := err.(*exec.ExitError); ok {
		if waitStatus, ok := exitError.Sys().(syscall.WaitStatus); ok {
			cmd.exitCode = waitStatus.ExitStatus()
		} else {
			panic(exitError) //TODO: damage control better.  consider setting some kind of CommandMonitorError.
		}
	} else {
		panic(err) //TODO: damage control better.  consider setting some kind of CommandMonitorError.
	}
	cmd.finalState(nil)
}

func (cmd *RunningCommand) finalState(err error) {
	// must hold cmd.mutex before calling this
	// golang is an epic troll: claims to be best buddy for concurrent code, SYNC PACKAGE DOES NOT HAVE REENTRANT LOCKS
	if cmd.IsRunning() {
		if err == nil {
			atomic.StoreInt32(&cmd.state, FINISHED)
		} else {
			cmd.err = err
			atomic.StoreInt32(&cmd.state, PANICKED)
		}
		//TODO iterate over exit listeners
		for _, cb := range cmd.exitListeners {
			func() {
				defer recover()
				cb(cmd)
			}()
		}
	}
	close(cmd.exitCh)
}

/**
 * Add a function to be called when this command completes.
 *
 * These listener functions will be invoked after the exit code and other command
 * state is final, but before other Wait() methods unblock.
 * (This means if you want for example to log a message that a process exited, and
 * your main function is Wait()'ing for that process... if you use AddExitListener()
 * to invoke your log function then you will always get the log.)
 *
 * The listener function should complete quickly and not try to perform other blocking
 * operations or locks, since other actions are waiting until the listeners have all
 * been called.  Panics that escape the function will be silently discarded; do not
 * panic in a listener.
 *
 * If the command is already in the state FINISHED or PANICKED, the callback function
 * will be invoked immediately in the current goroutine.
 */
func (cmd *RunningCommand) AddExitListener(callback func(*RunningCommand)) {
	cmd.mutex.Lock()
	defer cmd.mutex.Unlock()

	if cmd.IsDone() {
		func() {
			defer recover()
			callback(cmd)
		}()
	} else {
		cmd.exitListeners = append(cmd.exitListeners, callback)
	}
}

/**
 * Returns a channel that will be open until the command is complete.
 * This is suitable for use in a select block.
 */
func (cmd *RunningCommand) GetExitChannel() <-chan bool {
	return cmd.exitCh
}

/**
 * Waits for the command to exit before returning.
 *
 * There are no consequences to waiting on a single command repeatedly;
 * all wait calls will return normally when the command completes.  The order
 * in which multiple wait calls will return is undefined.  Similarly, there
 * are no consequences to waiting on a command that has not yet started;
 * the function will still wait without error until the command finishes.
 * (Much friendlier than os.exec.Cmd.Wait(), neh?)
 */
func (cmd *RunningCommand) Wait() {
	<-cmd.GetExitChannel()
}

/**
 * Waits for the command to exit before returning, or for the specified duration.
 * Returns true if the return was due to the command finishing, or false if the
 * return was due to timeout.
 */
func (cmd *RunningCommand) WaitSoon(d time.Duration) bool {
	select {
	case <-time.After(d):
		return false
	case <-cmd.GetExitChannel():
		return true
	}
}

/**
 * Waits for the command to exit if it has not already, then returns the exit code.
 */
func (cmd *RunningCommand) GetExitCode() int {
	if !cmd.IsDone() {
		cmd.Wait()
	}
	return cmd.exitCode
}

/**
 * Waits for the command to exit if it has not already, or for the specified duration,
 * then either returns the exit code, or -1 if the duration expired and the command
 * still hasn't returned.
 */
func (cmd *RunningCommand) GetExitCodeSoon(d time.Duration) int {
	if cmd.WaitSoon(d) {
		return cmd.exitCode
	} else {
		return -1
	}
}
<|endoftext|>