<commit_before>package client

import (
	"fmt"
	neturl "net/url"
	"reflect"
	"regexp"
	"strings"
	"time"

	_ "github.com/lib/pq"

	"github.com/jmoiron/sqlx"
	"github.com/sosedoff/pgweb/pkg/command"
	"github.com/sosedoff/pgweb/pkg/connection"
	"github.com/sosedoff/pgweb/pkg/history"
	"github.com/sosedoff/pgweb/pkg/shared"
	"github.com/sosedoff/pgweb/pkg/statements"
)

var (
	postgresSignature = regexp.MustCompile(`(?i)postgresql ([\d\.]+)\s`)
	postgresType      = "PostgreSQL"

	cockroachSignature = regexp.MustCompile(`(?i)cockroachdb ccl v([\d\.]+)\s`)
	cockroachType      = "CockroachDB"
)

type Client struct {
	db               *sqlx.DB
	tunnel           *Tunnel
	serverVersion    string
	serverType       string
	lastQueryTime    time.Time
	External         bool
	History          []history.Record `json:"history"`
	ConnectionString string           `json:"connection_string"`
}

// Struct to hold table rows browsing options
type RowsOptions struct {
	Where      string // Custom filter
	Offset     int    // Number of rows to skip
	Limit      int    // Number of rows to fetch
	SortColumn string // Column to sort by
	SortOrder  string // Sort direction (ASC, DESC)
}

func getSchemaAndTable(str string) (string, string) {
	chunks := strings.Split(str, ".")
	if len(chunks) == 1 {
		return "public", chunks[0]
	}
	return chunks[0], chunks[1]
}

func New() (*Client, error) {
	str, err := connection.BuildStringFromOptions(command.Opts)

	if command.Opts.Debug && str != "" {
		fmt.Println("Creating a new client for:", str)
	}

	if err != nil {
		return nil, err
	}

	db, err := sqlx.Open("postgres", str)
	if err != nil {
		return nil, err
	}

	client := Client{
		db:               db,
		ConnectionString: str,
		History:          history.New(),
	}

	client.setServerVersion()
	return &client, nil
}

func NewFromUrl(url string, sshInfo *shared.SSHInfo) (*Client, error) {
	var tunnel *Tunnel

	if sshInfo != nil {
		if command.Opts.DisableSSH {
			return nil, fmt.Errorf("ssh connections are disabled")
		}
		if command.Opts.Debug {
			fmt.Println("Opening SSH tunnel for:", sshInfo)
		}

		tunnel, err := NewTunnel(sshInfo, url)
		if err != nil {
			tunnel.Close()
			return nil, err
		}

		err = tunnel.Configure()
		if err != nil {
			tunnel.Close()
			return nil, err
		}

		go tunnel.Start()

		uri, err := neturl.Parse(url)
		if err != nil {
			tunnel.Close()
			return nil, err
		}

		// Override remote postgres port with local proxy port
		url = strings.Replace(url, uri.Host, fmt.Sprintf("127.0.0.1:%v", tunnel.Port), 1)
	}

	if command.Opts.Debug {
		fmt.Println("Creating a new client for:", url)
	}

	uri, err := neturl.Parse(url)
	if err == nil && uri.Path == "" {
		return nil, fmt.Errorf("Database name is not provided")
	}

	db, err := sqlx.Open("postgres", url)
	if err != nil {
		return nil, err
	}

	client := Client{
		db:               db,
		tunnel:           tunnel,
		ConnectionString: url,
		History:          history.New(),
	}

	client.setServerVersion()
	return &client, nil
}

func (client *Client) setServerVersion() {
	res, err := client.query("SELECT version()")
	if err != nil || len(res.Rows) < 1 {
		return
	}

	version := res.Rows[0][0].(string)

	// Detect postgresql
	matches := postgresSignature.FindAllStringSubmatch(version, 1)
	if len(matches) > 0 {
		client.serverType = postgresType
		client.serverVersion = matches[0][1]
		return
	}

	// Detect cockroachdb
	matches = cockroachSignature.FindAllStringSubmatch(version, 1)
	if len(matches) > 0 {
		client.serverType = cockroachType
		client.serverVersion = matches[0][1]
		return
	}
}

func (client *Client) Test() error {
	return client.db.Ping()
}

func (client *Client) Info() (*Result, error) {
	return client.query(statements.Info)
}

func (client *Client) Databases() ([]string, error) {
	return client.fetchRows(statements.Databases)
}

func (client *Client) Schemas() ([]string, error) {
	return client.fetchRows(statements.Schemas)
}

func (client *Client) Objects() (*Result, error) {
	return client.query(statements.Objects)
}

func (client *Client) Table(table string) (*Result, error) {
	schema, table := getSchemaAndTable(table)
	return client.query(statements.TableSchema, schema, table)
}

func (client *Client) MaterializedView(name string) (*Result, error) {
	return client.query(statements.MaterializedView, name)
}

func (client *Client) TableRows(table string, opts RowsOptions) (*Result, error) {
	schema, table := getSchemaAndTable(table)
	sql := fmt.Sprintf(`SELECT * FROM "%s"."%s"`, schema, table)

	if opts.Where != "" {
		sql += fmt.Sprintf(" WHERE %s", opts.Where)
	}

	if opts.SortColumn != "" {
		if opts.SortOrder == "" {
			opts.SortOrder = "ASC"
		}

		sql += fmt.Sprintf(` ORDER BY "%s" %s`, opts.SortColumn, opts.SortOrder)
	}

	if opts.Limit > 0 {
		sql += fmt.Sprintf(" LIMIT %d", opts.Limit)
	}

	if opts.Offset > 0 {
		sql += fmt.Sprintf(" OFFSET %d", opts.Offset)
	}

	return client.query(sql)
}

func (client *Client) EstimatedTableRowsCount(table string, opts RowsOptions) (*Result, error) {
	schema, table := getSchemaAndTable(table)
	sql := fmt.Sprintf(`SELECT reltuples FROM pg_class WHERE oid = '%s.%s'::regclass;`, schema, table)

	result, err := client.query(sql)
	if err != nil {
		return nil, err
	}
	// float64 to int64 conversion
	estimatedRowsCount := result.Rows[0][0].(float64)
	result.Rows[0] = Row{int64(estimatedRowsCount)}

	return result, nil
}

func (client *Client) TableRowsCount(table string, opts RowsOptions) (*Result, error) {
	schema, table := getSchemaAndTable(table)
	sql := fmt.Sprintf(`SELECT COUNT(1) FROM "%s"."%s"`, schema, table)

	if opts.Where != "" {
		sql += fmt.Sprintf(" WHERE %s", opts.Where)
	} else if client.serverType == postgresType {
		tableInfo, err := client.TableInfo(table)
		if err != nil {
			return nil, err
		}
		estimatedRowsCount := tableInfo.Rows[0][3].(float64)
		if estimatedRowsCount > 100000 {
			return client.EstimatedTableRowsCount(table, opts)
		}
	}

	return client.query(sql)
}

func (client *Client) TableInfo(table string) (*Result, error) {
	if client.serverType == cockroachType {
		return client.query(statements.TableInfoCockroach)
	}
	schema, table := getSchemaAndTable(table)
	return client.query(statements.TableInfo, fmt.Sprintf(`"%s"."%s"`, schema, table))
}

func (client *Client) TableIndexes(table string) (*Result, error) {
	schema, table := getSchemaAndTable(table)
	res, err := client.query(statements.TableIndexes, schema, table)

	if err != nil {
		return nil, err
	}

	return res, err
}

func (client *Client) TableConstraints(table string) (*Result, error) {
	schema, table := getSchemaAndTable(table)
	res, err := client.query(statements.TableConstraints, schema, table)

	if err != nil {
		return nil, err
	}

	return res, err
}

// Returns all active queriers on the server
func (client *Client) Activity() (*Result, error) {
	if client.serverType == cockroachType {
		return client.query("SHOW QUERIES")
	}

	version := getMajorMinorVersion(client.serverVersion)
	query := statements.Activity[version]
	if query == "" {
		query = statements.Activity["default"]
	}

	return client.query(query)
}

func (client *Client) Query(query string) (*Result, error) {
	res, err := client.query(query)

	// Save history records only if query did not fail
	if err == nil && !client.hasHistoryRecord(query) {
		client.History = append(client.History, history.NewRecord(query))
	}

	return res, err
}

func (client *Client) SetReadOnlyMode() error {
	var value string
	if err := client.db.Get(&value, "SHOW default_transaction_read_only;"); err != nil {
		return err
	}

	if value == "off" {
		_, err := client.db.Exec("SET default_transaction_read_only=on;")
		return err
	}

	return nil
}

func (client *Client) ServerVersion() string {
	return fmt.Sprintf("%s %s", client.serverType, client.serverVersion)
}

func (client *Client) query(query string, args ...interface{}) (*Result, error) {
	// Update the last usage time
	defer func() {
		client.lastQueryTime = time.Now().UTC()
	}()

	// We're going to force-set transaction mode on every query.
	// This is needed so that default mode could not be changed by user.
	if command.Opts.ReadOnly {
		if err := client.SetReadOnlyMode(); err != nil {
			return nil, err
		}
	}

	action := strings.ToLower(strings.Split(query, " ")[0])
	if action == "update" || action == "delete" {
		res, err := client.db.Exec(query, args...)
		if err != nil {
			return nil, err
		}

		affected, err := res.RowsAffected()
		if err != nil {
			return nil, err
		}

		result := Result{
			Columns: []string{"Rows Affected"},
			Rows: []Row{
				Row{affected},
			},
		}

		return &result, nil
	}

	rows, err := client.db.Queryx(query, args...)
	if err != nil {
		return nil, err
	}

	defer rows.Close()

	cols, err := rows.Columns()
	if err != nil {
		return nil, err
	}

	// Make sure to never return null colums
	if cols == nil {
		cols = []string{}
	}

	result := Result{
		Columns: cols,
		Rows:    []Row{},
	}

	for rows.Next() {
		obj, err := rows.SliceScan()

		for i, item := range obj {
			if item == nil {
				obj[i] = nil
			} else {
				t := reflect.TypeOf(item).Kind().String()

				if t == "slice" {
					obj[i] = string(item.([]byte))
				}
			}
		}

		if err == nil {
			result.Rows = append(result.Rows, obj)
		}
	}

	result.PrepareBigints()

	return &result, nil
}

// Close database connection
func (client *Client) Close() error {
	if client.tunnel != nil {
		client.tunnel.Close()
	}

	if client.db != nil {
		return client.db.Close()
	}

	return nil
}

func (client *Client) IsIdle() bool {
	mins := int(time.Since(client.lastQueryTime).Minutes())

	if command.Opts.ConnectionIdleTimeout > 0 {
		return mins >= command.Opts.ConnectionIdleTimeout
	}

	return false
}

// Fetch all rows as strings for a single column
func (client *Client) fetchRows(q string) ([]string, error) {
	res, err := client.query(q)

	if err != nil {
		return nil, err
	}

	// Init empty slice so json.Marshal will encode it to "[]" instead of "null"
	results := make([]string, 0)

	for _, row := range res.Rows {
		results = append(results, row[0].(string))
	}

	return results, nil
}

func (client *Client) hasHistoryRecord(query string) bool {
	result := false

	for _, record := range client.History {
		if record.Query == query {
			result = true
			break
		}
	}

	return result
}
<commit_msg>Fix row estimation bug<commit_after>package client

import (
	"fmt"
	neturl "net/url"
	"reflect"
	"regexp"
	"strings"
	"time"

	_ "github.com/lib/pq"

	"github.com/jmoiron/sqlx"
	"github.com/sosedoff/pgweb/pkg/command"
	"github.com/sosedoff/pgweb/pkg/connection"
	"github.com/sosedoff/pgweb/pkg/history"
	"github.com/sosedoff/pgweb/pkg/shared"
	"github.com/sosedoff/pgweb/pkg/statements"
)

var (
	postgresSignature = regexp.MustCompile(`(?i)postgresql ([\d\.]+)\s`)
	postgresType      = "PostgreSQL"

	cockroachSignature = regexp.MustCompile(`(?i)cockroachdb ccl v([\d\.]+)\s`)
	cockroachType      = "CockroachDB"
)

type Client struct {
	db               *sqlx.DB
	tunnel           *Tunnel
	serverVersion    string
	serverType       string
	lastQueryTime    time.Time
	External         bool
	History          []history.Record `json:"history"`
	ConnectionString string           `json:"connection_string"`
}

// Struct to hold table rows browsing options
type RowsOptions struct {
	Where      string // Custom filter
	Offset     int    // Number of rows to skip
	Limit      int    // Number of rows to fetch
	SortColumn string // Column to sort by
	SortOrder  string // Sort direction (ASC, DESC)
}

func getSchemaAndTable(str string) (string, string) {
	chunks := strings.Split(str, ".")
	if len(chunks) == 1 {
		return "public", chunks[0]
	}
	return chunks[0], chunks[1]
}

func New() (*Client, error) {
	str, err := connection.BuildStringFromOptions(command.Opts)

	if command.Opts.Debug && str != "" {
		fmt.Println("Creating a new client for:", str)
	}

	if err != nil {
		return nil, err
	}

	db, err := sqlx.Open("postgres", str)
	if err != nil {
		return nil, err
	}

	client := Client{
		db:               db,
		ConnectionString: str,
		History:          history.New(),
	}

	client.setServerVersion()
	return &client, nil
}

func NewFromUrl(url string, sshInfo *shared.SSHInfo) (*Client, error) {
	var tunnel *Tunnel

	if sshInfo != nil {
		if command.Opts.DisableSSH {
			return nil, fmt.Errorf("ssh connections are disabled")
		}
		if command.Opts.Debug {
			fmt.Println("Opening SSH tunnel for:", sshInfo)
		}

		tunnel, err := NewTunnel(sshInfo, url)
		if err != nil {
			tunnel.Close()
			return nil, err
		}

		err = tunnel.Configure()
		if err != nil {
			tunnel.Close()
			return nil, err
		}

		go tunnel.Start()

		uri, err := neturl.Parse(url)
		if err != nil {
			tunnel.Close()
			return nil, err
		}

		// Override remote postgres port with local proxy port
		url = strings.Replace(url, uri.Host, fmt.Sprintf("127.0.0.1:%v", tunnel.Port), 1)
	}

	if command.Opts.Debug {
		fmt.Println("Creating a new client for:", url)
	}

	uri, err := neturl.Parse(url)
	if err == nil && uri.Path == "" {
		return nil, fmt.Errorf("Database name is not provided")
	}

	db, err := sqlx.Open("postgres", url)
	if err != nil {
		return nil, err
	}

	client := Client{
		db:               db,
		tunnel:           tunnel,
		ConnectionString: url,
		History:          history.New(),
	}

	client.setServerVersion()
	return &client, nil
}

func (client *Client) setServerVersion() {
	res, err := client.query("SELECT version()")
	if err != nil || len(res.Rows) < 1 {
		return
	}

	version := res.Rows[0][0].(string)

	// Detect postgresql
	matches := postgresSignature.FindAllStringSubmatch(version, 1)
	if len(matches) > 0 {
		client.serverType = postgresType
		client.serverVersion = matches[0][1]
		return
	}

	// Detect cockroachdb
	matches = cockroachSignature.FindAllStringSubmatch(version, 1)
	if len(matches) > 0 {
		client.serverType = cockroachType
		client.serverVersion = matches[0][1]
		return
	}
}

func (client *Client) Test() error {
	return client.db.Ping()
}

func (client *Client) Info() (*Result, error) {
	return client.query(statements.Info)
}

func (client *Client) Databases() ([]string, error) {
	return client.fetchRows(statements.Databases)
}

func (client *Client) Schemas() ([]string, error) {
	return client.fetchRows(statements.Schemas)
}

func (client *Client) Objects() (*Result, error) {
	return client.query(statements.Objects)
}

func (client *Client) Table(table string) (*Result, error) {
	schema, table := getSchemaAndTable(table)
	return client.query(statements.TableSchema, schema, table)
}

func (client *Client) MaterializedView(name string) (*Result, error) {
	return client.query(statements.MaterializedView, name)
}

func (client *Client) TableRows(table string, opts RowsOptions) (*Result, error) {
	schema, table := getSchemaAndTable(table)
	sql := fmt.Sprintf(`SELECT * FROM "%s"."%s"`, schema, table)

	if opts.Where != "" {
		sql += fmt.Sprintf(" WHERE %s", opts.Where)
	}

	if opts.SortColumn != "" {
		if opts.SortOrder == "" {
			opts.SortOrder = "ASC"
		}

		sql += fmt.Sprintf(` ORDER BY "%s" %s`, opts.SortColumn, opts.SortOrder)
	}

	if opts.Limit > 0 {
		sql += fmt.Sprintf(" LIMIT %d", opts.Limit)
	}

	if opts.Offset > 0 {
		sql += fmt.Sprintf(" OFFSET %d", opts.Offset)
	}

	return client.query(sql)
}

func (client *Client) EstimatedTableRowsCount(table string, opts RowsOptions) (*Result, error) {
	schema, table := getSchemaAndTable(table)
	sql := fmt.Sprintf(`SELECT reltuples FROM pg_class WHERE oid = '%s.%s'::regclass;`, schema, table)

	result, err := client.query(sql)
	if err != nil {
		return nil, err
	}
	// float64 to int64 conversion
	estimatedRowsCount := result.Rows[0][0].(float64)
	result.Rows[0] = Row{int64(estimatedRowsCount)}

	return result, nil
}

func (client *Client) TableRowsCount(table string, opts RowsOptions) (*Result, error) {
	schema, tableName := getSchemaAndTable(table)
	sql := fmt.Sprintf(`SELECT COUNT(1) FROM "%s"."%s"`, schema, tableName)

	if opts.Where != "" {
		sql += fmt.Sprintf(" WHERE %s", opts.Where)
	} else if client.serverType == postgresType {
		tableInfo, err := client.TableInfo(table)
		if err != nil {
			return nil, err
		}
		estimatedRowsCount := tableInfo.Rows[0][3].(float64)
		if estimatedRowsCount > 100000 {
			return client.EstimatedTableRowsCount(table, opts)
		}
	}

	return client.query(sql)
}

func (client *Client) TableInfo(table string) (*Result, error) {
	if client.serverType == cockroachType {
		return client.query(statements.TableInfoCockroach)
	}
	schema, table := getSchemaAndTable(table)
	return client.query(statements.TableInfo, fmt.Sprintf(`"%s"."%s"`, schema, table))
}

func (client *Client) TableIndexes(table string) (*Result, error) {
	schema, table := getSchemaAndTable(table)
	res, err := client.query(statements.TableIndexes, schema, table)

	if err != nil {
		return nil, err
	}

	return res, err
}

func (client *Client) TableConstraints(table string) (*Result, error) {
	schema, table := getSchemaAndTable(table)
	res, err := client.query(statements.TableConstraints, schema, table)

	if err != nil {
		return nil, err
	}

	return res, err
}

// Returns all active queriers on the server
func (client *Client) Activity() (*Result, error) {
	if client.serverType == cockroachType {
		return client.query("SHOW QUERIES")
	}

	version := getMajorMinorVersion(client.serverVersion)
	query := statements.Activity[version]
	if query == "" {
		query = statements.Activity["default"]
	}

	return client.query(query)
}

func (client *Client) Query(query string) (*Result, error) {
	res, err := client.query(query)

	// Save history records only if query did not fail
	if err == nil && !client.hasHistoryRecord(query) {
		client.History = append(client.History, history.NewRecord(query))
	}

	return res, err
}

func (client *Client) SetReadOnlyMode() error {
	var value string
	if err := client.db.Get(&value, "SHOW default_transaction_read_only;"); err != nil {
		return err
	}

	if value == "off" {
		_, err := client.db.Exec("SET default_transaction_read_only=on;")
		return err
	}

	return nil
}

func (client *Client) ServerVersion() string {
	return fmt.Sprintf("%s %s", client.serverType, client.serverVersion)
}

func (client *Client) query(query string, args ...interface{}) (*Result, error) {
	// Update the last usage time
	defer func() {
		client.lastQueryTime = time.Now().UTC()
	}()

	// We're going to force-set transaction mode on every query.
	// This is needed so that default mode could not be changed by user.
	if command.Opts.ReadOnly {
		if err := client.SetReadOnlyMode(); err != nil {
			return nil, err
		}
	}

	action := strings.ToLower(strings.Split(query, " ")[0])
	if action == "update" || action == "delete" {
		res, err := client.db.Exec(query, args...)
		if err != nil {
			return nil, err
		}

		affected, err := res.RowsAffected()
		if err != nil {
			return nil, err
		}

		result := Result{
			Columns: []string{"Rows Affected"},
			Rows: []Row{
				Row{affected},
			},
		}

		return &result, nil
	}

	rows, err := client.db.Queryx(query, args...)
	if err != nil {
		return nil, err
	}

	defer rows.Close()

	cols, err := rows.Columns()
	if err != nil {
		return nil, err
	}

	// Make sure to never return null colums
	if cols == nil {
		cols = []string{}
	}

	result := Result{
		Columns: cols,
		Rows:    []Row{},
	}

	for rows.Next() {
		obj, err := rows.SliceScan()

		for i, item := range obj {
			if item == nil {
				obj[i] = nil
			} else {
				t := reflect.TypeOf(item).Kind().String()

				if t == "slice" {
					obj[i] = string(item.([]byte))
				}
			}
		}

		if err == nil {
			result.Rows = append(result.Rows, obj)
		}
	}

	result.PrepareBigints()

	return &result, nil
}

// Close database connection
func (client *Client) Close() error {
	if client.tunnel != nil {
		client.tunnel.Close()
	}

	if client.db != nil {
		return client.db.Close()
	}

	return nil
}

func (client *Client) IsIdle() bool {
	mins := int(time.Since(client.lastQueryTime).Minutes())

	if command.Opts.ConnectionIdleTimeout > 0 {
		return mins >= command.Opts.ConnectionIdleTimeout
	}

	return false
}

// Fetch all rows as strings for a single column
func (client *Client) fetchRows(q string) ([]string, error) {
	res, err := client.query(q)

	if err != nil {
		return nil, err
	}

	// Init empty slice so json.Marshal will encode it to "[]" instead of "null"
	results := make([]string, 0)

	for _, row := range res.Rows {
		results = append(results, row[0].(string))
	}

	return results, nil
}

func (client *Client) hasHistoryRecord(query string) bool {
	result := false

	for _, record := range client.History {
		if record.Query == query {
			result = true
			break
		}
	}

	return result
}
<|endoftext|>


<commit_before>/*******************************************************************************
*
* Copyright 2017 SAP SE
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You should have received a copy of the License along with this
* program. If not, you may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*******************************************************************************/

package db

import (
	"database/sql"
	"fmt"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"

	gorp "gopkg.in/gorp.v2"

	"github.com/majewsky/sqlproxy"
	"github.com/sapcc/limes/pkg/util"
	//enable postgres driver for database/sql
	_ "github.com/lib/pq"
)

//DB holds the main database connection. It will be `nil` until InitDatabase() is called.
var DB *gorp.DbMap

func init() {
	sql.Register("postgres-debug", &sqlproxy.Driver{
		ProxiedDriverName: "postgres",
		BeforeQueryHook:   traceQuery,
	})
	//this driver only used within unit tests
	sql.Register("sqlite3-debug", &sqlproxy.Driver{
		ProxiedDriverName: "sqlite3-limes", //this driver is defined in pkg/test/db.go
		BeforeQueryHook:   traceQuery,
		BeforePrepareHook: func(query string) (string, error) {
			//rewrite Postgres function name into SQLite function name
			query = regexp.MustCompile(`\bGREATEST\b`).ReplaceAllString(query, "MAX")
			//Postgres is okay with a no-op "WHERE TRUE" clause, but SQLite does not know the TRUE literal
			query = regexp.MustCompile(`\bWHERE TRUE\s*(GROUP|LIMIT|ORDER|$)`).ReplaceAllString(query, "$1")
			query = regexp.MustCompile(`\bWHERE TRUE AND\b`).ReplaceAllString(query, "WHERE")
			traceQuery(query, []interface{}{"PREPARE"})
			return query, nil
		},
	})
}

//Configuration is the section of the global configuration file that
//contains the data about
type Configuration struct {
	Location       string `yaml:"location"`
	MigrationsPath string `yaml:"migrations"`
}

//Init initializes the connection to the database.
func Init(cfg Configuration) error {
	sqlDriver := "postgres"
	if os.Getenv("LIMES_DEBUG_SQL") == "1" {
		util.LogInfo("Enabling SQL tracing... \x1B[1;31mTHIS VOIDS YOUR WARRANTY!\x1B[0m If database queries fail in unexpected ways, check first if the tracing causes the issue.")
		sqlDriver += "-debug"
	}

	db, err := sql.Open(sqlDriver, cfg.Location)
	if err != nil {
		return err
	}
	DB = &gorp.DbMap{Db: db, Dialect: gorp.PostgresDialect{}}
	InitGorp()

	//wait for database to reach our expected migration level (this is useful
	//because, depending on the rollout strategy, `limes-migrate` might still be
	//running when we are starting, so wait for it to complete)
	migrationLevel, err := getCurrentMigrationLevel(cfg)
	util.LogDebug("waiting for database to migrate to schema version %d", migrationLevel)
	if err != nil {
		return err
	}
	stmt, err := DB.Prepare(fmt.Sprintf("SELECT 1 FROM schema_migrations WHERE version = %d", migrationLevel))
	if err != nil {
		return err
	}
	defer stmt.Close()

	waitInterval := 1
	for {
		rows, err := stmt.Query()
		if err != nil {
			return err
		}
		if rows.Next() {
			//got a row - success
			break
		}
		//did not get a row - expected migration not there -> sleep with exponential backoff
		waitInterval *= 2
		util.LogInfo("database is not migrated to schema version %d yet - will retry in %d seconds", migrationLevel, waitInterval)
		time.Sleep(time.Duration(waitInterval) * time.Second)
	}

	util.LogDebug("database is migrated - commencing normal startup...")
	return nil
}

func getCurrentMigrationLevel(cfg Configuration) (int, error) {
	//list files in migration directory
	dir, err := os.Open(cfg.MigrationsPath)
	if err != nil {
		return 0, err
	}
	fileNames, err := dir.Readdirnames(-1)
	if err != nil {
		return 0, err
	}

	result := 0
	rx := regexp.MustCompile(`^([0-9]+)_.*\.(?:up|down)\.sql`)
	//find the relevant SQL files and extract their migration numbers
	for _, fileName := range fileNames {
		match := rx.FindStringSubmatch(fileName)
		if match != nil {
			migration, _ := strconv.Atoi(match[1])
			if migration > result {
				result = migration
			}
		}
	}

	return result, nil
}

var sqlWhitespaceRx = regexp.MustCompile(`(?:\s|--.*)+`) // `.*` matches until end of line!

func traceQuery(query string, args []interface{}) {
	//simplify query string - remove comments and reduce whitespace
	//(This logic assumes that there are no arbitrary strings in the SQL
	//statement, which is okay since values should be given as args anyway.)
	query = strings.TrimSpace(sqlWhitespaceRx.ReplaceAllString(query, " "))

	//early exit for easy option
	if len(args) == 0 {
		util.LogDebug(query)
		return
	}

	//if args contains time.Time objects, pretty-print these; use
	//fmt.Sprintf("%#v") for all other types of values
	argStrings := make([]string, len(args))
	for idx, argument := range args {
		switch arg := argument.(type) {
		case time.Time:
			argStrings[idx] = "time.Time [" + arg.Local().String() + "]"
		default:
			argStrings[idx] = fmt.Sprintf("%#v", arg)
		}
	}
	util.LogDebug(query + " [" + strings.Join(argStrings, ", ") + "]")
}

//RollbackUnlessCommitted calls Rollback() on a transaction if it hasn't been
//committed or rolled back yet. Use this with the defer keyword to make sure
//that a transaction is automatically rolled back when a function fails.
func RollbackUnlessCommitted(tx *gorp.Transaction) {
	err := tx.Rollback()
	switch err {
	case nil:
		//rolled back successfully
		util.LogInfo("implicit rollback done")
		return
	case sql.ErrTxDone:
		//already committed or rolled back - nothing to do
		return
	default:
		util.LogError("implicit rollback failed: %s", err.Error())
	}
}

//Interface provides the common methods that both SQL connections and
//transactions implement.
type Interface interface {
	Exec(query string, args ...interface{}) (sql.Result, error)
	Prepare(query string) (*sql.Stmt, error)
	Query(query string, args ...interface{}) (*sql.Rows, error)
	QueryRow(query string, args ...interface{}) *sql.Row
}
<commit_msg>deactivate some debug logging<commit_after>/*******************************************************************************
*
* Copyright 2017 SAP SE
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You should have received a copy of the License along with this
* program. If not, you may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*******************************************************************************/

package db

import (
	"database/sql"
	"fmt"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"

	gorp "gopkg.in/gorp.v2"

	"github.com/majewsky/sqlproxy"
	"github.com/sapcc/limes/pkg/util"
	//enable postgres driver for database/sql
	_ "github.com/lib/pq"
)

//DB holds the main database connection. It will be `nil` until InitDatabase() is called.
var DB *gorp.DbMap

func init() {
	sql.Register("postgres-debug", &sqlproxy.Driver{
		ProxiedDriverName: "postgres",
		BeforeQueryHook:   traceQuery,
	})
	//this driver only used within unit tests
	sql.Register("sqlite3-debug", &sqlproxy.Driver{
		ProxiedDriverName: "sqlite3-limes", //this driver is defined in pkg/test/db.go
		BeforeQueryHook:   traceQuery,
		BeforePrepareHook: func(query string) (string, error) {
			//rewrite Postgres function name into SQLite function name
			query = regexp.MustCompile(`\bGREATEST\b`).ReplaceAllString(query, "MAX")
			//Postgres is okay with a no-op "WHERE TRUE" clause, but SQLite does not know the TRUE literal
			query = regexp.MustCompile(`\bWHERE TRUE\s*(GROUP|LIMIT|ORDER|$)`).ReplaceAllString(query, "$1")
			query = regexp.MustCompile(`\bWHERE TRUE AND\b`).ReplaceAllString(query, "WHERE")
			// traceQuery(query, []interface{}{"PREPARE"})
			return query, nil
		},
	})
}

//Configuration is the section of the global configuration file that
//contains the data about
type Configuration struct {
	Location       string `yaml:"location"`
	MigrationsPath string `yaml:"migrations"`
}

//Init initializes the connection to the database.
func Init(cfg Configuration) error {
	sqlDriver := "postgres"
	if os.Getenv("LIMES_DEBUG_SQL") == "1" {
		util.LogInfo("Enabling SQL tracing... \x1B[1;31mTHIS VOIDS YOUR WARRANTY!\x1B[0m If database queries fail in unexpected ways, check first if the tracing causes the issue.")
		sqlDriver += "-debug"
	}

	db, err := sql.Open(sqlDriver, cfg.Location)
	if err != nil {
		return err
	}
	DB = &gorp.DbMap{Db: db, Dialect: gorp.PostgresDialect{}}
	InitGorp()

	//wait for database to reach our expected migration level (this is useful
	//because, depending on the rollout strategy, `limes-migrate` might still be
	//running when we are starting, so wait for it to complete)
	migrationLevel, err := getCurrentMigrationLevel(cfg)
	util.LogDebug("waiting for database to migrate to schema version %d", migrationLevel)
	if err != nil {
		return err
	}
	stmt, err := DB.Prepare(fmt.Sprintf("SELECT 1 FROM schema_migrations WHERE version = %d", migrationLevel))
	if err != nil {
		return err
	}
	defer stmt.Close()

	waitInterval := 1
	for {
		rows, err := stmt.Query()
		if err != nil {
			return err
		}
		if rows.Next() {
			//got a row - success
			break
		}
		//did not get a row - expected migration not there -> sleep with exponential backoff
		waitInterval *= 2
		util.LogInfo("database is not migrated to schema version %d yet - will retry in %d seconds", migrationLevel, waitInterval)
		time.Sleep(time.Duration(waitInterval) * time.Second)
	}

	util.LogDebug("database is migrated - commencing normal startup...")
	return nil
}

func getCurrentMigrationLevel(cfg Configuration) (int, error) {
	//list files in migration directory
	dir, err := os.Open(cfg.MigrationsPath)
	if err != nil {
		return 0, err
	}
	fileNames, err := dir.Readdirnames(-1)
	if err != nil {
		return 0, err
	}

	result := 0
	rx := regexp.MustCompile(`^([0-9]+)_.*\.(?:up|down)\.sql`)
	//find the relevant SQL files and extract their migration numbers
	for _, fileName := range fileNames {
		match := rx.FindStringSubmatch(fileName)
		if match != nil {
			migration, _ := strconv.Atoi(match[1])
			if migration > result {
				result = migration
			}
		}
	}

	return result, nil
}

var sqlWhitespaceRx = regexp.MustCompile(`(?:\s|--.*)+`) // `.*` matches until end of line!

func traceQuery(query string, args []interface{}) {
	//simplify query string - remove comments and reduce whitespace
	//(This logic assumes that there are no arbitrary strings in the SQL
	//statement, which is okay since values should be given as args anyway.)
	query = strings.TrimSpace(sqlWhitespaceRx.ReplaceAllString(query, " "))

	//early exit for easy option
	if len(args) == 0 {
		util.LogDebug(query)
		return
	}

	//if args contains time.Time objects, pretty-print these; use
	//fmt.Sprintf("%#v") for all other types of values
	argStrings := make([]string, len(args))
	for idx, argument := range args {
		switch arg := argument.(type) {
		case time.Time:
			argStrings[idx] = "time.Time [" + arg.Local().String() + "]"
		default:
			argStrings[idx] = fmt.Sprintf("%#v", arg)
		}
	}
	util.LogDebug(query + " [" + strings.Join(argStrings, ", ") + "]")
}

//RollbackUnlessCommitted calls Rollback() on a transaction if it hasn't been
//committed or rolled back yet. Use this with the defer keyword to make sure
//that a transaction is automatically rolled back when a function fails.
func RollbackUnlessCommitted(tx *gorp.Transaction) {
	err := tx.Rollback()
	switch err {
	case nil:
		//rolled back successfully
		util.LogInfo("implicit rollback done")
		return
	case sql.ErrTxDone:
		//already committed or rolled back - nothing to do
		return
	default:
		util.LogError("implicit rollback failed: %s", err.Error())
	}
}

//Interface provides the common methods that both SQL connections and
//transactions implement.
type Interface interface {
	Exec(query string, args ...interface{}) (sql.Result, error)
	Prepare(query string) (*sql.Stmt, error)
	Query(query string, args ...interface{}) (*sql.Rows, error)
	QueryRow(query string, args ...interface{}) *sql.Row
}
<|endoftext|>


<commit_before>/*
Copyright The Helm Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package engine

import (
	"fmt"
	"log"
	"path"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"text/template"

	"github.com/pkg/errors"

	"helm.sh/helm/pkg/chart"
	"helm.sh/helm/pkg/chartutil"
)

// Engine is an implementation of 'cmd/tiller/environment'.Engine that uses Go templates.
type Engine struct {
	// If strict is enabled, template rendering will fail if a template references
	// a value that was not passed in.
	Strict bool
	// In LintMode, some 'required' template values may be missing, so don't fail
	LintMode bool
}

// Render takes a chart, optional values, and value overrides, and attempts to render the Go templates.
//
// Render can be called repeatedly on the same engine.
//
// This will look in the chart's 'templates' data (e.g. the 'templates/' directory)
// and attempt to render the templates there using the values passed in.
//
// Values are scoped to their templates. A dependency template will not have
// access to the values set for its parent. If chart "foo" includes chart "bar",
// "bar" will not have access to the values for "foo".
//
// Values should be prepared with something like `chartutils.ReadValues`.
//
// Values are passed through the templates according to scope. If the top layer
// chart includes the chart foo, which includes the chart bar, the values map
// will be examined for a table called "foo". If "foo" is found in vals,
// that section of the values will be passed into the "foo" chart. And if that
// section contains a value named "bar", that value will be passed on to the
// bar chart during render time.
func (e Engine) Render(chrt *chart.Chart, values chartutil.Values) (map[string]string, error) {
	tmap := allTemplates(chrt, values)
	return e.render(tmap)
}

// Render takes a chart, optional values, and value overrides, and attempts to
// render the Go templates using the default options.
func Render(chrt *chart.Chart, values chartutil.Values) (map[string]string, error) {
	return new(Engine).Render(chrt, values)
}

// renderable is an object that can be rendered.
type renderable struct {
	// tpl is the current template.
	tpl string
	// vals are the values to be supplied to the template.
	vals chartutil.Values
	// namespace prefix to the templates of the current chart
	basePath string
}

const warnStartDelim = "HELM_ERR_START"
const warnEndDelim = "HELM_ERR_END"

var warnRegex = regexp.MustCompile(warnStartDelim + `(.*)` + warnEndDelim)

func warnWrap(warn string) string {
	return warnStartDelim + warn + warnEndDelim
}

// initFunMap creates the Engine's FuncMap and adds context-specific functions.
func (e Engine) initFunMap(t *template.Template, referenceTpls map[string]renderable) {
	funcMap := funcMap()

	// Add the 'include' function here so we can close over t.
	funcMap["include"] = func(name string, data interface{}) (string, error) {
		var buf strings.Builder
		err := t.ExecuteTemplate(&buf, name, data)
		return buf.String(), err
	}

	// Add the 'tpl' function here
	funcMap["tpl"] = func(tpl string, vals chartutil.Values) (string, error) {
		basePath, err := vals.PathValue("Template.BasePath")
		if err != nil {
			return "", errors.Wrapf(err, "cannot retrieve Template.Basepath from values inside tpl function: %s", tpl)
		}

		templateName, err := vals.PathValue("Template.Name")
		if err != nil {
			return "", errors.Wrapf(err, "cannot retrieve Template.Name from values inside tpl function: %s", tpl)
		}

		templates := map[string]renderable{
			templateName.(string): {
				tpl:      tpl,
				vals:     vals,
				basePath: basePath.(string),
			},
		}

		result, err := e.renderWithReferences(templates, referenceTpls)
		if err != nil {
			return "", errors.Wrapf(err, "error during tpl function execution for %q", tpl)
		}
		return result[templateName.(string)], nil
	}

	// Add the `required` function here so we can use lintMode
	funcMap["required"] = func(warn string, val interface{}) (interface{}, error) {
		if val == nil {
			if e.LintMode {
				// Don't fail on missing required values when linting
				log.Printf("[INFO] Missing required value: %s", warn)
				return "", nil
			}
			return val, errors.Errorf(warnWrap(warn))
		} else if _, ok := val.(string); ok {
			if val == "" {
				if e.LintMode {
					// Don't fail on missing required values when linting
					log.Printf("[INFO] Missing required value: %s", warn)
					return "", nil
				}
				return val, errors.Errorf(warnWrap(warn))
			}
		}
		return val, nil
	}

	t.Funcs(funcMap)
}

// render takes a map of templates/values and renders them.
func (e Engine) render(tpls map[string]renderable) (map[string]string, error) {
	return e.renderWithReferences(tpls, tpls)
}

// renderWithReferences takes a map of templates/values to render, and a map of
// templates which can be referenced within them.
func (e Engine) renderWithReferences(tpls, referenceTpls map[string]renderable) (rendered map[string]string, err error) {
	// Basically, what we do here is start with an empty parent template and then
	// build up a list of templates -- one for each file. Once all of the templates
	// have been parsed, we loop through again and execute every template.
	//
	// The idea with this process is to make it possible for more complex templates
	// to share common blocks, but to make the entire thing feel like a file-based
	// template engine.
	defer func() {
		if r := recover(); r != nil {
			err = errors.Errorf("rendering template failed: %v", r)
		}
	}()
	t := template.New("gotpl")
	if e.Strict {
		t.Option("missingkey=error")
	} else {
		// Not that zero will attempt to add default values for types it knows,
		// but will still emit <no value> for others. We mitigate that later.
		t.Option("missingkey=zero")
	}

	e.initFunMap(t, referenceTpls)

	// We want to parse the templates in a predictable order. The order favors
	// higher-level (in file system) templates over deeply nested templates.
	keys := sortTemplates(tpls)

	for _, filename := range keys {
		r := tpls[filename]
		if _, err := t.New(filename).Parse(r.tpl); err != nil {
			return map[string]string{}, cleanupParseError(filename, err)
		}
	}

	// Adding the reference templates to the template context
	// so they can be referenced in the tpl function
	for filename, r := range referenceTpls {
		if t.Lookup(filename) == nil {
			if _, err := t.New(filename).Parse(r.tpl); err != nil {
				return map[string]string{}, cleanupParseError(filename, err)
			}
		}
	}

	rendered = make(map[string]string, len(keys))
	for _, filename := range keys {
		// Don't render partials. We don't care out the direct output of partials.
		// They are only included from other templates.
		if strings.HasPrefix(path.Base(filename), "_") {
			continue
		}
		// At render time, add information about the template that is being rendered.
		vals := tpls[filename].vals
		vals["Template"] = chartutil.Values{"Name": filename, "BasePath": tpls[filename].basePath}
		var buf strings.Builder
		if err := t.ExecuteTemplate(&buf, filename, vals); err != nil {
			return map[string]string{}, cleanupExecError(filename, err)
		}

		// Work around the issue where Go will emit "<no value>" even if Options(missing=zero)
		// is set. Since missing=error will never get here, we do not need to handle
		// the Strict case.
		f := &chart.File{
			Name: strings.ReplaceAll(filename, "/templates", "/manifests"),
			Data: []byte(strings.ReplaceAll(buf.String(), "<no value>", "")),
		}
		rendered[filename] = string(f.Data)
	}

	return rendered, nil
}

func cleanupParseError(filename string, err error) error {
	tokens := strings.Split(err.Error(), ": ")
	if len(tokens) == 1 {
		// This might happen if a non-templating error occurs
		return fmt.Errorf("parse error in (%s): %s", filename, err)
	}
	// The first token is "template"
	// The second token is either "filename:lineno" or "filename:lineNo:columnNo"
	location := tokens[1]
	// The remaining tokens make up a stacktrace-like chain, ending with the relevant error
	errMsg := tokens[len(tokens)-1]
	return fmt.Errorf("parse error at (%s): %s", string(location), errMsg)
}

func cleanupExecError(filename string, err error) error {
	if _, isExecError := err.(template.ExecError); !isExecError {
		return err
	}

	tokens := strings.SplitN(err.Error(), ": ", 3)
	if len(tokens) != 3 {
		// This might happen if a non-templating error occurs
		return fmt.Errorf("execution error in (%s): %s", filename, err)
	}

	// The first token is "template"
	// The second token is either "filename:lineno" or "filename:lineNo:columnNo"
	location := tokens[1]

	parts := warnRegex.FindStringSubmatch(tokens[2])
	if len(parts) >= 2 {
		return fmt.Errorf("execution error at (%s): %s", string(location), parts[1])
	}

	return err
}

func sortTemplates(tpls map[string]renderable) []string {
	keys := make([]string, len(tpls))
	i := 0
	for key := range tpls {
		keys[i] = key
		i++
	}
	sort.Sort(sort.Reverse(byPathLen(keys)))
	return keys
}

type byPathLen []string

func (p byPathLen) Len() int      { return len(p) }
func (p byPathLen) Swap(i, j int) { p[j], p[i] = p[i], p[j] }
func (p byPathLen) Less(i, j int) bool {
	a, b := p[i], p[j]
	ca, cb := strings.Count(a, "/"), strings.Count(b, "/")
	if ca == cb {
		return strings.Compare(a, b) == -1
	}
	return ca < cb
}

// allTemplates returns all templates for a chart and its dependencies.
//
// As it goes, it also prepares the values in a scope-sensitive manner.
func allTemplates(c *chart.Chart, vals chartutil.Values) map[string]renderable {
	templates := make(map[string]renderable)
	recAllTpls(c, templates, vals)
	return templates
}

// recAllTpls recurses through the templates in a chart.
//
// As it recurses, it also sets the values to be appropriate for the template
// scope.
func recAllTpls(c *chart.Chart, templates map[string]renderable, vals chartutil.Values) {
	next := map[string]interface{}{
		"Chart":        c.Metadata,
		"Files":        newFiles(c.Files),
		"Release":      vals["Release"],
		"Capabilities": vals["Capabilities"],
		"Values":       make(chartutil.Values),
	}

	// If there is a {{.Values.ThisChart}} in the parent metadata,
	// copy that into the {{.Values}} for this template.
	if c.IsRoot() {
		next["Values"] = vals["Values"]
	} else if vs, err := vals.Table("Values." + c.Name()); err == nil {
		next["Values"] = vs
	}

	for _, child := range c.Dependencies() {
		recAllTpls(child, templates, next)
	}

	newParentID := c.ChartFullPath()
	for _, t := range c.Templates {
		if !isTemplateValid(c, t.Name) {
			continue
		}
		templates[path.Join(newParentID, t.Name)] = renderable{
			tpl:      string(t.Data),
			vals:     next,
			basePath: path.Join(newParentID, "templates"),
		}
	}
}

// isTemplateValid returns true if the template is valid for the chart type
func isTemplateValid(ch *chart.Chart, templateName string) bool {
	if isLibraryChart(ch) {
		return strings.HasPrefix(filepath.Base(templateName), "_")
	}
	return true
}

// isLibraryChart returns true if the chart is a library chart
func isLibraryChart(c *chart.Chart) bool {
	return strings.EqualFold(c.Metadata.Type, "library")
}
<commit_msg>ref(pkg/engine): cleanup of development `hack`<commit_after>/*
Copyright The Helm Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package engine

import (
	"fmt"
	"log"
	"path"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"text/template"

	"github.com/pkg/errors"

	"helm.sh/helm/pkg/chart"
	"helm.sh/helm/pkg/chartutil"
)

// Engine is an implementation of 'cmd/tiller/environment'.Engine that uses Go templates.
type Engine struct {
	// If strict is enabled, template rendering will fail if a template references
	// a value that was not passed in.
	Strict bool
	// In LintMode, some 'required' template values may be missing, so don't fail
	LintMode bool
}

// Render takes a chart, optional values, and value overrides, and attempts to render the Go templates.
//
// Render can be called repeatedly on the same engine.
//
// This will look in the chart's 'templates' data (e.g. the 'templates/' directory)
// and attempt to render the templates there using the values passed in.
//
// Values are scoped to their templates. A dependency template will not have
// access to the values set for its parent. If chart "foo" includes chart "bar",
// "bar" will not have access to the values for "foo".
//
// Values should be prepared with something like `chartutils.ReadValues`.
//
// Values are passed through the templates according to scope. If the top layer
// chart includes the chart foo, which includes the chart bar, the values map
// will be examined for a table called "foo". If "foo" is found in vals,
// that section of the values will be passed into the "foo" chart. And if that
// section contains a value named "bar", that value will be passed on to the
// bar chart during render time.
func (e Engine) Render(chrt *chart.Chart, values chartutil.Values) (map[string]string, error) {
	tmap := allTemplates(chrt, values)
	return e.render(tmap)
}

// Render takes a chart, optional values, and value overrides, and attempts to
// render the Go templates using the default options.
func Render(chrt *chart.Chart, values chartutil.Values) (map[string]string, error) {
	return new(Engine).Render(chrt, values)
}

// renderable is an object that can be rendered.
type renderable struct {
	// tpl is the current template.
	tpl string
	// vals are the values to be supplied to the template.
	vals chartutil.Values
	// namespace prefix to the templates of the current chart
	basePath string
}

const warnStartDelim = "HELM_ERR_START"
const warnEndDelim = "HELM_ERR_END"

var warnRegex = regexp.MustCompile(warnStartDelim + `(.*)` + warnEndDelim)

func warnWrap(warn string) string {
	return warnStartDelim + warn + warnEndDelim
}

// initFunMap creates the Engine's FuncMap and adds context-specific functions.
func (e Engine) initFunMap(t *template.Template, referenceTpls map[string]renderable) {
	funcMap := funcMap()

	// Add the 'include' function here so we can close over t.
	funcMap["include"] = func(name string, data interface{}) (string, error) {
		var buf strings.Builder
		err := t.ExecuteTemplate(&buf, name, data)
		return buf.String(), err
	}

	// Add the 'tpl' function here
	funcMap["tpl"] = func(tpl string, vals chartutil.Values) (string, error) {
		basePath, err := vals.PathValue("Template.BasePath")
		if err != nil {
			return "", errors.Wrapf(err, "cannot retrieve Template.Basepath from values inside tpl function: %s", tpl)
		}

		templateName, err := vals.PathValue("Template.Name")
		if err != nil {
			return "", errors.Wrapf(err, "cannot retrieve Template.Name from values inside tpl function: %s", tpl)
		}

		templates := map[string]renderable{
			templateName.(string): {
				tpl:      tpl,
				vals:     vals,
				basePath: basePath.(string),
			},
		}

		result, err := e.renderWithReferences(templates, referenceTpls)
		if err != nil {
			return "", errors.Wrapf(err, "error during tpl function execution for %q", tpl)
		}
		return result[templateName.(string)], nil
	}

	// Add the `required` function here so we can use lintMode
	funcMap["required"] = func(warn string, val interface{}) (interface{}, error) {
		if val == nil {
			if e.LintMode {
				// Don't fail on missing required values when linting
				log.Printf("[INFO] Missing required value: %s", warn)
				return "", nil
			}
			return val, errors.Errorf(warnWrap(warn))
		} else if _, ok := val.(string); ok {
			if val == "" {
				if e.LintMode {
					// Don't fail on missing required values when linting
					log.Printf("[INFO] Missing required value: %s", warn)
					return "", nil
				}
				return val, errors.Errorf(warnWrap(warn))
			}
		}
		return val, nil
	}

	t.Funcs(funcMap)
}

// render takes a map of templates/values and renders them.
func (e Engine) render(tpls map[string]renderable) (map[string]string, error) {
	return e.renderWithReferences(tpls, tpls)
}

// renderWithReferences takes a map of templates/values to render, and a map of
// templates which can be referenced within them.
func (e Engine) renderWithReferences(tpls, referenceTpls map[string]renderable) (rendered map[string]string, err error) {
	// Basically, what we do here is start with an empty parent template and then
	// build up a list of templates -- one for each file. Once all of the templates
	// have been parsed, we loop through again and execute every template.
	//
	// The idea with this process is to make it possible for more complex templates
	// to share common blocks, but to make the entire thing feel like a file-based
	// template engine.
	defer func() {
		if r := recover(); r != nil {
			err = errors.Errorf("rendering template failed: %v", r)
		}
	}()
	t := template.New("gotpl")
	if e.Strict {
		t.Option("missingkey=error")
	} else {
		// Not that zero will attempt to add default values for types it knows,
		// but will still emit <no value> for others. We mitigate that later.
		t.Option("missingkey=zero")
	}

	e.initFunMap(t, referenceTpls)

	// We want to parse the templates in a predictable order. The order favors
	// higher-level (in file system) templates over deeply nested templates.
	keys := sortTemplates(tpls)

	for _, filename := range keys {
		r := tpls[filename]
		if _, err := t.New(filename).Parse(r.tpl); err != nil {
			return map[string]string{}, cleanupParseError(filename, err)
		}
	}

	// Adding the reference templates to the template context
	// so they can be referenced in the tpl function
	for filename, r := range referenceTpls {
		if t.Lookup(filename) == nil {
			if _, err := t.New(filename).Parse(r.tpl); err != nil {
				return map[string]string{}, cleanupParseError(filename, err)
			}
		}
	}

	rendered = make(map[string]string, len(keys))
	for _, filename := range keys {
		// Don't render partials. We don't care out the direct output of partials.
		// They are only included from other templates.
		if strings.HasPrefix(path.Base(filename), "_") {
			continue
		}
		// At render time, add information about the template that is being rendered.
		vals := tpls[filename].vals
		vals["Template"] = chartutil.Values{"Name": filename, "BasePath": tpls[filename].basePath}
		var buf strings.Builder
		if err := t.ExecuteTemplate(&buf, filename, vals); err != nil {
			return map[string]string{}, cleanupExecError(filename, err)
		}

		// Work around the issue where Go will emit "<no value>" even if Options(missing=zero)
		// is set. Since missing=error will never get here, we do not need to handle
		// the Strict case.
		rendered[filename] = strings.ReplaceAll(buf.String(), "<no value>", "")
	}

	return rendered, nil
}

func cleanupParseError(filename string, err error) error {
	tokens := strings.Split(err.Error(), ": ")
	if len(tokens) == 1 {
		// This might happen if a non-templating error occurs
		return fmt.Errorf("parse error in (%s): %s", filename, err)
	}
	// The first token is "template"
	// The second token is either "filename:lineno" or "filename:lineNo:columnNo"
	location := tokens[1]
	// The remaining tokens make up a stacktrace-like chain, ending with the relevant error
	errMsg := tokens[len(tokens)-1]
	return fmt.Errorf("parse error at (%s): %s", string(location), errMsg)
}

func cleanupExecError(filename string, err error) error {
	if _, isExecError := err.(template.ExecError); !isExecError {
		return err
	}

	tokens := strings.SplitN(err.Error(), ": ", 3)
	if len(tokens) != 3 {
		// This might happen if a non-templating error occurs
		return fmt.Errorf("execution error in (%s): %s", filename, err)
	}

	// The first token is "template"
	// The second token is either "filename:lineno" or "filename:lineNo:columnNo"
	location := tokens[1]

	parts := warnRegex.FindStringSubmatch(tokens[2])
	if len(parts) >= 2 {
		return fmt.Errorf("execution error at (%s): %s", string(location), parts[1])
	}

	return err
}

func sortTemplates(tpls map[string]renderable) []string {
	keys := make([]string, len(tpls))
	i := 0
	for key := range tpls {
		keys[i] = key
		i++
	}
	sort.Sort(sort.Reverse(byPathLen(keys)))
	return keys
}

type byPathLen []string

func (p byPathLen) Len() int      { return len(p) }
func (p byPathLen) Swap(i, j int) { p[j], p[i] = p[i], p[j] }
func (p byPathLen) Less(i, j int) bool {
	a, b := p[i], p[j]
	ca, cb := strings.Count(a, "/"), strings.Count(b, "/")
	if ca == cb {
		return strings.Compare(a, b) == -1
	}
	return ca < cb
}

// allTemplates returns all templates for a chart and its dependencies.
//
// As it goes, it also prepares the values in a scope-sensitive manner.
func allTemplates(c *chart.Chart, vals chartutil.Values) map[string]renderable {
	templates := make(map[string]renderable)
	recAllTpls(c, templates, vals)
	return templates
}

// recAllTpls recurses through the templates in a chart.
//
// As it recurses, it also sets the values to be appropriate for the template
// scope.
func recAllTpls(c *chart.Chart, templates map[string]renderable, vals chartutil.Values) {
	next := map[string]interface{}{
		"Chart":        c.Metadata,
		"Files":        newFiles(c.Files),
		"Release":      vals["Release"],
		"Capabilities": vals["Capabilities"],
		"Values":       make(chartutil.Values),
	}

	// If there is a {{.Values.ThisChart}} in the parent metadata,
	// copy that into the {{.Values}} for this template.
	if c.IsRoot() {
		next["Values"] = vals["Values"]
	} else if vs, err := vals.Table("Values." + c.Name()); err == nil {
		next["Values"] = vs
	}

	for _, child := range c.Dependencies() {
		recAllTpls(child, templates, next)
	}

	newParentID := c.ChartFullPath()
	for _, t := range c.Templates {
		if !isTemplateValid(c, t.Name) {
			continue
		}
		templates[path.Join(newParentID, t.Name)] = renderable{
			tpl:      string(t.Data),
			vals:     next,
			basePath: path.Join(newParentID, "templates"),
		}
	}
}

// isTemplateValid returns true if the template is valid for the chart type
func isTemplateValid(ch *chart.Chart, templateName string) bool {
	if isLibraryChart(ch) {
		return strings.HasPrefix(filepath.Base(templateName), "_")
	}
	return true
}

// isLibraryChart returns true if the chart is a library chart
func isLibraryChart(c *chart.Chart) bool {
	return strings.EqualFold(c.Metadata.Type, "library")
}
<|endoftext|>


<commit_before>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

//go:generate bash -c "echo -en '// AUTOGENERATED FILE\n\n' > generated.go"
//go:generate bash -c "echo -en 'package kernel\n\n' >> generated.go"
//go:generate bash -c "echo -en 'const createImageScript = `#!/bin/bash\n' >> generated.go"
//go:generate bash -c "cat ../../tools/create-gce-image.sh | grep -v '#' >> generated.go"
//go:generate bash -c "echo -en '`\n\n' >> generated.go"

// Package kernel contains helper functions for working with Linux kernel
// (building kernel/image).
package kernel

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/google/syzkaller/pkg/osutil"
)

func Build(dir, compiler, config string) error {
	if err := osutil.CopyFile(config, filepath.Join(dir, ".config")); err != nil {
		return fmt.Errorf("failed to write config file: %v", err)
	}
	return build(dir, compiler)
}

// TODO(dvyukov): this is only for syz-gce, remove when syz-gce is deleted.
func BuildWithPartConfig(dir, compiler, config string) error {
	const timeout = 10 * time.Minute // default timeout for command invocations
	os.Remove(filepath.Join(dir, ".config"))
	configFile := filepath.Join(dir, "syz.config")
	if err := osutil.WriteFile(configFile, []byte(config)); err != nil {
		return fmt.Errorf("failed to write config file: %v", err)
	}
	defer os.Remove(configFile)
	if _, err := osutil.RunCmd(timeout, dir, "make", "defconfig"); err != nil {
		return err
	}
	if _, err := osutil.RunCmd(timeout, dir, "make", "kvmconfig"); err != nil {
		return err
	}
	if _, err := osutil.RunCmd(timeout, dir, "scripts/kconfig/merge_config.sh", "-n", ".config", configFile); err != nil {
		return err
	}
	return build(dir, compiler)
}

func build(dir, compiler string) error {
	const timeout = 10 * time.Minute // default timeout for command invocations
	if _, err := osutil.RunCmd(timeout, dir, "make", "olddefconfig"); err != nil {
		return err
	}
	// We build only bzImage as we currently don't use modules.
	// Build of a large kernel can take a while on a 1 CPU VM.
	cpu := strconv.Itoa(runtime.NumCPU())
	if _, err := osutil.RunCmd(3*time.Hour, dir, "make", "bzImage", "-j", cpu, "CC="+compiler); err != nil {
		return err
	}
	return nil
}

// CreateImage creates a disk image that is suitable for syzkaller.
// Kernel is taken from kernelDir, userspace system is taken from userspaceDir.
// If cmdlineFile is not empty, contents of the file are appended to the kernel command line.
// If sysctlFile is not empty, contents of the file are appended to the image /etc/sysctl.conf.
// Produces image and root ssh key in the specified files.
func CreateImage(kernelDir, userspaceDir, cmdlineFile, sysctlFile, image, sshkey string) error {
	tempDir, err := ioutil.TempDir("", "syz-build")
	if err != nil {
		return err
	}
	defer os.RemoveAll(tempDir)
	scriptFile := filepath.Join(tempDir, "create.sh")
	if err := osutil.WriteExecFile(scriptFile, []byte(createImageScript)); err != nil {
		return fmt.Errorf("failed to write script file: %v", err)
	}
	bzImage := filepath.Join(kernelDir, filepath.FromSlash("arch/x86/boot/bzImage"))
	env := []string{
		"SYZ_CMDLINE_FILE=" + osutil.Abs(cmdlineFile),
		"SYZ_SYSCTL_FILE=" + osutil.Abs(sysctlFile),
	}
	_, err = osutil.RunCmdEnv(time.Hour, env, tempDir, scriptFile, userspaceDir, bzImage)
	if err != nil {
		return fmt.Errorf("image build failed: %v", err)
	}
	if err := osutil.CopyFile(filepath.Join(tempDir, "disk.raw"), image); err != nil {
		return err
	}
	if err := osutil.CopyFile(filepath.Join(tempDir, "key"), sshkey); err != nil {
		return err
	}
	if err := os.Chmod(sshkey, 0600); err != nil {
		return err
	}
	return nil
}

func CompilerIdentity(compiler string) (string, error) {
	output, err := osutil.RunCmd(time.Minute, "", compiler, "--version")
	if err != nil {
		return "", err
	}
	if len(output) == 0 {
		return "", fmt.Errorf("no output from compiler --version")
	}
	return strings.Split(string(output), "\n")[0], nil
}
<commit_msg>pkg/kernel: remove unused function<commit_after>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

//go:generate bash -c "echo -en '// AUTOGENERATED FILE\n\n' > generated.go"
//go:generate bash -c "echo -en 'package kernel\n\n' >> generated.go"
//go:generate bash -c "echo -en 'const createImageScript = `#!/bin/bash\n' >> generated.go"
//go:generate bash -c "cat ../../tools/create-gce-image.sh | grep -v '#' >> generated.go"
//go:generate bash -c "echo -en '`\n\n' >> generated.go"

// Package kernel contains helper functions for working with Linux kernel
// (building kernel/image).
package kernel

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/google/syzkaller/pkg/osutil"
)

func Build(dir, compiler, config string) error {
	if err := osutil.CopyFile(config, filepath.Join(dir, ".config")); err != nil {
		return fmt.Errorf("failed to write config file: %v", err)
	}
	return build(dir, compiler)
}

func build(dir, compiler string) error {
	const timeout = 10 * time.Minute // default timeout for command invocations
	if _, err := osutil.RunCmd(timeout, dir, "make", "olddefconfig"); err != nil {
		return err
	}
	// We build only bzImage as we currently don't use modules.
	// Build of a large kernel can take a while on a 1 CPU VM.
	cpu := strconv.Itoa(runtime.NumCPU())
	if _, err := osutil.RunCmd(3*time.Hour, dir, "make", "bzImage", "-j", cpu, "CC="+compiler); err != nil {
		return err
	}
	return nil
}

// CreateImage creates a disk image that is suitable for syzkaller.
// Kernel is taken from kernelDir, userspace system is taken from userspaceDir.
// If cmdlineFile is not empty, contents of the file are appended to the kernel command line.
// If sysctlFile is not empty, contents of the file are appended to the image /etc/sysctl.conf.
// Produces image and root ssh key in the specified files.
func CreateImage(kernelDir, userspaceDir, cmdlineFile, sysctlFile, image, sshkey string) error {
	tempDir, err := ioutil.TempDir("", "syz-build")
	if err != nil {
		return err
	}
	defer os.RemoveAll(tempDir)
	scriptFile := filepath.Join(tempDir, "create.sh")
	if err := osutil.WriteExecFile(scriptFile, []byte(createImageScript)); err != nil {
		return fmt.Errorf("failed to write script file: %v", err)
	}
	bzImage := filepath.Join(kernelDir, filepath.FromSlash("arch/x86/boot/bzImage"))
	env := []string{
		"SYZ_CMDLINE_FILE=" + osutil.Abs(cmdlineFile),
		"SYZ_SYSCTL_FILE=" + osutil.Abs(sysctlFile),
	}
	_, err = osutil.RunCmdEnv(time.Hour, env, tempDir, scriptFile, userspaceDir, bzImage)
	if err != nil {
		return fmt.Errorf("image build failed: %v", err)
	}
	if err := osutil.CopyFile(filepath.Join(tempDir, "disk.raw"), image); err != nil {
		return err
	}
	if err := osutil.CopyFile(filepath.Join(tempDir, "key"), sshkey); err != nil {
		return err
	}
	if err := os.Chmod(sshkey, 0600); err != nil {
		return err
	}
	return nil
}

func CompilerIdentity(compiler string) (string, error) {
	output, err := osutil.RunCmd(time.Minute, "", compiler, "--version")
	if err != nil {
		return "", err
	}
	if len(output) == 0 {
		return "", fmt.Errorf("no output from compiler --version")
	}
	return strings.Split(string(output), "\n")[0], nil
}
<|endoftext|>


<commit_before>package paxos

import (
	"log"
	"os"

	"junta/store"
	"junta/util"
)

type result struct {
	seqn uint64
	v    string
}

type instReq struct {
	seqn uint64 // 0 means to fill in a fresh seqn
	ch   chan *instance
}

type Manager struct {
	st      *store.Store
	rg      *Registrar
	learned chan result
	seqns   chan uint64
	reqs    chan *instReq
	logger  *log.Logger
	Self    string
	alpha   int
	outs    PutterTo
}

func NewManager(self string, start uint64, alpha int, st *store.Store, outs PutterTo) *Manager {
	m := &Manager{
		st:      st,
		rg:      NewRegistrar(st, start, alpha),
		learned: make(chan result),
		seqns:   make(chan uint64),
		reqs:    make(chan *instReq),
		logger:  util.NewLogger("manager"),
		Self:    self,
		alpha:   alpha,
		outs:    outs,
	}

	go m.gen(start+uint64(alpha))
	go m.process()

	return m
}

func (m *Manager) Alpha() int {
	return m.alpha
}

func (m *Manager) cluster(seqn uint64) *cluster {
	members, cals := m.rg.setsForSeqn(seqn)
	return newCluster(m.Self, members, cals, putToWrapper{seqn, m.outs})
}

func (mg *Manager) gen(next uint64) {
	for {
		mg.seqns <- next
		next++
	}
}

func (m *Manager) process() {
	instances := make(map[uint64]*instance)
	for req := range m.reqs {
		inst, ok := instances[req.seqn]
		if !ok {
			inst = newInstance(req.seqn, m)
			instances[req.seqn] = inst
			go func(seqn uint64, it *instance) {
				m.learned <- result{seqn, it.Value()}
			}(req.seqn, inst)
		}
		req.ch <- inst
	}
}

func (m *Manager) getInstance(seqn uint64) *instance {
	r := &instReq{seqn, make(chan *instance)}
	m.reqs <- r
	it := <-r.ch
	return it
}

func (m *Manager) PutFrom(addr string, msg Msg) {
	if !msg.Ok() {
		return
	}
	it := m.getInstance(msg.Seqn())
	it.PutFrom(addr, msg)
}

func (m *Manager) Propose(v string) (uint64, string, os.Error) {
	ch := make(chan store.Event)
	seqn := <-m.seqns
	inst := m.getInstance(seqn)
	m.st.Wait(seqn, ch)
	m.logger.Logf("paxos propose -> %q", v)
	inst.Propose(v)
	ev := <-ch
	return seqn, ev.Mut, ev.Err
}

func (m *Manager) Recv() (uint64, string) {
	result := <-m.learned
	m.logger.Logf("paxos %d learned <- %q", result.seqn, result.v)
	return result.seqn, result.v
}
<commit_msg>refactor<commit_after>package paxos

import (
	"log"
	"os"

	"junta/store"
	"junta/util"
)

type result struct {
	seqn uint64
	v    string
}

type instReq struct {
	seqn uint64 // 0 means to fill in a fresh seqn
	ch   chan *instance
}

type Manager struct {
	st      *store.Store
	rg      *Registrar
	learned chan result
	seqns   chan uint64
	reqs    chan instReq
	logger  *log.Logger
	Self    string
	alpha   int
	outs    PutterTo
}

func NewManager(self string, start uint64, alpha int, st *store.Store, outs PutterTo) *Manager {
	m := &Manager{
		st:      st,
		rg:      NewRegistrar(st, start, alpha),
		learned: make(chan result),
		seqns:   make(chan uint64),
		reqs:    make(chan instReq),
		logger:  util.NewLogger("manager"),
		Self:    self,
		alpha:   alpha,
		outs:    outs,
	}

	go m.gen(start+uint64(alpha))
	go m.process()

	return m
}

func (m *Manager) Alpha() int {
	return m.alpha
}

func (m *Manager) cluster(seqn uint64) *cluster {
	members, cals := m.rg.setsForSeqn(seqn)
	return newCluster(m.Self, members, cals, putToWrapper{seqn, m.outs})
}

func (mg *Manager) gen(next uint64) {
	for {
		mg.seqns <- next
		next++
	}
}

func (m *Manager) process() {
	instances := make(map[uint64]*instance)
	for req := range m.reqs {
		inst, ok := instances[req.seqn]
		if !ok {
			inst = newInstance(req.seqn, m)
			instances[req.seqn] = inst
			go func(seqn uint64, it *instance) {
				m.learned <- result{seqn, it.Value()}
			}(req.seqn, inst)
		}
		req.ch <- inst
	}
}

func (m *Manager) getInstance(seqn uint64) *instance {
	ch := make(chan *instance)
	m.reqs <- instReq{seqn, ch}
	return <-ch
}

func (m *Manager) PutFrom(addr string, msg Msg) {
	if !msg.Ok() {
		return
	}
	it := m.getInstance(msg.Seqn())
	it.PutFrom(addr, msg)
}

func (m *Manager) Propose(v string) (uint64, string, os.Error) {
	ch := make(chan store.Event)
	seqn := <-m.seqns
	inst := m.getInstance(seqn)
	m.st.Wait(seqn, ch)
	m.logger.Logf("paxos propose -> %q", v)
	inst.Propose(v)
	ev := <-ch
	return seqn, ev.Mut, ev.Err
}

func (m *Manager) Recv() (uint64, string) {
	result := <-m.learned
	m.logger.Logf("paxos %d learned <- %q", result.seqn, result.v)
	return result.seqn, result.v
}
<|endoftext|>


<commit_before>package txtdirect

import (
	"net/http"
	"net/http/httptest"
	"testing"
)

func TestParsePlaceholders(t *testing.T) {
	tests := []struct {
		url       string
		requested string
		expected  string
	}{
		{
			"example.com{uri}",
			"https://example.com/?test=test",
			"example.com/?test=test",
		},
		{
			"example.com{uri}/{~test}",
			"https://example.com/?test=test",
			"example.com/?test=test/test",
		},
		{
			"example.com{uri}/{>Test}",
			"https://example.com/?test=test",
			"example.com/?test=test/test-header",
		},
		{
			"example.com{uri}/{?test}",
			"https://example.com/?test=test",
			"example.com/?test=test/test",
		},
		{
			"example.com{dir}",
			"https://example.com/directory/test",
			"example.com/directory/",
		},
		{
			"example.com/directory/{file}",
			"https://example.com/directory/file.pdf",
			"example.com/directory/file.pdf",
		},
		{
			"example.com/?test={fragment}",
			"https://example.com/?test=#fragment",
			"example.com/?test=fragment",
		},
		{
			"example.com/{host}",
			"https://project.example.com:8080",
			"example.com/project.example.com:8080",
		},
		{
			"example.com/{hostonly}",
			"https://project.example.com",
			"example.com/project.example.com",
		},
		{
			"example.com/{method}",
			"https://example.com",
			"example.com/GET",
		},
		{
			"subdomain.example.com/{label1}",
			"https://subdomain.example.com/subdomain",
			"subdomain.example.com/subdomain",
		},
		{
			"subdomain.example.com/{label2}",
			"https://subdomain.example.com/example",
			"subdomain.example.com/example",
		},
		{
			"subdomain.example.com/{label3}",
			"https://subdomain.example.com/com",
			"subdomain.example.com/com",
		},
	}
	for _, test := range tests {
		req := httptest.NewRequest("GET", test.requested, nil)
		req.AddCookie(&http.Cookie{Name: "test", Value: "test"})
		req.Header.Add("Test", "test-header")
		req.URL.Fragment = "fragment"
		result, err := parsePlaceholders(test.url, req)
		if err != nil {
			t.Fatal(err)
		}
		if result != test.expected {
			t.Errorf("Expected %s, got %s", test.expected, result)
		}
	}
}

func TestParsePlaceholdersFails(t *testing.T) {
	tests := []struct {
		url       string
		requested string
	}{
		{
			"example.com/{label0}",
			"https://example.com/test",
		},
		{
			"example.com/{label9000}",
			"https://example.com/test",
		},
	}
	for _, test := range tests {
		req := httptest.NewRequest("GET", test.requested, nil)
		_, err := parsePlaceholders(test.url, req)
		if err == nil {
			t.Errorf("Expected error, got nil")
		}
	}
}
<commit_msg>Add {path} and {path_escaped} placeholder tests<commit_after>package txtdirect

import (
	"net/http"
	"net/http/httptest"
	"testing"
)

func TestParsePlaceholders(t *testing.T) {
	tests := []struct {
		url       string
		requested string
		expected  string
	}{
		{
			"example.com{uri}",
			"https://example.com/?test=test",
			"example.com/?test=test",
		},
		{
			"example.com{uri}/{~test}",
			"https://example.com/?test=test",
			"example.com/?test=test/test",
		},
		{
			"example.com{uri}/{>Test}",
			"https://example.com/?test=test",
			"example.com/?test=test/test-header",
		},
		{
			"example.com{uri}/{?test}",
			"https://example.com/?test=test",
			"example.com/?test=test/test",
		},
		{
			"example.com{dir}",
			"https://example.com/directory/test",
			"example.com/directory/",
		},
		{
			"example.com/directory/{file}",
			"https://example.com/directory/file.pdf",
			"example.com/directory/file.pdf",
		},
		{
			"example.com/?test={fragment}",
			"https://example.com/?test=#fragment",
			"example.com/?test=fragment",
		},
		{
			"example.com/{host}",
			"https://project.example.com:8080",
			"example.com/project.example.com:8080",
		},
		{
			"example.com/{hostonly}",
			"https://project.example.com",
			"example.com/project.example.com",
		},
		{
			"example.com/{method}",
			"https://example.com",
			"example.com/GET",
		},
		{
			"example.com{path}",
			"https://example.com/path",
			"example.com/path",
		},
		{
			"example.com{path_escaped}",
			"https://example.com/path_escaped",
			"example.com%2Fpath_escaped",
		},
		{
			"subdomain.example.com/{label1}",
			"https://subdomain.example.com/subdomain",
			"subdomain.example.com/subdomain",
		},
		{
			"subdomain.example.com/{label2}",
			"https://subdomain.example.com/example",
			"subdomain.example.com/example",
		},
		{
			"subdomain.example.com/{label3}",
			"https://subdomain.example.com/com",
			"subdomain.example.com/com",
		},
	}
	for _, test := range tests {
		req := httptest.NewRequest("GET", test.requested, nil)
		req.AddCookie(&http.Cookie{Name: "test", Value: "test"})
		req.Header.Add("Test", "test-header")
		req.URL.Fragment = "fragment"
		result, err := parsePlaceholders(test.url, req)
		if err != nil {
			t.Fatal(err)
		}
		if result != test.expected {
			t.Errorf("Expected %s, got %s", test.expected, result)
		}
	}
}

func TestParsePlaceholdersFails(t *testing.T) {
	tests := []struct {
		url       string
		requested string
	}{
		{
			"example.com/{label0}",
			"https://example.com/test",
		},
		{
			"example.com/{label9000}",
			"https://example.com/test",
		},
	}
	for _, test := range tests {
		req := httptest.NewRequest("GET", test.requested, nil)
		_, err := parsePlaceholders(test.url, req)
		if err == nil {
			t.Errorf("Expected error, got nil")
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
	"strings"
	"testing"
	"time"
)

// Should cache first response for an unspecified period of time if when it
// doesn't specify it's own cache headers. Subsequent requests should return
// a cached response.
func TestCacheFirstResponse(t *testing.T) {
	ResetBackends(backendsByPriority)

	testRequestsCachedIndefinite(t, nil)
}

// Should cache responses for the period defined in a `Expires: n` response
// header.
func TestCacheExpires(t *testing.T) {
	ResetBackends(backendsByPriority)

	const cacheDuration = time.Duration(5 * time.Second)

	handler := func(w http.ResponseWriter) {
		headerValue := time.Now().UTC().Add(cacheDuration).Format(http.TimeFormat)
		w.Header().Set("Expires", headerValue)
	}

	testRequestsCachedDuration(t, handler, cacheDuration)
}

// Should cache responses for the period defined in a `Cache-Control:
// max-age=n` response header.
func TestCacheCacheControlMaxAge(t *testing.T) {
	ResetBackends(backendsByPriority)

	const cacheDuration = time.Duration(5 * time.Second)
	headerValue := fmt.Sprintf("max-age=%.0f", cacheDuration.Seconds())

	handler := func(w http.ResponseWriter) {
		w.Header().Set("Cache-Control", headerValue)
	}

	testRequestsCachedDuration(t, handler, cacheDuration)
}

// Should cache responses for the period defined in a `Cache-Control:
// max-age=n` response header when a `Expires: n*2` header is also present.
func TestCacheExpiresAndMaxAge(t *testing.T) {
	ResetBackends(backendsByPriority)

	const cacheDuration = time.Duration(5 * time.Second)
	const expiresDuration = cacheDuration * 2

	maxAgeValue := fmt.Sprintf("max-age=%.0f", cacheDuration.Seconds())

	handler := func(w http.ResponseWriter) {
		expiresValue := time.Now().UTC().Add(expiresDuration).Format(http.TimeFormat)

		w.Header().Set("Expires", expiresValue)
		w.Header().Set("Cache-Control", maxAgeValue)
	}

	testRequestsCachedDuration(t, handler, cacheDuration)
}

// Should cache responses with a status code of 404. It's a common
// misconception that 404 responses shouldn't be cached; they should because
// they can be expensive to generate.
func TestCache404Response(t *testing.T) {
	ResetBackends(backendsByPriority)

	handler := func(w http.ResponseWriter) {
		w.WriteHeader(http.StatusNotFound)
	}

	testRequestsCachedIndefinite(t, handler)
}

// Should cache multiple distinct responses for the same URL when origin responds
// with a `Vary` header and clients provide requests with different values
// for that header.
func TestCacheVary(t *testing.T) {
	ResetBackends(backendsByPriority)

	const reqHeaderName = "CustomThing"
	const respHeaderName = "Reflected-" + reqHeaderName
	headerVals := []string{
		"first distinct",
		"second distinct",
		"third distinct",
	}

	req := NewUniqueEdgeGET(t)

	for _, populateCache := range []bool{true, false} {
		for _, headerVal := range headerVals {
			if populateCache {
				originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
					w.Header().Set("Vary", reqHeaderName)
					w.Header().Set(respHeaderName, r.Header.Get(reqHeaderName))
				})
			} else {
				originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
					t.Error("Request should not have made it to origin")
					w.Header().Set(respHeaderName, "not cached")
				})
			}

			req.Header.Set(reqHeaderName, headerVal)
			resp := RoundTripCheckError(t, req)
			defer resp.Body.Close()

			if recVal := resp.Header.Get(respHeaderName); recVal != headerVal {
				t.Errorf(
					"Request received wrong %q header. Expected %q, got %q",
					respHeaderName,
					headerVal,
					recVal,
				)
			}
		}
	}
}

// Should cache distinct responses for requests with the same path but
// different query params.
func TestCacheUniqueQueryParams(t *testing.T) {
	ResetBackends(backendsByPriority)

	const respHeaderName = "Request-RawQuery"

	req1 := NewUniqueEdgeGET(t)
	req2 := NewUniqueEdgeGET(t)

	if req1.URL.Path != req2.URL.Path {
		t.Fatalf(
			"Request paths do not match. Expected %q, got %q",
			req1.URL.Path,
			req2.URL.Path,
		)
	}
	if req1.URL.RawQuery == req2.URL.RawQuery {
		t.Fatalf(
			"Request query params do not differ. Expected %q != %q",
			req1.URL.RawQuery,
			req2.URL.RawQuery,
		)
	}

	for _, populateCache := range []bool{true, false} {
		for _, req := range []*http.Request{req1, req2} {
			if populateCache {
				originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
					w.Header().Set(respHeaderName, r.URL.RawQuery)
				})
			} else {
				originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
					t.Errorf(
						"Request with query param %q should not have made it to origin",
						r.URL.RawQuery,
					)
				})
			}

			resp := RoundTripCheckError(t, req)
			defer resp.Body.Close()

			if recVal := resp.Header.Get(respHeaderName); recVal != req.URL.RawQuery {
				t.Errorf(
					"Request received wrong %q header. Expected %q, got %q",
					respHeaderName,
					req.URL.RawQuery,
					recVal,
				)
			}
		}
	}
}

// Should cache distinct responses for requests with the same query params
// but paths of different case-sensitivity.
func TestCacheUniqueCaseSensitive(t *testing.T) {
	ResetBackends(backendsByPriority)

	const reqPath = "/CaseSensitive"
	const respHeaderName = "Request-Path"

	req1 := NewUniqueEdgeGET(t)
	req2 := NewUniqueEdgeGET(t)

	req1.URL.Path = strings.ToLower(reqPath)
	req2.URL.Path = strings.ToUpper(reqPath)
	req1.URL.RawQuery = req2.URL.RawQuery

	if req1.URL.Path == req2.URL.Path {
		t.Fatalf(
			"Request paths do not differ. Expected %q != %q",
			req1.URL.Path,
			req2.URL.Path,
		)
	}
	if req1.URL.RawQuery != req2.URL.RawQuery {
		t.Fatalf(
			"Request query params do not match. Expected %q, got %q",
			req1.URL.RawQuery,
			req2.URL.RawQuery,
		)
	}

	for _, populateCache := range []bool{true, false} {
		for _, req := range []*http.Request{req1, req2} {
			if populateCache {
				originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
					w.Header().Set(respHeaderName, r.URL.Path)
				})
			} else {
				originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
					t.Errorf(
						"Request with path %q should not have made it to origin",
						r.URL.Path,
					)
				})
			}

			resp := RoundTripCheckError(t, req)
			defer resp.Body.Close()

			if recVal := resp.Header.Get(respHeaderName); recVal != req.URL.Path {
				t.Errorf(
					"Request received wrong %q header. Expected %q, got %q",
					respHeaderName,
					req.URL.Path,
					recVal,
				)
			}
		}
	}
}
<commit_msg>[#76769514] Disable TestCacheVary for CloudFlare<commit_after>package main

import (
	"fmt"
	"net/http"
	"strings"
	"testing"
	"time"
)

// Should cache first response for an unspecified period of time if when it
// doesn't specify it's own cache headers. Subsequent requests should return
// a cached response.
func TestCacheFirstResponse(t *testing.T) {
	ResetBackends(backendsByPriority)

	testRequestsCachedIndefinite(t, nil)
}

// Should cache responses for the period defined in a `Expires: n` response
// header.
func TestCacheExpires(t *testing.T) {
	ResetBackends(backendsByPriority)

	const cacheDuration = time.Duration(5 * time.Second)

	handler := func(w http.ResponseWriter) {
		headerValue := time.Now().UTC().Add(cacheDuration).Format(http.TimeFormat)
		w.Header().Set("Expires", headerValue)
	}

	testRequestsCachedDuration(t, handler, cacheDuration)
}

// Should cache responses for the period defined in a `Cache-Control:
// max-age=n` response header.
func TestCacheCacheControlMaxAge(t *testing.T) {
	ResetBackends(backendsByPriority)

	const cacheDuration = time.Duration(5 * time.Second)
	headerValue := fmt.Sprintf("max-age=%.0f", cacheDuration.Seconds())

	handler := func(w http.ResponseWriter) {
		w.Header().Set("Cache-Control", headerValue)
	}

	testRequestsCachedDuration(t, handler, cacheDuration)
}

// Should cache responses for the period defined in a `Cache-Control:
// max-age=n` response header when a `Expires: n*2` header is also present.
func TestCacheExpiresAndMaxAge(t *testing.T) {
	ResetBackends(backendsByPriority)

	const cacheDuration = time.Duration(5 * time.Second)
	const expiresDuration = cacheDuration * 2

	maxAgeValue := fmt.Sprintf("max-age=%.0f", cacheDuration.Seconds())

	handler := func(w http.ResponseWriter) {
		expiresValue := time.Now().UTC().Add(expiresDuration).Format(http.TimeFormat)

		w.Header().Set("Expires", expiresValue)
		w.Header().Set("Cache-Control", maxAgeValue)
	}

	testRequestsCachedDuration(t, handler, cacheDuration)
}

// Should cache responses with a status code of 404. It's a common
// misconception that 404 responses shouldn't be cached; they should because
// they can be expensive to generate.
func TestCache404Response(t *testing.T) {
	ResetBackends(backendsByPriority)

	handler := func(w http.ResponseWriter) {
		w.WriteHeader(http.StatusNotFound)
	}

	testRequestsCachedIndefinite(t, handler)
}

// Should cache multiple distinct responses for the same URL when origin responds
// with a `Vary` header and clients provide requests with different values
// for that header.
func TestCacheVary(t *testing.T) {
	ResetBackends(backendsByPriority)

	if vendorCloudflare {
		t.Skip(notSupportedByVendor)
	}

	const reqHeaderName = "CustomThing"
	const respHeaderName = "Reflected-" + reqHeaderName
	headerVals := []string{
		"first distinct",
		"second distinct",
		"third distinct",
	}

	req := NewUniqueEdgeGET(t)

	for _, populateCache := range []bool{true, false} {
		for _, headerVal := range headerVals {
			if populateCache {
				originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
					w.Header().Set("Vary", reqHeaderName)
					w.Header().Set(respHeaderName, r.Header.Get(reqHeaderName))
				})
			} else {
				originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
					t.Error("Request should not have made it to origin")
					w.Header().Set(respHeaderName, "not cached")
				})
			}

			req.Header.Set(reqHeaderName, headerVal)
			resp := RoundTripCheckError(t, req)
			defer resp.Body.Close()

			if recVal := resp.Header.Get(respHeaderName); recVal != headerVal {
				t.Errorf(
					"Request received wrong %q header. Expected %q, got %q",
					respHeaderName,
					headerVal,
					recVal,
				)
			}
		}
	}
}

// Should cache distinct responses for requests with the same path but
// different query params.
func TestCacheUniqueQueryParams(t *testing.T) {
	ResetBackends(backendsByPriority)

	const respHeaderName = "Request-RawQuery"

	req1 := NewUniqueEdgeGET(t)
	req2 := NewUniqueEdgeGET(t)

	if req1.URL.Path != req2.URL.Path {
		t.Fatalf(
			"Request paths do not match. Expected %q, got %q",
			req1.URL.Path,
			req2.URL.Path,
		)
	}
	if req1.URL.RawQuery == req2.URL.RawQuery {
		t.Fatalf(
			"Request query params do not differ. Expected %q != %q",
			req1.URL.RawQuery,
			req2.URL.RawQuery,
		)
	}

	for _, populateCache := range []bool{true, false} {
		for _, req := range []*http.Request{req1, req2} {
			if populateCache {
				originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
					w.Header().Set(respHeaderName, r.URL.RawQuery)
				})
			} else {
				originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
					t.Errorf(
						"Request with query param %q should not have made it to origin",
						r.URL.RawQuery,
					)
				})
			}

			resp := RoundTripCheckError(t, req)
			defer resp.Body.Close()

			if recVal := resp.Header.Get(respHeaderName); recVal != req.URL.RawQuery {
				t.Errorf(
					"Request received wrong %q header. Expected %q, got %q",
					respHeaderName,
					req.URL.RawQuery,
					recVal,
				)
			}
		}
	}
}

// Should cache distinct responses for requests with the same query params
// but paths of different case-sensitivity.
func TestCacheUniqueCaseSensitive(t *testing.T) {
	ResetBackends(backendsByPriority)

	const reqPath = "/CaseSensitive"
	const respHeaderName = "Request-Path"

	req1 := NewUniqueEdgeGET(t)
	req2 := NewUniqueEdgeGET(t)

	req1.URL.Path = strings.ToLower(reqPath)
	req2.URL.Path = strings.ToUpper(reqPath)
	req1.URL.RawQuery = req2.URL.RawQuery

	if req1.URL.Path == req2.URL.Path {
		t.Fatalf(
			"Request paths do not differ. Expected %q != %q",
			req1.URL.Path,
			req2.URL.Path,
		)
	}
	if req1.URL.RawQuery != req2.URL.RawQuery {
		t.Fatalf(
			"Request query params do not match. Expected %q, got %q",
			req1.URL.RawQuery,
			req2.URL.RawQuery,
		)
	}

	for _, populateCache := range []bool{true, false} {
		for _, req := range []*http.Request{req1, req2} {
			if populateCache {
				originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
					w.Header().Set(respHeaderName, r.URL.Path)
				})
			} else {
				originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
					t.Errorf(
						"Request with path %q should not have made it to origin",
						r.URL.Path,
					)
				})
			}

			resp := RoundTripCheckError(t, req)
			defer resp.Body.Close()

			if recVal := resp.Header.Get(respHeaderName); recVal != req.URL.Path {
				t.Errorf(
					"Request received wrong %q header. Expected %q, got %q",
					respHeaderName,
					req.URL.Path,
					recVal,
				)
			}
		}
	}
}
<|endoftext|>


<commit_before>// Copyright © 2016 Alan A. A. Donovan & Brian W. Kernighan.
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/

// See page 10.
//!+

// Dup2 prints the count and text of lines that appear more than once
// in the input.  It reads from stdin or from a list of named files.
package main

import (
	"bufio"
	"fmt"
	"os"
)

func main() {
	counts := make(map[string]int)
	files := os.Args[1:]
	if len(files) == 0 {
		countLines(os.Stdin, counts)
	} else {
		for _, arg := range files {
			f, err := os.Open(arg)
			if err != nil {
				fmt.Fprintf(os.Stderr, "dup2: %v\n", err)
				continue
			}
			countLines(f, counts)
			f.Close()
		}
	}
	for line, n := range counts {
		if n > 1 {
			fmt.Printf("%d\t%s\n", n, line)
		}
	}
}

func countLines(f *os.File, counts map[string]int) {
	input := bufio.NewScanner(f)
	for input.Scan() {
		counts[input.Text()]++
	}
	// NOTE: ignoring potential errors from input.Err()
}

//!-
<commit_msg>Modified main.go.<commit_after>// Copyright © 2016 Alan A. A. Donovan & Brian W. Kernighan.
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/

// See page 10.
//!+

// Dup2 prints the count and text of lines that appear more than once
// in the input.  It reads from stdin or from a list of named files.
package main

import (
	"bufio"
	"fmt"
	"os"
)

func main() {
	counts := make(map[string]int)
    dupFiles := make(map[string]map[string]bool)

    files := os.Args[1:]
	if len(files) == 0 {
		countLines("-", os.Stdin, counts, dupFiles)
	} else {
		for _, arg := range files {
			f, err := os.Open(arg)
			if err != nil {
				fmt.Fprintf(os.Stderr, "dup2: %v\n", err)
				continue
			}
			countLines(arg, f, counts, dupFiles)
			f.Close()
		}
	}
	for line, n := range counts {
		if n > 1 {
			fmt.Printf("%d\t%s\t%s\n", n, joinKeys(dupFiles[line]), line)
		}
	}
}

func countLines(file string, f *os.File, counts map[string]int, files map[string]map[string]bool) {
	input := bufio.NewScanner(f)
	for input.Scan() {
        text := input.Text()
		counts[text]++
        fs := files[text]
        if (fs == nil) {
            fs = make(map[string]bool)
            files[text] = fs
        }
        fs[file] = true
	}
	// NOTE: ignoring potential errors from input.Err()
}

func joinKeys(set map[string]bool) string {
    sep := ""
    s := ""

    for k, _ := range set {
        s += sep + k
        sep = " "
    }
    return s
}

//!-
<|endoftext|>