<commit_before>// Copyright 2016 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.

// Package crdb provides helpers for using CockroachDB in client
// applications.
package crdb

import (
	"context"
	"database/sql"

	"github.com/jackc/pgconn"
	"github.com/lib/pq"
)

// ExecuteTx runs fn inside a transaction and retries it as needed.
// On non-retryable failures, the transaction is aborted and rolled
// back; on success, the transaction is committed.
// There are cases where the state of a transaction is inherently ambiguous: if
// we err on RELEASE with a communication error it's unclear if the transaction
// has been committed or not (similar to erroring on COMMIT in other databases).
// In that case, we return AmbiguousCommitError.
// There are cases when restarting a transaction fails: we err on ROLLBACK
// to the SAVEPOINT. In that case, we return a TxnRestartError.
//
// For more information about CockroachDB's transaction model see
// https://cockroachlabs.com/docs/stable/transactions.html.
//
// NOTE: the supplied fn closure should not have external side
// effects beyond changes to the database.
//
// fn must take care when wrapping errors returned from the database driver with
// additional context. For example, if the UPDATE statement fails in the
// following snippet, the original retryable error will be masked by the call to
// fmt.Errorf, and the transaction will not be automatically retried.
//
//    crdb.ExecuteTx(ctx, db, txopts, func (tx *sql.Tx) error {
//        if err := tx.ExecContext(ctx, "UPDATE..."); err != nil {
//            return fmt.Errorf("updating record: %s", err)
//        }
//        return nil
//    })
//
// Instead, add context by returning an error that implements the ErrorCauser
// interface. Either create a custom error type that implements ErrorCauser or
// use a helper function that does so automatically, like pkg/errors.Wrap:
//
//    import "github.com/pkg/errors"
//
//    crdb.ExecuteTx(ctx, db, txopts, func (tx *sql.Tx) error {
//        if err := tx.ExecContext(ctx, "UPDATE..."); err != nil {
//            return errors.Wrap(err, "updating record")
//        }
//        return nil
//    })
//
func ExecuteTx(
	ctx context.Context, db *sql.DB, txopts *sql.TxOptions, fn func(*sql.Tx) error,
) error {
	// Start a transaction.
	tx, err := db.BeginTx(ctx, txopts)
	if err != nil {
		return err
	}
	return ExecuteInTx(ctx, tx, func() error { return fn(tx) })
}

// Tx is used to permit clients to implement custom transaction logic.
type Tx interface {
	ExecContext(context.Context, string, ...interface{}) (sql.Result, error)
	Commit() error
	Rollback() error
}

// ExecuteInTx runs fn inside tx which should already have begun.
// *WARNING*: Do not execute any statements on the supplied tx before calling this function.
// ExecuteInTx will only retry statements that are performed within the supplied
// closure (fn). Any statements performed on the tx before ExecuteInTx is invoked will *not*
// be re-run if the transaction needs to be retried.
//
// fn is subject to the same restrictions as the fn passed to ExecuteTx.
func ExecuteInTx(ctx context.Context, tx Tx, fn func() error) (err error) {
	defer func() {
		if err == nil {
			// Ignore commit errors. The tx has already been committed by RELEASE.
			_ = tx.Commit()
		} else {
			// We always need to execute a Rollback() so sql.DB releases the
			// connection.
			_ = tx.Rollback()
		}
	}()
	// Specify that we intend to retry this txn in case of CockroachDB retryable
	// errors.
	if _, err = tx.ExecContext(ctx, "SAVEPOINT cockroach_restart"); err != nil {
		return err
	}

	for {
		released := false
		err = fn()
		if err == nil {
			// RELEASE acts like COMMIT in CockroachDB. We use it since it gives us an
			// opportunity to react to retryable errors, whereas tx.Commit() doesn't.
			released = true
			if _, err = tx.ExecContext(ctx, "RELEASE SAVEPOINT cockroach_restart"); err == nil {
				return nil
			}
		}
		// We got an error; let's see if it's a retryable one and, if so, restart.
		// We look for either:
		//  - the standard PG errcode SerializationFailureError:40001 or
		//  - the Cockroach extension errcode RetriableError:CR000. This extension
		//    has been removed server-side, but support for it has been left here for
		//    now to maintain backwards compatibility.
		code := errCode(err)
		if retryable := (code == "CR000" || code == "40001"); !retryable {
			if released {
				err = newAmbiguousCommitError(err)
			}
			return err
		}

		if _, retryErr := tx.ExecContext(ctx, "ROLLBACK TO SAVEPOINT cockroach_restart"); retryErr != nil {
			return newTxnRestartError(retryErr, err)
		}
	}
}

func errCode(err error) string {
	switch t := errorCause(err).(type) {
	case *pq.Error:
		return string(t.Code)

	case *pgconn.PgError:
		return t.Code

	default:
		return ""
	}
}
<commit_msg>Create Execute library function<commit_after>// Copyright 2016 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.

// Package crdb provides helpers for using CockroachDB in client
// applications.
package crdb

import (
	"context"
	"database/sql"

	"github.com/jackc/pgconn"
	"github.com/lib/pq"
)

// Execute runs fn and retries it as needed. It is used to add retry handling to
// the execution of a single statement or single batch of statements. If a
// multi-statement transaction is being run, use ExecuteTx instead.
//
// Retry handling for individual statements (implicit transactions) is usually
// performed automatically on the CockroachDB SQL gateway. As such, use of this
// function is generally not necessary. The exception to this rule is that
// automatic retries for individual statements are disabled once CockroachDB
// begins streaming results for the statements back to the client. By default,
// result streaming does not begin until the size of the result being produced
// for the client, including protocol overhead, exceeds 16KiB. As long as the
// results of a single statement or batch of statements are known to stay clear
// of this limit, the client does not need to worry about retries and should not
// need to use this function.
//
// For more information about automatic transaction retries in CockroachDB, see
// https://cockroachlabs.com/docs/stable/transactions.html#automatic-retries.
//
// NOTE: the supplied fn closure should not have external side effects beyond
// changes to the database.
//
// fn must take care when wrapping errors returned from the database driver with
// additional context. For example, if the SELECT statement fails in the
// following snippet, the original retryable error will be masked by the call to
// fmt.Errorf, and the transaction will not be automatically retried.
//
//    crdb.Execute(func () error {
//        rows, err := db.QueryContext(ctx, "SELECT ...")
//        if err != nil {
//            return fmt.Errorf("scanning row: %s", err)
//        }
//        defer rows.Close()
//        for rows.Next() {
//            // ...
//        }
//        if err := rows.Err(); err != nil {
//            return fmt.Errorf("scanning row: %s", err)
//        }
//        return nil
//    })
//
// Instead, add context by returning an error that implements the ErrorCauser
// interface. Either create a custom error type that implements ErrorCauser or
// use a helper function that does so automatically, like pkg/errors.Wrap:
//
//    import "github.com/pkg/errors"
//
//    crdb.Execute(func () error {
//        rows, err := db.QueryContext(ctx, "SELECT ...")
//        if err != nil {
//            return errors.Wrap(err, "scanning row")
//        }
//        defer rows.Close()
//        for rows.Next() {
//            // ...
//        }
//        if err := rows.Err(); err != nil {
//            return errors.Wrap(err, "scanning row")
//        }
//        return nil
//    })
//
func Execute(fn func() error) (err error) {
	for {
		err = fn()
		if err == nil || !errIsRetryable(err) {
			return err
		}
	}
}

// ExecuteTx runs fn inside a transaction and retries it as needed. On
// non-retryable failures, the transaction is aborted and rolled back; on
// success, the transaction is committed.
//
// There are cases where the state of a transaction is inherently ambiguous: if
// we err on RELEASE with a communication error it's unclear if the transaction
// has been committed or not (similar to erroring on COMMIT in other databases).
// In that case, we return AmbiguousCommitError.
//
// There are cases when restarting a transaction fails: we err on ROLLBACK to
// the SAVEPOINT. In that case, we return a TxnRestartError.
//
// For more information about CockroachDB's transaction model, see
// https://cockroachlabs.com/docs/stable/transactions.html.
//
// NOTE: the supplied fn closure should not have external side effects beyond
// changes to the database.
//
// fn must take care when wrapping errors returned from the database driver with
// additional context. For example, if the UPDATE statement fails in the
// following snippet, the original retryable error will be masked by the call to
// fmt.Errorf, and the transaction will not be automatically retried.
//
//    crdb.ExecuteTx(ctx, db, txopts, func (tx *sql.Tx) error {
//        if err := tx.ExecContext(ctx, "UPDATE..."); err != nil {
//            return fmt.Errorf("updating record: %s", err)
//        }
//        return nil
//    })
//
// Instead, add context by returning an error that implements the ErrorCauser
// interface. Either create a custom error type that implements ErrorCauser or
// use a helper function that does so automatically, like pkg/errors.Wrap:
//
//    import "github.com/pkg/errors"
//
//    crdb.ExecuteTx(ctx, db, txopts, func (tx *sql.Tx) error {
//        if err := tx.ExecContext(ctx, "UPDATE..."); err != nil {
//            return errors.Wrap(err, "updating record")
//        }
//        return nil
//    })
//
func ExecuteTx(
	ctx context.Context, db *sql.DB, txopts *sql.TxOptions, fn func(*sql.Tx) error,
) error {
	// Start a transaction.
	tx, err := db.BeginTx(ctx, txopts)
	if err != nil {
		return err
	}
	return ExecuteInTx(ctx, tx, func() error { return fn(tx) })
}

// Tx is used to permit clients to implement custom transaction logic.
type Tx interface {
	ExecContext(context.Context, string, ...interface{}) (sql.Result, error)
	Commit() error
	Rollback() error
}

// ExecuteInTx runs fn inside tx which should already have begun.
//
// *WARNING*: Do not execute any statements on the supplied tx before calling this function.
// ExecuteInTx will only retry statements that are performed within the supplied
// closure (fn). Any statements performed on the tx before ExecuteInTx is invoked will *not*
// be re-run if the transaction needs to be retried.
//
// fn is subject to the same restrictions as the fn passed to ExecuteTx.
func ExecuteInTx(ctx context.Context, tx Tx, fn func() error) (err error) {
	defer func() {
		if err == nil {
			// Ignore commit errors. The tx has already been committed by RELEASE.
			_ = tx.Commit()
		} else {
			// We always need to execute a Rollback() so sql.DB releases the
			// connection.
			_ = tx.Rollback()
		}
	}()
	// Specify that we intend to retry this txn in case of CockroachDB retryable
	// errors.
	if _, err = tx.ExecContext(ctx, "SAVEPOINT cockroach_restart"); err != nil {
		return err
	}

	for {
		released := false
		err = fn()
		if err == nil {
			// RELEASE acts like COMMIT in CockroachDB. We use it since it gives us an
			// opportunity to react to retryable errors, whereas tx.Commit() doesn't.
			released = true
			if _, err = tx.ExecContext(ctx, "RELEASE SAVEPOINT cockroach_restart"); err == nil {
				return nil
			}
		}
		// We got an error; let's see if it's a retryable one and, if so, restart.
		if !errIsRetryable(err) {
			if released {
				err = newAmbiguousCommitError(err)
			}
			return err
		}

		if _, retryErr := tx.ExecContext(ctx, "ROLLBACK TO SAVEPOINT cockroach_restart"); retryErr != nil {
			return newTxnRestartError(retryErr, err)
		}
	}
}

func errIsRetryable(err error) bool {
	// We look for either:
	//  - the standard PG errcode SerializationFailureError:40001 or
	//  - the Cockroach extension errcode RetriableError:CR000. This extension
	//    has been removed server-side, but support for it has been left here for
	//    now to maintain backwards compatibility.
	code := errCode(err)
	return code == "CR000" || code == "40001"
}

func errCode(err error) string {
	switch t := errorCause(err).(type) {
	case *pq.Error:
		return string(t.Code)

	case *pgconn.PgError:
		return t.Code

	default:
		return ""
	}
}
<|endoftext|>


<commit_before>package action

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strings"

	"github.com/fatih/color"
	"github.com/gopasspw/gopass/pkg/ctxutil"
	"github.com/gopasspw/gopass/pkg/termutil"
	"github.com/gopasspw/gopass/pkg/tree"

	shellquote "github.com/kballard/go-shellquote"
	"github.com/pkg/errors"
	"github.com/urfave/cli/v2"
)

// List all secrets as a tree. If the filter argument is non-empty
// display only those that have this prefix
func (s *Action) List(c *cli.Context) error {
	ctx := ctxutil.WithGlobalFlags(c)
	filter := c.Args().First()
	flat := c.Bool("flat")
	stripPrefix := c.Bool("strip-prefix")
	folders := c.Bool("folders")
	// we only support listing folders in flat mode currently
	if folders {
		flat = true
	}

	ctx = s.Store.WithConfig(ctx, filter)

	l, err := s.Store.Tree(ctx)
	if err != nil {
		return ExitError(ctx, ExitList, err, "failed to list store: %s", err)
	}

	//set limit to len of store to loop over all values later
	limit := l.Len()
	if c.IsSet("limit") {
		limit = c.Int("limit")
	}

	if filter == "" {
		return s.listAll(ctx, l, limit, flat, folders)
	}

	return s.listFiltered(ctx, l, limit, flat, folders, stripPrefix, filter)
}

func (s *Action) listFiltered(ctx context.Context, l tree.Tree, limit int, flat, folders, stripPrefix bool, filter string) error {
	subtree, err := l.FindFolder(filter)
	if err != nil {
		return ExitError(ctx, ExitNotFound, nil, "Entry '%s' not found", filter)
	}

	// SetRoot formats the root entry properly
	subtree.SetRoot(true)
	subtree.SetName(filter)
	if flat {
		sep := "/"
		if strings.HasSuffix(filter, "/") {
			sep = ""
		}
		listOver := subtree.List
		if folders {
			listOver = subtree.ListFolders
		}
		for _, e := range listOver(limit) {
			if stripPrefix {
				fmt.Fprintln(stdout, e)
				continue
			}
			fmt.Fprintln(stdout, filter+sep+e)
		}
		return nil
	}

	// we may need to redirect stdout for the pager support
	so, buf := redirectPager(ctx, subtree)

	fmt.Fprintln(so, subtree.Format(limit))
	if buf != nil {
		if err := s.pager(ctx, buf); err != nil {
			return ExitError(ctx, ExitUnknown, err, "failed to invoke pager: %s", err)
		}
	}
	return nil
}

// redirectPager returns a redirected io.Writer if the output would exceed
// the terminal size
func redirectPager(ctx context.Context, subtree tree.Tree) (io.Writer, *bytes.Buffer) {
	if ctxutil.IsNoPager(ctx) {
		return stdout, nil
	}
	rows, _ := termutil.GetTermsize()
	if rows <= 0 {
		return stdout, nil
	}
	if subtree == nil || subtree.Len() < rows {
		return stdout, nil
	}
	if pager := os.Getenv("PAGER"); pager == "" {
		return stdout, nil
	}
	color.NoColor = true
	buf := &bytes.Buffer{}
	return buf, buf
}

// listAll will unconditionally list all entries, used if no filter is given
func (s *Action) listAll(ctx context.Context, l tree.Tree, limit int, flat, folders bool) error {
	if flat {
		listOver := l.List
		if folders {
			listOver = l.ListFolders
		}
		for _, e := range listOver(limit) {
			fmt.Fprintln(stdout, e)
		}
		return nil
	}

	// we may need to redirect stdout for the pager support
	so, buf := redirectPager(ctx, l)

	fmt.Fprintln(so, l.Format(limit))
	if buf != nil {
		if err := s.pager(ctx, buf); err != nil {
			return ExitError(ctx, ExitUnknown, err, "failed to invoke pager: %s", err)
		}
	}
	return nil
}

// pager invokes the default pager with the given content
func (s *Action) pager(ctx context.Context, buf io.Reader) error {
	pager := os.Getenv("PAGER")
	if pager == "" {
		fmt.Fprintln(stdout, buf)
		return nil
	}

	args, err := shellquote.Split(pager)
	if err != nil {
		return errors.Wrapf(err, "failed to split pager command")
	}

	cmd := exec.CommandContext(ctx, args[0], args[1:]...)
	cmd.Stdin = buf
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	return cmd.Run()
}
<commit_msg>Do not error our if listing a single entry (#1303)<commit_after>package action

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strings"

	"github.com/fatih/color"
	"github.com/gopasspw/gopass/pkg/ctxutil"
	"github.com/gopasspw/gopass/pkg/termutil"
	"github.com/gopasspw/gopass/pkg/tree"

	shellquote "github.com/kballard/go-shellquote"
	"github.com/pkg/errors"
	"github.com/urfave/cli/v2"
)

// List all secrets as a tree. If the filter argument is non-empty
// display only those that have this prefix
func (s *Action) List(c *cli.Context) error {
	ctx := ctxutil.WithGlobalFlags(c)
	filter := c.Args().First()
	flat := c.Bool("flat")
	stripPrefix := c.Bool("strip-prefix")
	folders := c.Bool("folders")

	// print the path if the argument is a direct hit
	if s.Store.Exists(ctx, filter) && !s.Store.IsDir(ctx, filter) {
		fmt.Println(filter)
		return nil
	}

	// we only support listing folders in flat mode currently
	if folders {
		flat = true
	}

	ctx = s.Store.WithConfig(ctx, filter)

	l, err := s.Store.Tree(ctx)
	if err != nil {
		return ExitError(ctx, ExitList, err, "failed to list store: %s", err)
	}

	//set limit to len of store to loop over all values later
	limit := l.Len()
	if c.IsSet("limit") {
		limit = c.Int("limit")
	}

	if filter == "" {
		return s.listAll(ctx, l, limit, flat, folders)
	}

	return s.listFiltered(ctx, l, limit, flat, folders, stripPrefix, filter)
}

func (s *Action) listFiltered(ctx context.Context, l tree.Tree, limit int, flat, folders, stripPrefix bool, filter string) error {
	subtree, err := l.FindFolder(filter)
	if err != nil {
		return ExitError(ctx, ExitNotFound, nil, "Entry '%s' not found", filter)
	}

	// SetRoot formats the root entry properly
	subtree.SetRoot(true)
	subtree.SetName(filter)
	if flat {
		sep := "/"
		if strings.HasSuffix(filter, "/") {
			sep = ""
		}
		listOver := subtree.List
		if folders {
			listOver = subtree.ListFolders
		}
		for _, e := range listOver(limit) {
			if stripPrefix {
				fmt.Fprintln(stdout, e)
				continue
			}
			fmt.Fprintln(stdout, filter+sep+e)
		}
		return nil
	}

	// we may need to redirect stdout for the pager support
	so, buf := redirectPager(ctx, subtree)

	fmt.Fprintln(so, subtree.Format(limit))
	if buf != nil {
		if err := s.pager(ctx, buf); err != nil {
			return ExitError(ctx, ExitUnknown, err, "failed to invoke pager: %s", err)
		}
	}
	return nil
}

// redirectPager returns a redirected io.Writer if the output would exceed
// the terminal size
func redirectPager(ctx context.Context, subtree tree.Tree) (io.Writer, *bytes.Buffer) {
	if ctxutil.IsNoPager(ctx) {
		return stdout, nil
	}
	rows, _ := termutil.GetTermsize()
	if rows <= 0 {
		return stdout, nil
	}
	if subtree == nil || subtree.Len() < rows {
		return stdout, nil
	}
	if pager := os.Getenv("PAGER"); pager == "" {
		return stdout, nil
	}
	color.NoColor = true
	buf := &bytes.Buffer{}
	return buf, buf
}

// listAll will unconditionally list all entries, used if no filter is given
func (s *Action) listAll(ctx context.Context, l tree.Tree, limit int, flat, folders bool) error {
	if flat {
		listOver := l.List
		if folders {
			listOver = l.ListFolders
		}
		for _, e := range listOver(limit) {
			fmt.Fprintln(stdout, e)
		}
		return nil
	}

	// we may need to redirect stdout for the pager support
	so, buf := redirectPager(ctx, l)

	fmt.Fprintln(so, l.Format(limit))
	if buf != nil {
		if err := s.pager(ctx, buf); err != nil {
			return ExitError(ctx, ExitUnknown, err, "failed to invoke pager: %s", err)
		}
	}
	return nil
}

// pager invokes the default pager with the given content
func (s *Action) pager(ctx context.Context, buf io.Reader) error {
	pager := os.Getenv("PAGER")
	if pager == "" {
		fmt.Fprintln(stdout, buf)
		return nil
	}

	args, err := shellquote.Split(pager)
	if err != nil {
		return errors.Wrapf(err, "failed to split pager command")
	}

	cmd := exec.CommandContext(ctx, args[0], args[1:]...)
	cmd.Stdin = buf
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	return cmd.Run()
}
<|endoftext|>


<commit_before>package audit

import (
	"bytes"
	"context"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"time"

	"github.com/pborman/uuid"

	k8stypes "k8s.io/apimachinery/pkg/types"
)

const (
	contentTypeJSON = "application/json"
)

const (
	levelNull = iota
	levelMetadata
	levelRequest
	levelRequestResponse
)

var (
	bodyMethods = map[string]bool{
		http.MethodPut:  true,
		http.MethodPost: true,
	}
	sensitiveRequestHeader  = []string{"Cookie", "Authorization"}
	sensitiveResponseHeader = []string{"Cookie", "Set-Cookie"}
)

type auditLog struct {
	log     *log
	writer  *LogWriter
	reqBody []byte
}

type log struct {
	AuditID           k8stypes.UID `json:"auditID,omitempty"`
	RequestURI        string       `json:"requestURI,omitempty"`
	User              *UserInfo    `json:"user,omitempty"`
	Method            string       `json:"method,omitempty"`
	RemoteAddr        string       `json:"remoteAddr,omitempty"`
	RequestTimestamp  string       `json:"requestTimestamp,omitempty"`
	ResponseTimestamp string       `json:"responseTimestamp,omitempty"`
	ResponseCode      int          `json:"responseCode,omitempty"`
	RequestHeader     http.Header  `json:"requestHeader,omitempty"`
	ResponseHeader    http.Header  `json:"responseHeader,omitempty"`
	RequestBody       []byte       `json:"requestBody,omitempty"`
	ResponseBody      []byte       `json:"responseBody,omitempty"`
}

var userKey struct{}

type UserInfo struct {
	Name  string   `json:"name,omitempty"`
	Group []string `json:"group,omitempty"`
	// RequestUser is the --as user
	RequestUser string `json:"requestUser,omitempty"`
	// RequestGroups is the --as-group list
	RequestGroups []string `json:"requestGroups,omitempty"`
}

func GetUserInfo(req *http.Request) *UserInfo {
	return &UserInfo{
		Name:  req.Header.Get("Impersonate-User"),
		Group: req.Header["Impersonate-Group"],
	}
}

func FromContext(ctx context.Context) (*UserInfo, bool) {
	u, ok := ctx.Value(userKey).(*UserInfo)
	return u, ok
}

func new(writer *LogWriter, req *http.Request) (*auditLog, error) {
	auditLog := &auditLog{
		writer: writer,
		log: &log{
			AuditID:          k8stypes.UID(uuid.NewRandom().String()),
			RequestURI:       req.RequestURI,
			Method:           req.Method,
			RemoteAddr:       req.RemoteAddr,
			RequestTimestamp: time.Now().Format(time.RFC3339),
		},
	}

	contentType := req.Header.Get("Content-Type")
	if writer.Level >= levelRequest && bodyMethods[req.Method] && contentType == contentTypeJSON {
		reqBody, err := readBodyWithoutLosingContent(req)
		if err != nil {
			return nil, err
		}
		auditLog.reqBody = reqBody
	}
	return auditLog, nil
}

func (a *auditLog) write(userInfo *UserInfo, reqHeaders, resHeaders http.Header, resCode int, resBody []byte) error {
	a.log.User = userInfo
	a.log.ResponseTimestamp = time.Now().Format(time.RFC3339)
	a.log.RequestHeader = filterOutHeaders(reqHeaders, sensitiveRequestHeader)
	a.log.ResponseHeader = filterOutHeaders(resHeaders, sensitiveResponseHeader)
	a.log.ResponseCode = resCode

	var buffer bytes.Buffer
	alByte, err := json.Marshal(a.log)
	if err != nil {
		return err
	}

	buffer.Write(bytes.TrimSuffix(alByte, []byte("}")))
	if a.writer.Level >= levelRequest && len(a.reqBody) > 0 {
		buffer.WriteString(`,"requestBody":`)
		buffer.Write(bytes.TrimSuffix(a.reqBody, []byte("\n")))
	}
	if a.writer.Level >= levelRequestResponse && resHeaders.Get("Content-Type") == contentTypeJSON && len(resBody) > 0 {
		buffer.WriteString(`,"responseBody":`)
		buffer.Write(bytes.TrimSuffix(resBody, []byte("\n")))
	}
	buffer.WriteString("}\n")
	_, err = a.writer.Output.Write(buffer.Bytes())
	return err
}

func readBodyWithoutLosingContent(req *http.Request) ([]byte, error) {
	if !bodyMethods[req.Method] {
		return nil, nil
	}

	bodyBytes, err := ioutil.ReadAll(req.Body)
	if err != nil {
		return nil, err
	}
	req.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes))

	return bodyBytes, nil
}

func filterOutHeaders(headers http.Header, filterKeys []string) map[string][]string {
	newHeader := make(map[string][]string)
	for k, v := range headers {
		if isExist(filterKeys, k) {
			continue
		}
		newHeader[k] = v
	}
	return newHeader
}

func isExist(array []string, key string) bool {
	for _, v := range array {
		if v == key {
			return true
		}
	}
	return false
}
<commit_msg>fix json response from proxy are not compacted in audit log<commit_after>package audit

import (
	"bytes"
	"context"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"time"

	"github.com/pborman/uuid"
	"github.com/pkg/errors"
	k8stypes "k8s.io/apimachinery/pkg/types"
)

const (
	contentTypeJSON = "application/json"
)

const (
	levelNull = iota
	levelMetadata
	levelRequest
	levelRequestResponse
)

var (
	bodyMethods = map[string]bool{
		http.MethodPut:  true,
		http.MethodPost: true,
	}
	sensitiveRequestHeader  = []string{"Cookie", "Authorization"}
	sensitiveResponseHeader = []string{"Cookie", "Set-Cookie"}
)

type auditLog struct {
	log     *log
	writer  *LogWriter
	reqBody []byte
}

type log struct {
	AuditID           k8stypes.UID `json:"auditID,omitempty"`
	RequestURI        string       `json:"requestURI,omitempty"`
	User              *UserInfo    `json:"user,omitempty"`
	Method            string       `json:"method,omitempty"`
	RemoteAddr        string       `json:"remoteAddr,omitempty"`
	RequestTimestamp  string       `json:"requestTimestamp,omitempty"`
	ResponseTimestamp string       `json:"responseTimestamp,omitempty"`
	ResponseCode      int          `json:"responseCode,omitempty"`
	RequestHeader     http.Header  `json:"requestHeader,omitempty"`
	ResponseHeader    http.Header  `json:"responseHeader,omitempty"`
	RequestBody       []byte       `json:"requestBody,omitempty"`
	ResponseBody      []byte       `json:"responseBody,omitempty"`
}

var userKey struct{}

type UserInfo struct {
	Name  string   `json:"name,omitempty"`
	Group []string `json:"group,omitempty"`
	// RequestUser is the --as user
	RequestUser string `json:"requestUser,omitempty"`
	// RequestGroups is the --as-group list
	RequestGroups []string `json:"requestGroups,omitempty"`
}

func GetUserInfo(req *http.Request) *UserInfo {
	return &UserInfo{
		Name:  req.Header.Get("Impersonate-User"),
		Group: req.Header["Impersonate-Group"],
	}
}

func FromContext(ctx context.Context) (*UserInfo, bool) {
	u, ok := ctx.Value(userKey).(*UserInfo)
	return u, ok
}

func new(writer *LogWriter, req *http.Request) (*auditLog, error) {
	auditLog := &auditLog{
		writer: writer,
		log: &log{
			AuditID:          k8stypes.UID(uuid.NewRandom().String()),
			RequestURI:       req.RequestURI,
			Method:           req.Method,
			RemoteAddr:       req.RemoteAddr,
			RequestTimestamp: time.Now().Format(time.RFC3339),
		},
	}

	contentType := req.Header.Get("Content-Type")
	if writer.Level >= levelRequest && bodyMethods[req.Method] && contentType == contentTypeJSON {
		reqBody, err := readBodyWithoutLosingContent(req)
		if err != nil {
			return nil, err
		}
		auditLog.reqBody = reqBody
	}
	return auditLog, nil
}

func (a *auditLog) write(userInfo *UserInfo, reqHeaders, resHeaders http.Header, resCode int, resBody []byte) error {
	a.log.User = userInfo
	a.log.ResponseTimestamp = time.Now().Format(time.RFC3339)
	a.log.RequestHeader = filterOutHeaders(reqHeaders, sensitiveRequestHeader)
	a.log.ResponseHeader = filterOutHeaders(resHeaders, sensitiveResponseHeader)
	a.log.ResponseCode = resCode

	var buffer bytes.Buffer
	alByte, err := json.Marshal(a.log)
	if err != nil {
		return err
	}

	buffer.Write(bytes.TrimSuffix(alByte, []byte("}")))
	if a.writer.Level >= levelRequest && len(a.reqBody) > 0 {
		buffer.WriteString(`,"requestBody":`)
		buffer.Write(bytes.TrimSuffix(a.reqBody, []byte("\n")))
	}
	if a.writer.Level >= levelRequestResponse && resHeaders.Get("Content-Type") == contentTypeJSON && len(resBody) > 0 {
		buffer.WriteString(`,"responseBody":`)
		buffer.Write(bytes.TrimSuffix(resBody, []byte("\n")))
	}
	buffer.WriteString("}")

	var compactBuffer bytes.Buffer
	err = json.Compact(&compactBuffer, buffer.Bytes())
	if err != nil {
		return errors.Wrap(err, "compact audit log json failed")
	}

	compactBuffer.WriteString("\n")
	_, err = a.writer.Output.Write(compactBuffer.Bytes())
	return err
}

func readBodyWithoutLosingContent(req *http.Request) ([]byte, error) {
	if !bodyMethods[req.Method] {
		return nil, nil
	}

	bodyBytes, err := ioutil.ReadAll(req.Body)
	if err != nil {
		return nil, err
	}
	req.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes))

	return bodyBytes, nil
}

func filterOutHeaders(headers http.Header, filterKeys []string) map[string][]string {
	newHeader := make(map[string][]string)
	for k, v := range headers {
		if isExist(filterKeys, k) {
			continue
		}
		newHeader[k] = v
	}
	return newHeader
}

func isExist(array []string, key string) bool {
	for _, v := range array {
		if v == key {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

//go:generate bash -c "echo -en '// AUTOGENERATED FILE\n\n' > linux_generated.go"
//go:generate bash -c "echo -en 'package build\n\n' >> linux_generated.go"
//go:generate bash -c "echo -en 'const createImageScript = `#!/bin/bash\n' >> linux_generated.go"
//go:generate bash -c "cat ../../tools/create-gce-image.sh | grep -v '#' >> linux_generated.go"
//go:generate bash -c "echo -en '`\n\n' >> linux_generated.go"

package build

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"time"

	"github.com/google/syzkaller/pkg/osutil"
)

type linux struct{}

func (linux linux) build(targetArch, vmType, kernelDir, outputDir, compiler, userspaceDir,
	cmdlineFile, sysctlFile string, config []byte) error {
	if err := linux.buildKernel(kernelDir, outputDir, compiler, config); err != nil {
		return err
	}
	if err := linux.createImage(vmType, kernelDir, outputDir, userspaceDir, cmdlineFile, sysctlFile); err != nil {
		return err
	}
	return nil
}

func (linux) buildKernel(kernelDir, outputDir, compiler string, config []byte) error {
	configFile := filepath.Join(kernelDir, ".config")
	if err := osutil.WriteFile(configFile, config); err != nil {
		return fmt.Errorf("failed to write config file: %v", err)
	}
	if err := osutil.SandboxChown(configFile); err != nil {
		return err
	}
	// One would expect olddefconfig here, but olddefconfig is not present in v3.6 and below.
	// oldconfig is the same as olddefconfig if stdin is not set.
	// Note: passing in compiler is important since 4.17 (at the very least it's noted in the config).
	cmd := osutil.Command("make", "oldconfig", "CC="+compiler)
	if err := osutil.Sandbox(cmd, true, true); err != nil {
		return err
	}
	cmd.Dir = kernelDir
	if _, err := osutil.Run(10*time.Minute, cmd); err != nil {
		return err
	}
	// Write updated kernel config early, so that it's captured on build failures.
	outputConfig := filepath.Join(outputDir, "kernel.config")
	if err := osutil.CopyFile(configFile, outputConfig); err != nil {
		return err
	}
	// We build only bzImage as we currently don't use modules.
	cpu := strconv.Itoa(runtime.NumCPU())
	cmd = osutil.Command("make", "bzImage", "-j", cpu, "CC="+compiler)
	if err := osutil.Sandbox(cmd, true, true); err != nil {
		return err
	}
	cmd.Dir = kernelDir
	if _, err := osutil.Run(time.Hour, cmd); err != nil {
		return err
	}
	vmlinux := filepath.Join(kernelDir, "vmlinux")
	outputVmlinux := filepath.Join(outputDir, "obj", "vmlinux")
	if err := osutil.Rename(vmlinux, outputVmlinux); err != nil {
		return fmt.Errorf("failed to rename vmlinux: %v", err)
	}
	return nil
}

func (linux) createImage(vmType, kernelDir, outputDir, userspaceDir, cmdlineFile, sysctlFile string) error {
	tempDir, err := ioutil.TempDir("", "syz-build")
	if err != nil {
		return err
	}
	defer os.RemoveAll(tempDir)
	scriptFile := filepath.Join(tempDir, "create.sh")
	if err := osutil.WriteExecFile(scriptFile, []byte(createImageScript)); err != nil {
		return fmt.Errorf("failed to write script file: %v", err)
	}
	bzImage := filepath.Join(kernelDir, filepath.FromSlash("arch/x86/boot/bzImage"))
	cmd := osutil.Command(scriptFile, userspaceDir, bzImage)
	cmd.Dir = tempDir
	cmd.Env = append([]string{}, os.Environ()...)
	cmd.Env = append(cmd.Env,
		"SYZ_VM_TYPE="+vmType,
		"SYZ_CMDLINE_FILE="+osutil.Abs(cmdlineFile),
		"SYZ_SYSCTL_FILE="+osutil.Abs(sysctlFile),
	)
	if _, err = osutil.Run(time.Hour, cmd); err != nil {
		return fmt.Errorf("image build failed: %v", err)
	}
	// Note: we use CopyFile instead of Rename because src and dst can be on different filesystems.
	imageFile := filepath.Join(outputDir, "image")
	if err := osutil.CopyFile(filepath.Join(tempDir, "disk.raw"), imageFile); err != nil {
		return err
	}
	keyFile := filepath.Join(outputDir, "key")
	if err := osutil.CopyFile(filepath.Join(tempDir, "key"), keyFile); err != nil {
		return err
	}
	if err := os.Chmod(keyFile, 0600); err != nil {
		return err
	}
	return nil
}

func (linux) clean(kernelDir, targetArch string) error {
	cpu := strconv.Itoa(runtime.NumCPU())
	cmd := osutil.Command("make", "distclean", "-j", cpu)
	if err := osutil.Sandbox(cmd, true, true); err != nil {
		return err
	}
	cmd.Dir = kernelDir
	_, err := osutil.Run(10*time.Minute, cmd)
	return err
}
<commit_msg>pkg/build: Call Linux image creation script with target arch argument<commit_after>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

//go:generate bash -c "echo -en '// AUTOGENERATED FILE\n\n' > linux_generated.go"
//go:generate bash -c "echo -en 'package build\n\n' >> linux_generated.go"
//go:generate bash -c "echo -en 'const createImageScript = `#!/bin/bash\n' >> linux_generated.go"
//go:generate bash -c "cat ../../tools/create-gce-image.sh | grep -v '#' >> linux_generated.go"
//go:generate bash -c "echo -en '`\n\n' >> linux_generated.go"

package build

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"time"

	"github.com/google/syzkaller/pkg/osutil"
)

type linux struct{}

func (linux linux) build(targetArch, vmType, kernelDir, outputDir, compiler, userspaceDir,
	cmdlineFile, sysctlFile string, config []byte) error {
	if err := linux.buildKernel(kernelDir, outputDir, compiler, config); err != nil {
		return err
	}
	if err := linux.createImage(targetArch, vmType, kernelDir, outputDir, userspaceDir, cmdlineFile,
		sysctlFile); err != nil {
		return err
	}
	return nil
}

func (linux) buildKernel(kernelDir, outputDir, compiler string, config []byte) error {
	configFile := filepath.Join(kernelDir, ".config")
	if err := osutil.WriteFile(configFile, config); err != nil {
		return fmt.Errorf("failed to write config file: %v", err)
	}
	if err := osutil.SandboxChown(configFile); err != nil {
		return err
	}
	// One would expect olddefconfig here, but olddefconfig is not present in v3.6 and below.
	// oldconfig is the same as olddefconfig if stdin is not set.
	// Note: passing in compiler is important since 4.17 (at the very least it's noted in the config).
	cmd := osutil.Command("make", "oldconfig", "CC="+compiler)
	if err := osutil.Sandbox(cmd, true, true); err != nil {
		return err
	}
	cmd.Dir = kernelDir
	if _, err := osutil.Run(10*time.Minute, cmd); err != nil {
		return err
	}
	// Write updated kernel config early, so that it's captured on build failures.
	outputConfig := filepath.Join(outputDir, "kernel.config")
	if err := osutil.CopyFile(configFile, outputConfig); err != nil {
		return err
	}
	// We build only bzImage as we currently don't use modules.
	cpu := strconv.Itoa(runtime.NumCPU())
	cmd = osutil.Command("make", "bzImage", "-j", cpu, "CC="+compiler)
	if err := osutil.Sandbox(cmd, true, true); err != nil {
		return err
	}
	cmd.Dir = kernelDir
	if _, err := osutil.Run(time.Hour, cmd); err != nil {
		return err
	}
	vmlinux := filepath.Join(kernelDir, "vmlinux")
	outputVmlinux := filepath.Join(outputDir, "obj", "vmlinux")
	if err := osutil.Rename(vmlinux, outputVmlinux); err != nil {
		return fmt.Errorf("failed to rename vmlinux: %v", err)
	}
	return nil
}

func (linux) createImage(targetArch, vmType, kernelDir, outputDir, userspaceDir, cmdlineFile, sysctlFile string) error {
	tempDir, err := ioutil.TempDir("", "syz-build")
	if err != nil {
		return err
	}
	defer os.RemoveAll(tempDir)
	scriptFile := filepath.Join(tempDir, "create.sh")
	if err := osutil.WriteExecFile(scriptFile, []byte(createImageScript)); err != nil {
		return fmt.Errorf("failed to write script file: %v", err)
	}
	bzImage := filepath.Join(kernelDir, filepath.FromSlash("arch/x86/boot/bzImage"))
	cmd := osutil.Command(scriptFile, userspaceDir, bzImage, targetArch)
	cmd.Dir = tempDir
	cmd.Env = append([]string{}, os.Environ()...)
	cmd.Env = append(cmd.Env,
		"SYZ_VM_TYPE="+vmType,
		"SYZ_CMDLINE_FILE="+osutil.Abs(cmdlineFile),
		"SYZ_SYSCTL_FILE="+osutil.Abs(sysctlFile),
	)
	if _, err = osutil.Run(time.Hour, cmd); err != nil {
		return fmt.Errorf("image build failed: %v", err)
	}
	// Note: we use CopyFile instead of Rename because src and dst can be on different filesystems.
	imageFile := filepath.Join(outputDir, "image")
	if err := osutil.CopyFile(filepath.Join(tempDir, "disk.raw"), imageFile); err != nil {
		return err
	}
	keyFile := filepath.Join(outputDir, "key")
	if err := osutil.CopyFile(filepath.Join(tempDir, "key"), keyFile); err != nil {
		return err
	}
	if err := os.Chmod(keyFile, 0600); err != nil {
		return err
	}
	return nil
}

func (linux) clean(kernelDir, targetArch string) error {
	cpu := strconv.Itoa(runtime.NumCPU())
	cmd := osutil.Command("make", "distclean", "-j", cpu)
	if err := osutil.Sandbox(cmd, true, true); err != nil {
		return err
	}
	cmd.Dir = kernelDir
	_, err := osutil.Run(10*time.Minute, cmd)
	return err
}
<|endoftext|>


<commit_before>package data

import (
	"fmt"
	"strconv"

	pb "rsprd.com/spread/pkg/spreadproto"
)

func ResolveRelativeField(field *pb.Field, fieldpath string) (resolvedField *pb.Field, err error) {
	fieldKey, arrIndex, next := nextField(fieldpath)
	if arrIndex >= 0 {
		resolvedField, err = getFromArrayField(field, arrIndex)
	} else if len(fieldKey) > 0 {
		resolvedField, err = getFromMapField(field, fieldKey)
	} else {
		err = fmt.Errorf("could not resolve fieldpath '%s'", fieldpath)
	}

	// return if err or no more fields to traverse (end of field path)
	if err != nil || len(next) == 0 {
		return
	}
	return ResolveRelativeField(resolvedField, next)
}

func getFromArrayField(field *pb.Field, index int) (*pb.Field, error) {
	fieldArr := field.GetArray()
	if fieldArr == nil {
		return nil, fmt.Errorf("field '%s' isn't an array, cannot access %s[%d]", field.Key, field.Key, index)
	}

	items := fieldArr.GetItems()
	if items == nil {
		return nil, fmt.Errorf("the array wrapper struct for the value of field '%s' had nil for items, cannot access %s[%d]", field.Key, field.Key, index)
	} else if len(items)-1 < index {
		return nil, fmt.Errorf("could not access %s[%d], the size of '%s' is %d", field.Key, index, field.Key, len(items))
	}
	return items[index], nil
}

func getFromMapField(field *pb.Field, key string) (*pb.Field, error) {
	fieldMap := field.GetObject()
	if fieldMap == nil {
		return nil, fmt.Errorf("field '%s' isn't an object, cannot access %s['%s']", field.Key, field.Key, key)
	}

	items := fieldMap.GetItems()
	if items == nil {
		return nil, fmt.Errorf("the object wrapper struct for the value of field '%s' had nil for items, cannot access %s[%d]", field.Key, field.Key, key)
	}

	item, ok := items[key]
	if !ok {
		return nil, fmt.Errorf("no key '%s' in map for field '%s", key, field.Key)
	}
	return item, nil
}

// nextField returns the first field in a fieldpath and returns the remainder after removing the root element.
// It will return array as positive number or 0 if refers to array. If there is no next field, an empty string in field will be returned.
func nextField(fieldpath string) (field string, array int, next string) {
	array = -1
	fieldpath, err := ValidateField(fieldpath)
	if err != nil {
		return
	}

	for i, c := range fieldpath {
		// check for end of field chars
		if (c == '.' || c == '(') && i > 0 {
			field = fieldpath[:i]
			if c == '.' {
				i++
			}

			if len(fieldpath) > i+1 {
				next = fieldpath[i:]
			}
			return
		} else if c == ')' {
			indexStr := fieldpath[1:i]
			if len(fieldpath) > i+2 {
				next = fieldpath[i+1:]
			}

			if num, err := strconv.ParseInt(indexStr, 10, 64); err == nil {
				array = int(num)
			}
			return
		} else if i+1 == len(fieldpath) {
			field = fieldpath
			return
		}
	}
	return
}
<commit_msg>fixed improper type in error formatting<commit_after>package data

import (
	"fmt"
	"strconv"

	pb "rsprd.com/spread/pkg/spreadproto"
)

func ResolveRelativeField(field *pb.Field, fieldpath string) (resolvedField *pb.Field, err error) {
	fieldKey, arrIndex, next := nextField(fieldpath)
	if arrIndex >= 0 {
		resolvedField, err = getFromArrayField(field, arrIndex)
	} else if len(fieldKey) > 0 {
		resolvedField, err = getFromMapField(field, fieldKey)
	} else {
		err = fmt.Errorf("could not resolve fieldpath '%s'", fieldpath)
	}

	// return if err or no more fields to traverse (end of field path)
	if err != nil || len(next) == 0 {
		return
	}
	return ResolveRelativeField(resolvedField, next)
}

func getFromArrayField(field *pb.Field, index int) (*pb.Field, error) {
	fieldArr := field.GetArray()
	if fieldArr == nil {
		return nil, fmt.Errorf("field '%s' isn't an array, cannot access %s[%d]", field.Key, field.Key, index)
	}

	items := fieldArr.GetItems()
	if items == nil {
		return nil, fmt.Errorf("the array wrapper struct for the value of field '%s' had nil for items, cannot access %s[%d]", field.Key, field.Key, index)
	} else if len(items)-1 < index {
		return nil, fmt.Errorf("could not access %s[%d], the size of '%s' is %d", field.Key, index, field.Key, len(items))
	}
	return items[index], nil
}

func getFromMapField(field *pb.Field, key string) (*pb.Field, error) {
	fieldMap := field.GetObject()
	if fieldMap == nil {
		return nil, fmt.Errorf("field '%s' isn't an object, cannot access %s['%s']", field.Key, field.Key, key)
	}

	items := fieldMap.GetItems()
	if items == nil {
		return nil, fmt.Errorf("the object wrapper struct for the value of field '%s' had nil for items, cannot access %s[%s]", field.Key, field.Key, key)
	}

	item, ok := items[key]
	if !ok {
		return nil, fmt.Errorf("no key '%s' in map for field '%s", key, field.Key)
	}
	return item, nil
}

// nextField returns the first field in a fieldpath and returns the remainder after removing the root element.
// It will return array as positive number or 0 if refers to array. If there is no next field, an empty string in field will be returned.
func nextField(fieldpath string) (field string, array int, next string) {
	array = -1
	fieldpath, err := ValidateField(fieldpath)
	if err != nil {
		return
	}

	for i, c := range fieldpath {
		// check for end of field chars
		if (c == '.' || c == '(') && i > 0 {
			field = fieldpath[:i]
			if c == '.' {
				i++
			}

			if len(fieldpath) > i+1 {
				next = fieldpath[i:]
			}
			return
		} else if c == ')' {
			indexStr := fieldpath[1:i]
			if len(fieldpath) > i+2 {
				next = fieldpath[i+1:]
			}

			if num, err := strconv.ParseInt(indexStr, 10, 64); err == nil {
				array = int(num)
			}
			return
		} else if i+1 == len(fieldpath) {
			field = fieldpath
			return
		}
	}
	return
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"context"
	"fmt"
	"time"

	"github.com/golang/glog"
	"k8s.io/api/extensions/v1beta1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/ingress-gce/pkg/fuzz"
	"k8s.io/ingress-gce/pkg/fuzz/features"
	"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
)

const (
	ingressPollInterval = 30 * time.Second
	ingressPollTimeout  = 20 * time.Minute

	gclbDeletionInterval = 30 * time.Second
	gclbDeletionTimeout  = 5 * time.Minute
)

// WaitForIngress to stabilize.
func WaitForIngress(s *Sandbox, ing *v1beta1.Ingress) (*v1beta1.Ingress, error) {
	err := wait.Poll(ingressPollInterval, ingressPollTimeout, func() (bool, error) {
		var err error
		ing, err = s.f.Clientset.Extensions().Ingresses(s.Namespace).Get(ing.Name, metav1.GetOptions{})
		if err != nil {
			return true, err
		}
		validator, err := fuzz.NewIngressValidator(s.ValidatorEnv, ing, features.All, nil)
		if err != nil {
			return true, err
		}
		result := validator.Check(context.Background())
		if result.Err == nil {
			return true, nil
		}
		return false, nil
	})
	return ing, err
}

// WaitForIngressDeletion deletes the given ingress and waits for the
// resources associated with it to be deleted.
func WaitForIngressDeletion(ctx context.Context, g *fuzz.GCLB, s *Sandbox, ing *v1beta1.Ingress, options *fuzz.GCLBDeleteOptions) error {
	if err := s.f.Clientset.Extensions().Ingresses(s.Namespace).Delete(ing.Name, &metav1.DeleteOptions{}); err != nil {
		return fmt.Errorf("Delete(%q) = %v, want nil", ing.Name, err)
	}
	glog.Infof("Waiting for GCLB resources to be deleted (%s/%s), IngressDeletionOptions=%+v", s.Namespace, ing.Name, options)
	if err := WaitForGCLBDeletion(ctx, s.f.Cloud, g, options); err != nil {
		return fmt.Errorf("WaitForGCLBDeletion(...) = %v, want nil", err)
	}
	glog.Infof("GCLB resources deleted (%s/%s)", s.Namespace, ing.Name)
	return nil
}

// WaitForGCLBDeletion waits for the resources associated with the GLBC to be
// deleted.
func WaitForGCLBDeletion(ctx context.Context, c cloud.Cloud, g *fuzz.GCLB, options *fuzz.GCLBDeleteOptions) error {
	return wait.Poll(gclbDeletionInterval, gclbDeletionTimeout, func() (bool, error) {
		if err := g.CheckResourceDeletion(ctx, c, options); err != nil {
			glog.Infof("WaitForGCLBDeletion(%q) = %v", g.VIP, err)
			return false, nil
		}
		return true, nil
	})
}
<commit_msg>Increase timeout on waiting for GLBC resource deletion<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"context"
	"fmt"
	"time"

	"github.com/golang/glog"
	"k8s.io/api/extensions/v1beta1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/ingress-gce/pkg/fuzz"
	"k8s.io/ingress-gce/pkg/fuzz/features"
	"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
)

const (
	ingressPollInterval = 30 * time.Second
	ingressPollTimeout  = 20 * time.Minute

	gclbDeletionInterval = 30 * time.Second
	gclbDeletionTimeout  = 10 * time.Minute
)

// WaitForIngress to stabilize.
func WaitForIngress(s *Sandbox, ing *v1beta1.Ingress) (*v1beta1.Ingress, error) {
	err := wait.Poll(ingressPollInterval, ingressPollTimeout, func() (bool, error) {
		var err error
		ing, err = s.f.Clientset.Extensions().Ingresses(s.Namespace).Get(ing.Name, metav1.GetOptions{})
		if err != nil {
			return true, err
		}
		validator, err := fuzz.NewIngressValidator(s.ValidatorEnv, ing, features.All, nil)
		if err != nil {
			return true, err
		}
		result := validator.Check(context.Background())
		if result.Err == nil {
			return true, nil
		}
		return false, nil
	})
	return ing, err
}

// WaitForIngressDeletion deletes the given ingress and waits for the
// resources associated with it to be deleted.
func WaitForIngressDeletion(ctx context.Context, g *fuzz.GCLB, s *Sandbox, ing *v1beta1.Ingress, options *fuzz.GCLBDeleteOptions) error {
	if err := s.f.Clientset.Extensions().Ingresses(s.Namespace).Delete(ing.Name, &metav1.DeleteOptions{}); err != nil {
		return fmt.Errorf("Delete(%q) = %v, want nil", ing.Name, err)
	}
	glog.Infof("Waiting for GCLB resources to be deleted (%s/%s), IngressDeletionOptions=%+v", s.Namespace, ing.Name, options)
	if err := WaitForGCLBDeletion(ctx, s.f.Cloud, g, options); err != nil {
		return fmt.Errorf("WaitForGCLBDeletion(...) = %v, want nil", err)
	}
	glog.Infof("GCLB resources deleted (%s/%s)", s.Namespace, ing.Name)
	return nil
}

// WaitForGCLBDeletion waits for the resources associated with the GLBC to be
// deleted.
func WaitForGCLBDeletion(ctx context.Context, c cloud.Cloud, g *fuzz.GCLB, options *fuzz.GCLBDeleteOptions) error {
	return wait.Poll(gclbDeletionInterval, gclbDeletionTimeout, func() (bool, error) {
		if err := g.CheckResourceDeletion(ctx, c, options); err != nil {
			glog.Infof("WaitForGCLBDeletion(%q) = %v", g.VIP, err)
			return false, nil
		}
		return true, nil
	})
}
<|endoftext|>


<commit_before>package jobs

import (
	"context"
	"fmt"
	"math/rand"
	"time"

	log "github.com/Sirupsen/logrus"
)

// contextKey are the keys used in the worker context
type contextKey int

const (
	// ContextDomainKey is the used to store the domain string name
	ContextDomainKey contextKey = iota
	// ContextWorkerKey is the used to store the workerID string
	ContextWorkerKey
)

var (
	defaultConcurrency  = 1
	defaultMaxExecCount = 3
	defaultMaxExecTime  = 60 * time.Second
	defaultRetryDelay   = 60 * time.Millisecond
	defaultTimeout      = 10 * time.Second
)

type (
	// WorkerFunc represent the work function that a worker should implement.
	WorkerFunc func(context context.Context, msg *Message) error

	// WorkerCommit is an optional method that is always called once after the
	// execution of the WorkerFunc.
	WorkerCommit func(context context.Context, msg *Message, errjob error) error

	// WorkerConfig is the configuration parameter of a worker defined by the job
	// system. It contains parameters of the worker along with the worker main
	// function that perform the work against a job's message.
	WorkerConfig struct {
		WorkerFunc   WorkerFunc
		WorkerCommit WorkerCommit
		Concurrency  uint          `json:"concurrency"`
		MaxExecCount uint          `json:"max_exec_count"`
		MaxExecTime  time.Duration `json:"max_exec_time"`
		Timeout      time.Duration `json:"timeout"`
		RetryDelay   time.Duration `json:"retry_delay"`
	}

	// Worker is a unit of work that will consume from a queue and execute the do
	// method for each jobs it pulls.
	Worker struct {
		Type string
		Conf *WorkerConfig
		jobs chan Job
	}
)

// NewWorkerContext returns a context.Context usable by a worker.
func NewWorkerContext(domain, workerID string) context.Context {
	ctx := context.Background()
	ctx = context.WithValue(ctx, ContextDomainKey, domain)
	ctx = context.WithValue(ctx, ContextWorkerKey, workerID)
	return ctx
}

// Start is used to start the worker consumption of messages from its queue.
func (w *Worker) Start(jobs chan Job) {
	w.jobs = jobs
	for i := 0; i < int(w.Conf.Concurrency); i++ {
		name := fmt.Sprintf("%s/%d", w.Type, i)
		go w.work(name)
	}
}

func (w *Worker) work(workerID string) {
	// TODO: err handling and persistence
	for job := range w.jobs {
		domain := job.Domain()
		if domain == "" {
			log.Errorf("[job] %s: missing domain from job request", workerID)
			continue
		}
		parentCtx := NewWorkerContext(domain, workerID)
		infos := job.Infos()
		if err := job.AckConsumed(); err != nil {
			log.Errorf("[job] %s: error acking consume job %s: %s",
				workerID, infos.ID(), err.Error())
			continue
		}
		t := &task{
			ctx:      parentCtx,
			infos:    infos,
			conf:     w.defaultedConf(infos.Options),
			workerID: workerID,
		}
		var err error
		if err = t.run(); err != nil {
			log.Errorf("[job] %s: error while performing job %s: %s",
				workerID, infos.ID(), err.Error())
			err = job.Nack(err)
		} else {
			err = job.Ack()
		}
		if err != nil {
			log.Errorf("[job] %s: error while acking job done %s: %s",
				workerID, infos.ID(), err.Error())
		}
	}
}

func (w *Worker) defaultedConf(opts *JobOptions) *WorkerConfig {
	c := w.Conf.clone()
	if c.Concurrency == 0 {
		c.Concurrency = uint(defaultConcurrency)
	}
	if c.MaxExecCount == 0 {
		c.MaxExecCount = uint(defaultMaxExecCount)
	}
	if c.MaxExecTime == 0 {
		c.MaxExecTime = defaultMaxExecTime
	}
	if c.RetryDelay == 0 {
		c.RetryDelay = defaultRetryDelay
	}
	if c.Timeout == 0 {
		c.Timeout = defaultTimeout
	}
	if opts == nil {
		return c
	}
	if opts.MaxExecCount != 0 && opts.MaxExecCount < c.MaxExecCount {
		c.MaxExecCount = opts.MaxExecCount
	}
	if opts.MaxExecTime > 0 && opts.MaxExecTime < c.MaxExecTime {
		c.MaxExecTime = opts.MaxExecTime
	}
	if opts.Timeout > 0 && opts.Timeout < c.Timeout {
		c.Timeout = opts.Timeout
	}
	return c
}

type task struct {
	ctx   context.Context
	infos *JobInfos
	conf  *WorkerConfig

	workerID  string
	startTime time.Time
	execCount uint
}

func (t *task) run() (err error) {
	t.startTime = time.Now()
	t.execCount = 0
	defer func() {
		if t.conf.WorkerCommit != nil {
			if errc := t.conf.WorkerCommit(t.ctx, t.infos.Message, err); errc != nil {
				log.Warnf("[job] %s: error while commiting job %s: %s",
					t.workerID, t.infos.ID(), errc.Error())
			}
		}
	}()
	for {
		retry, delay, timeout := t.nextDelay()
		if !retry {
			return err
		}
		if err != nil {
			log.Warnf("[job] %s: error while performing job %s: %s (retry in %s)",
				t.workerID, t.infos.ID(), err.Error(), delay)
		}
		if delay > 0 {
			time.Sleep(delay)
		}
		log.Debugf("[job] %s: executing job %s(%d) (timeout %s)",
			t.workerID, t.infos.ID(), t.execCount, timeout)
		ctx, cancel := context.WithTimeout(t.ctx, timeout)
		if err = t.exec(ctx); err == nil {
			cancel()
			break
		}
		// Even though ctx should have expired already, it is good practice to call
		// its cancelation function in any case. Failure to do so may keep the
		// context and its parent alive longer than necessary.
		cancel()
		t.execCount++
	}
	return nil
}

func (t *task) exec(ctx context.Context) (err error) {
	defer func() {
		if r := recover(); r != nil {
			var ok bool
			err, ok = r.(error)
			if !ok {
				err = fmt.Errorf("%v", r)
			}
		}
	}()
	return t.conf.WorkerFunc(ctx, t.infos.Message)
}

func (t *task) nextDelay() (bool, time.Duration, time.Duration) {
	c := t.conf
	execTime := time.Since(t.startTime)

	if t.execCount >= c.MaxExecCount || execTime > c.MaxExecTime {
		return false, 0, 0
	}

	// the worker timeout should take into account the maximum execution time
	// allowed to the task
	timeout := c.Timeout
	if execTime+timeout > c.MaxExecTime {
		timeout = c.MaxExecTime - execTime
	}

	var nextDelay time.Duration
	if t.execCount == 0 {
		// on first execution, execute immediately
		nextDelay = 0
	} else {
		nextDelay = c.RetryDelay << (t.execCount - 1)

		// fuzzDelay number between delay * (1 +/- 0.1)
		fuzzDelay := int(0.1 * float64(nextDelay))
		nextDelay = nextDelay + time.Duration((rand.Intn(2*fuzzDelay) - fuzzDelay))
	}

	if execTime+nextDelay > c.MaxExecTime {
		return false, 0, 0
	}

	return true, nextDelay, timeout
}
<commit_msg>Remove a TODO<commit_after>package jobs

import (
	"context"
	"fmt"
	"math/rand"
	"time"

	log "github.com/Sirupsen/logrus"
)

// contextKey are the keys used in the worker context
type contextKey int

const (
	// ContextDomainKey is the used to store the domain string name
	ContextDomainKey contextKey = iota
	// ContextWorkerKey is the used to store the workerID string
	ContextWorkerKey
)

var (
	defaultConcurrency  = 1
	defaultMaxExecCount = 3
	defaultMaxExecTime  = 60 * time.Second
	defaultRetryDelay   = 60 * time.Millisecond
	defaultTimeout      = 10 * time.Second
)

type (
	// WorkerFunc represent the work function that a worker should implement.
	WorkerFunc func(context context.Context, msg *Message) error

	// WorkerCommit is an optional method that is always called once after the
	// execution of the WorkerFunc.
	WorkerCommit func(context context.Context, msg *Message, errjob error) error

	// WorkerConfig is the configuration parameter of a worker defined by the job
	// system. It contains parameters of the worker along with the worker main
	// function that perform the work against a job's message.
	WorkerConfig struct {
		WorkerFunc   WorkerFunc
		WorkerCommit WorkerCommit
		Concurrency  uint          `json:"concurrency"`
		MaxExecCount uint          `json:"max_exec_count"`
		MaxExecTime  time.Duration `json:"max_exec_time"`
		Timeout      time.Duration `json:"timeout"`
		RetryDelay   time.Duration `json:"retry_delay"`
	}

	// Worker is a unit of work that will consume from a queue and execute the do
	// method for each jobs it pulls.
	Worker struct {
		Type string
		Conf *WorkerConfig
		jobs chan Job
	}
)

// NewWorkerContext returns a context.Context usable by a worker.
func NewWorkerContext(domain, workerID string) context.Context {
	ctx := context.Background()
	ctx = context.WithValue(ctx, ContextDomainKey, domain)
	ctx = context.WithValue(ctx, ContextWorkerKey, workerID)
	return ctx
}

// Start is used to start the worker consumption of messages from its queue.
func (w *Worker) Start(jobs chan Job) {
	w.jobs = jobs
	for i := 0; i < int(w.Conf.Concurrency); i++ {
		name := fmt.Sprintf("%s/%d", w.Type, i)
		go w.work(name)
	}
}

func (w *Worker) work(workerID string) {
	for job := range w.jobs {
		domain := job.Domain()
		if domain == "" {
			log.Errorf("[job] %s: missing domain from job request", workerID)
			continue
		}
		parentCtx := NewWorkerContext(domain, workerID)
		infos := job.Infos()
		if err := job.AckConsumed(); err != nil {
			log.Errorf("[job] %s: error acking consume job %s: %s",
				workerID, infos.ID(), err.Error())
			continue
		}
		t := &task{
			ctx:      parentCtx,
			infos:    infos,
			conf:     w.defaultedConf(infos.Options),
			workerID: workerID,
		}
		var err error
		if err = t.run(); err != nil {
			log.Errorf("[job] %s: error while performing job %s: %s",
				workerID, infos.ID(), err.Error())
			err = job.Nack(err)
		} else {
			err = job.Ack()
		}
		if err != nil {
			log.Errorf("[job] %s: error while acking job done %s: %s",
				workerID, infos.ID(), err.Error())
		}
	}
}

func (w *Worker) defaultedConf(opts *JobOptions) *WorkerConfig {
	c := w.Conf.clone()
	if c.Concurrency == 0 {
		c.Concurrency = uint(defaultConcurrency)
	}
	if c.MaxExecCount == 0 {
		c.MaxExecCount = uint(defaultMaxExecCount)
	}
	if c.MaxExecTime == 0 {
		c.MaxExecTime = defaultMaxExecTime
	}
	if c.RetryDelay == 0 {
		c.RetryDelay = defaultRetryDelay
	}
	if c.Timeout == 0 {
		c.Timeout = defaultTimeout
	}
	if opts == nil {
		return c
	}
	if opts.MaxExecCount != 0 && opts.MaxExecCount < c.MaxExecCount {
		c.MaxExecCount = opts.MaxExecCount
	}
	if opts.MaxExecTime > 0 && opts.MaxExecTime < c.MaxExecTime {
		c.MaxExecTime = opts.MaxExecTime
	}
	if opts.Timeout > 0 && opts.Timeout < c.Timeout {
		c.Timeout = opts.Timeout
	}
	return c
}

type task struct {
	ctx   context.Context
	infos *JobInfos
	conf  *WorkerConfig

	workerID  string
	startTime time.Time
	execCount uint
}

func (t *task) run() (err error) {
	t.startTime = time.Now()
	t.execCount = 0
	defer func() {
		if t.conf.WorkerCommit != nil {
			if errc := t.conf.WorkerCommit(t.ctx, t.infos.Message, err); errc != nil {
				log.Warnf("[job] %s: error while commiting job %s: %s",
					t.workerID, t.infos.ID(), errc.Error())
			}
		}
	}()
	for {
		retry, delay, timeout := t.nextDelay()
		if !retry {
			return err
		}
		if err != nil {
			log.Warnf("[job] %s: error while performing job %s: %s (retry in %s)",
				t.workerID, t.infos.ID(), err.Error(), delay)
		}
		if delay > 0 {
			time.Sleep(delay)
		}
		log.Debugf("[job] %s: executing job %s(%d) (timeout %s)",
			t.workerID, t.infos.ID(), t.execCount, timeout)
		ctx, cancel := context.WithTimeout(t.ctx, timeout)
		if err = t.exec(ctx); err == nil {
			cancel()
			break
		}
		// Even though ctx should have expired already, it is good practice to call
		// its cancelation function in any case. Failure to do so may keep the
		// context and its parent alive longer than necessary.
		cancel()
		t.execCount++
	}
	return nil
}

func (t *task) exec(ctx context.Context) (err error) {
	defer func() {
		if r := recover(); r != nil {
			var ok bool
			err, ok = r.(error)
			if !ok {
				err = fmt.Errorf("%v", r)
			}
		}
	}()
	return t.conf.WorkerFunc(ctx, t.infos.Message)
}

func (t *task) nextDelay() (bool, time.Duration, time.Duration) {
	c := t.conf
	execTime := time.Since(t.startTime)

	if t.execCount >= c.MaxExecCount || execTime > c.MaxExecTime {
		return false, 0, 0
	}

	// the worker timeout should take into account the maximum execution time
	// allowed to the task
	timeout := c.Timeout
	if execTime+timeout > c.MaxExecTime {
		timeout = c.MaxExecTime - execTime
	}

	var nextDelay time.Duration
	if t.execCount == 0 {
		// on first execution, execute immediately
		nextDelay = 0
	} else {
		nextDelay = c.RetryDelay << (t.execCount - 1)

		// fuzzDelay number between delay * (1 +/- 0.1)
		fuzzDelay := int(0.1 * float64(nextDelay))
		nextDelay = nextDelay + time.Duration((rand.Intn(2*fuzzDelay) - fuzzDelay))
	}

	if execTime+nextDelay > c.MaxExecTime {
		return false, 0, 0
	}

	return true, nextDelay, timeout
}
<|endoftext|>


<commit_before>// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package nginx contains nginx buildpack library code.
package nginx

import (
	"text/template"
)

// PHPFpmTemplate is a template that produces a snippet of php-fpm config that sets up the PHP with Nginx.
var PHPFpmTemplate = template.Must(template.New("phpfpm").Parse(`
; Send errors to stderr.
error_log = /proc/self/fd/2

log_level = warning

pid = {{.PidPath}}

; Pool configuration
[app]

; Unix user/group of processes
user = {{.Username}}
group = {{.Username}}

; The address on which to accept FastCGI requests
listen = {{.ListenAddress}}

{{if .DynamicWorkers}}
; Create child processes with a dynamic policy.
pm = dynamic

; The number of child processes to be created
pm.start_servers = 1
pm.min_spare_servers = 1
pm.max_spare_servers = {{.NumWorkers}}
pm.max_children = {{.NumWorkers}}
{{else}}
; Create child processes with a static policy.
pm = static

; The number of child processes to be created
pm.max_children = {{.NumWorkers}}
{{end}}

; Keep the environment variables of the parent.
clear_env = no

catch_workers_output = yes
`))

// NginxTemplate is a template that produces a snippet of nginx config that sets up the
// upstream and server for PHP. It is included in the http{} section of the config by
// the pid1 program.
var NginxTemplate = template.Must(template.New("nginx").Parse(`
fastcgi_read_timeout 24h;

# proxy_* are not set for PHP because fastcgi is used.

upstream fast_cgi_app {
	server         unix:{{.AppListenAddress}} fail_timeout=0;
}

server {
	listen	{{.Port}} default_server;
	listen	[::]:{{.Port}} default_server;
	server_name	"";
	root	{{.Root}};

	rewrite	^/(.*)$	/{{.FrontControllerScript}}$uri;

	location	~	^/{{.FrontControllerScript}}	{
		error_log stderr;

		fastcgi_pass	fast_cgi_app;
		fastcgi_buffering	off;
		fastcgi_request_buffering	off;
		fastcgi_cache	off;
		fastcgi_store	off;
		fastcgi_intercept_errors	off;

		fastcgi_index	index.php;
		fastcgi_split_path_info	^(.+\.php)(.*)$;

		fastcgi_param	QUERY_STRING	$query_string;
		fastcgi_param	REQUEST_METHOD	$request_method;
		fastcgi_param	CONTENT_TYPE	$content_type;
		fastcgi_param	CONTENT_LENGTH	$content_length;

		fastcgi_param	SCRIPT_NAME	$fastcgi_script_name;
		fastcgi_param	SCRIPT_FILENAME	$document_root/{{.FrontControllerScript}};
		fastcgi_param	PATH_INFO	$fastcgi_path_info;
		fastcgi_param	REQUEST_URI	$request_uri;
		fastcgi_param	DOCUMENT_URI	$fastcgi_script_name;
		fastcgi_param	DOCUMENT_ROOT	$document_root;
		fastcgi_param	SERVER_PROTOCOL	$server_protocol;
		fastcgi_param	REQUEST_SCHEME	$scheme;
		if ($http_x_forwarded_proto = 'https') {
			set $https_setting 'on';
		}
		fastcgi_param	HTTPS	$https_setting if_not_empty;

		fastcgi_param	GATEWAY_INTERFACE	CGI/1.1;
		fastcgi_param	REMOTE_ADDR	$remote_addr;
		fastcgi_param	REMOTE_PORT	$remote_port;
		fastcgi_param	REMOTE_HOST	$remote_addr;
		fastcgi_param	REMOTE_USER	$remote_user;
		fastcgi_param	SERVER_ADDR	$server_addr;
		fastcgi_param	SERVER_PORT	$server_port;
		fastcgi_param	SERVER_NAME	$server_name;
		fastcgi_param X_FORWARDED_FOR $proxy_add_x_forwarded_for;
		fastcgi_param X_FORWARDED_HOST $http_x_forwarded_host;
		fastcgi_param X_FORWARDED_PROTO $http_x_forwarded_proto;
		fastcgi_param FORWARDED $http_forwarded;
	}
}
`))

// FPMConfig represents the content values of a php-fpm config file.
type FPMConfig struct {
	PidPath        string
	ListenAddress  string
	DynamicWorkers bool
	NumWorkers     int
	Username       string
}

// Config represents the content values of a nginx config file.
type Config struct {
	Port                  int
	Root                  string
	AppListenAddress      string
	FrontControllerScript string
}
<commit_msg>Do not decorate php-fpm worker output<commit_after>// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package nginx contains nginx buildpack library code.
package nginx

import (
	"text/template"
)

// PHPFpmTemplate is a template that produces a snippet of php-fpm config that sets up the PHP with Nginx.
var PHPFpmTemplate = template.Must(template.New("phpfpm").Parse(`
; Send errors to stderr.
error_log = /proc/self/fd/2

log_level = warning

pid = {{.PidPath}}

; Pool configuration
[app]

; Unix user/group of processes
user = {{.Username}}
group = {{.Username}}

; The address on which to accept FastCGI requests
listen = {{.ListenAddress}}

{{if .DynamicWorkers}}
; Create child processes with a dynamic policy.
pm = dynamic

; The number of child processes to be created
pm.start_servers = 1
pm.min_spare_servers = 1
pm.max_spare_servers = {{.NumWorkers}}
pm.max_children = {{.NumWorkers}}
{{else}}
; Create child processes with a static policy.
pm = static

; The number of child processes to be created
pm.max_children = {{.NumWorkers}}
{{end}}

; Keep the environment variables of the parent.
clear_env = no

catch_workers_output = yes
decorate_workers_output = no
`))

// NginxTemplate is a template that produces a snippet of nginx config that sets up the
// upstream and server for PHP. It is included in the http{} section of the config by
// the pid1 program.
var NginxTemplate = template.Must(template.New("nginx").Parse(`
fastcgi_read_timeout 24h;

# proxy_* are not set for PHP because fastcgi is used.

upstream fast_cgi_app {
	server         unix:{{.AppListenAddress}} fail_timeout=0;
}

server {
	listen	{{.Port}} default_server;
	listen	[::]:{{.Port}} default_server;
	server_name	"";
	root	{{.Root}};

	rewrite	^/(.*)$	/{{.FrontControllerScript}}$uri;

	location	~	^/{{.FrontControllerScript}}	{
		error_log stderr;

		fastcgi_pass	fast_cgi_app;
		fastcgi_buffering	off;
		fastcgi_request_buffering	off;
		fastcgi_cache	off;
		fastcgi_store	off;
		fastcgi_intercept_errors	off;

		fastcgi_index	index.php;
		fastcgi_split_path_info	^(.+\.php)(.*)$;

		fastcgi_param	QUERY_STRING	$query_string;
		fastcgi_param	REQUEST_METHOD	$request_method;
		fastcgi_param	CONTENT_TYPE	$content_type;
		fastcgi_param	CONTENT_LENGTH	$content_length;

		fastcgi_param	SCRIPT_NAME	$fastcgi_script_name;
		fastcgi_param	SCRIPT_FILENAME	$document_root/{{.FrontControllerScript}};
		fastcgi_param	PATH_INFO	$fastcgi_path_info;
		fastcgi_param	REQUEST_URI	$request_uri;
		fastcgi_param	DOCUMENT_URI	$fastcgi_script_name;
		fastcgi_param	DOCUMENT_ROOT	$document_root;
		fastcgi_param	SERVER_PROTOCOL	$server_protocol;
		fastcgi_param	REQUEST_SCHEME	$scheme;
		if ($http_x_forwarded_proto = 'https') {
			set $https_setting 'on';
		}
		fastcgi_param	HTTPS	$https_setting if_not_empty;

		fastcgi_param	GATEWAY_INTERFACE	CGI/1.1;
		fastcgi_param	REMOTE_ADDR	$remote_addr;
		fastcgi_param	REMOTE_PORT	$remote_port;
		fastcgi_param	REMOTE_HOST	$remote_addr;
		fastcgi_param	REMOTE_USER	$remote_user;
		fastcgi_param	SERVER_ADDR	$server_addr;
		fastcgi_param	SERVER_PORT	$server_port;
		fastcgi_param	SERVER_NAME	$server_name;
		fastcgi_param X_FORWARDED_FOR $proxy_add_x_forwarded_for;
		fastcgi_param X_FORWARDED_HOST $http_x_forwarded_host;
		fastcgi_param X_FORWARDED_PROTO $http_x_forwarded_proto;
		fastcgi_param FORWARDED $http_forwarded;
	}
}
`))

// FPMConfig represents the content values of a php-fpm config file.
type FPMConfig struct {
	PidPath        string
	ListenAddress  string
	DynamicWorkers bool
	NumWorkers     int
	Username       string
}

// Config represents the content values of a nginx config file.
type Config struct {
	Port                  int
	Root                  string
	AppListenAddress      string
	FrontControllerScript string
}
<|endoftext|>


<commit_before>// Copyright 2019 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/*
Package patch implements a simple patching mechanism for k8s resources.
Paths are specified in the form a.b.c.[key:value].d.[list_entry_value], where:
 -  [key:value] selects a list entry in list c which contains an entry with key:value
 -  [list_entry_value] selects a list entry in list d which is a regex match of list_entry_value.

Some examples are given below. Given a resource:

kind: Deployment
metadata:
  name: istio-citadel
  namespace: istio-system
a:
  b:
  - name: n1
    value: v1
  - name: n2
    list:
    - "vv1"
    - vv2=foo

values and list entries can be added, modifed or deleted.

MODIFY

1. set v1 to v1new

  path: a.b.[name:n1].value
  value: v1

2. set vv1 to vv3

  // Note the lack of quotes around vv1 (see NOTES below).
  path: a.b.[name:n2].list.[vv1]
  value: vv3

3. set vv2=foo to vv2=bar (using regex match)

  path: a.b.[name:n2].list.[vv2]
  value: vv2=bar

DELETE

1. Delete container with name: n1

  path: a.b.[name:n1]

2. Delete list value vv1

  path: a.b.[name:n2].list.[vv1]

ADD

1. Add vv3 to list

  path: a.b.[name:n2].list
  value: vv3

2. Add new key:value to container name: n1

  path: a.b.[name:n1]
  value:
    new_attr: v3

*NOTES*
- Due to loss of string quoting during unmarshaling, keys and values should not be string quoted, even if they appear
that way in the object being patched.
- [key:value] treats ':' as a special separator character. Any ':' in the key or value string must be escaped as \:.
*/
package patch

import (
	"fmt"
	"sort"
	"strings"

	"github.com/kr/pretty"
	"gopkg.in/yaml.v2"

	"istio.io/operator/pkg/apis/istio/v1alpha2"
	"istio.io/operator/pkg/object"
	"istio.io/operator/pkg/tpath"
	"istio.io/operator/pkg/util"
	"istio.io/pkg/log"
)

var (
	scope = log.RegisterScope("patch", "patch", 0)
)

// YAMLManifestPatch patches a base YAML in the given namespace with a list of overlays.
// Each overlay has the format described in the K8SObjectOverlay definition.
// It returns the patched manifest YAML.
func YAMLManifestPatch(baseYAML string, namespace string, overlays []*v1alpha2.K8SObjectOverlay) (string, error) {
	baseObjs, err := object.ParseK8sObjectsFromYAMLManifest(baseYAML)
	if err != nil {
		return "", err
	}

	bom := baseObjs.ToMap()
	oom := objectOverrideMap(overlays, namespace)

	var ret strings.Builder

	var errs util.Errors
	// Try to apply the defined overlays.
	for k, oo := range oom {
		bo := bom[k]
		if bo == nil {
			os := ""
			for k2 := range bom {
				os += k2 + "\n"
			}
			errs = util.AppendErr(errs, fmt.Errorf("overlay for %s does not match any object in output manifest:\n%s\n\nAvailable objects are:\n%s",
				k, pretty.Sprint(oo), os))
			continue
		}
		patched, err := applyPatches(bo, oo)
		if err != nil {
			errs = util.AppendErr(errs, fmt.Errorf("patch error: %s", err))
			continue
		}
		if _, err := ret.Write(patched); err != nil {
			errs = util.AppendErr(errs, fmt.Errorf("write: %s", err))
		}
		if _, err := ret.WriteString("\n---\n"); err != nil {
			errs = util.AppendErr(errs, fmt.Errorf("patch WriteString error: %s", err))
		}
	}
	// Render the remaining objects with no overlays.
	// keep the original sorted order
	keys := make([]string, 0)
	for key := range bom {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	for _, k := range keys {
		oo := bom[k]
		if oom[k] != nil {
			// Skip objects that have overlays, these were rendered above.
			continue
		}
		oy, err := oo.YAML()
		if err != nil {
			errs = util.AppendErr(errs, fmt.Errorf("object to YAML error (%s) for base object: \n%v", err, oo))
			continue
		}
		if _, err := ret.Write(oy); err != nil {
			errs = util.AppendErr(errs, fmt.Errorf("write: %s", err))
		}
		if _, err := ret.WriteString("\n---\n"); err != nil {
			errs = util.AppendErr(errs, fmt.Errorf("writeString: %s", err))
		}
	}
	return ret.String(), errs.ToError()
}

// applyPatches applies the given patches against the given object. It returns the resulting patched YAML if successful,
// or a list of errors otherwise.
func applyPatches(base *object.K8sObject, patches []*v1alpha2.K8SObjectOverlay_PathValue) (outYAML []byte, errs util.Errors) {
	bo := make(map[interface{}]interface{})
	by, err := base.YAML()
	if err != nil {
		return nil, util.NewErrs(err)
	}
	err = yaml.Unmarshal(by, bo)
	if err != nil {
		return nil, util.NewErrs(err)
	}
	for _, p := range patches {
		scope.Debugf("applying path=%s, value=%s\n", p.Path, p.Value)
		inc, _, err := tpath.GetPathContext(bo, util.PathFromString(p.Path))
		if err != nil {
			errs = util.AppendErr(errs, err)
			continue
		}
		errs = util.AppendErr(errs, tpath.WritePathContext(inc, p.Value))
	}
	oy, err := yaml.Marshal(bo)
	if err != nil {
		return nil, util.AppendErr(errs, err)
	}
	return oy, errs
}

// objectOverrideMap converts oos, a slice of object overlays, into a map of the same overlays where the key is the
// object manifest.Hash.
func objectOverrideMap(oos []*v1alpha2.K8SObjectOverlay, namespace string) map[string][]*v1alpha2.K8SObjectOverlay_PathValue {
	ret := make(map[string][]*v1alpha2.K8SObjectOverlay_PathValue)
	for _, o := range oos {
		ret[object.Hash(o.Kind, namespace, o.Name)] = o.Patches
	}
	return ret
}
<commit_msg>Fix order for overlay as well (#510)<commit_after>// Copyright 2019 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/*
Package patch implements a simple patching mechanism for k8s resources.
Paths are specified in the form a.b.c.[key:value].d.[list_entry_value], where:
 -  [key:value] selects a list entry in list c which contains an entry with key:value
 -  [list_entry_value] selects a list entry in list d which is a regex match of list_entry_value.

Some examples are given below. Given a resource:

kind: Deployment
metadata:
  name: istio-citadel
  namespace: istio-system
a:
  b:
  - name: n1
    value: v1
  - name: n2
    list:
    - "vv1"
    - vv2=foo

values and list entries can be added, modifed or deleted.

MODIFY

1. set v1 to v1new

  path: a.b.[name:n1].value
  value: v1

2. set vv1 to vv3

  // Note the lack of quotes around vv1 (see NOTES below).
  path: a.b.[name:n2].list.[vv1]
  value: vv3

3. set vv2=foo to vv2=bar (using regex match)

  path: a.b.[name:n2].list.[vv2]
  value: vv2=bar

DELETE

1. Delete container with name: n1

  path: a.b.[name:n1]

2. Delete list value vv1

  path: a.b.[name:n2].list.[vv1]

ADD

1. Add vv3 to list

  path: a.b.[name:n2].list
  value: vv3

2. Add new key:value to container name: n1

  path: a.b.[name:n1]
  value:
    new_attr: v3

*NOTES*
- Due to loss of string quoting during unmarshaling, keys and values should not be string quoted, even if they appear
that way in the object being patched.
- [key:value] treats ':' as a special separator character. Any ':' in the key or value string must be escaped as \:.
*/
package patch

import (
	"fmt"
	"sort"
	"strings"

	"github.com/kr/pretty"
	"gopkg.in/yaml.v2"

	"istio.io/operator/pkg/apis/istio/v1alpha2"
	"istio.io/operator/pkg/object"
	"istio.io/operator/pkg/tpath"
	"istio.io/operator/pkg/util"
	"istio.io/pkg/log"
)

var (
	scope = log.RegisterScope("patch", "patch", 0)
)

// YAMLManifestPatch patches a base YAML in the given namespace with a list of overlays.
// Each overlay has the format described in the K8SObjectOverlay definition.
// It returns the patched manifest YAML.
func YAMLManifestPatch(baseYAML string, namespace string, overlays []*v1alpha2.K8SObjectOverlay) (string, error) {
	baseObjs, err := object.ParseK8sObjectsFromYAMLManifest(baseYAML)
	if err != nil {
		return "", err
	}

	bom := baseObjs.ToMap()
	oom := objectOverrideMap(overlays, namespace)

	var ret strings.Builder

	var errs util.Errors
	keys := make([]string, 0)
	for key := range oom {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	// Try to apply the defined overlays.
	for _, k := range keys {
		oo := oom[k]
		bo := bom[k]
		if bo == nil {
			os := ""
			for k2 := range bom {
				os += k2 + "\n"
			}
			errs = util.AppendErr(errs, fmt.Errorf("overlay for %s does not match any object in output manifest:\n%s\n\nAvailable objects are:\n%s",
				k, pretty.Sprint(oo), os))
			continue
		}
		patched, err := applyPatches(bo, oo)
		if err != nil {
			errs = util.AppendErr(errs, fmt.Errorf("patch error: %s", err))
			continue
		}
		if _, err := ret.Write(patched); err != nil {
			errs = util.AppendErr(errs, fmt.Errorf("write: %s", err))
		}
		if _, err := ret.WriteString("\n---\n"); err != nil {
			errs = util.AppendErr(errs, fmt.Errorf("patch WriteString error: %s", err))
		}
	}
	// Render the remaining objects with no overlays.
	// keep the original sorted order
	keys = make([]string, 0)
	for key := range bom {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	for _, k := range keys {
		oo := bom[k]
		if oom[k] != nil {
			// Skip objects that have overlays, these were rendered above.
			continue
		}
		oy, err := oo.YAML()
		if err != nil {
			errs = util.AppendErr(errs, fmt.Errorf("object to YAML error (%s) for base object: \n%v", err, oo))
			continue
		}
		if _, err := ret.Write(oy); err != nil {
			errs = util.AppendErr(errs, fmt.Errorf("write: %s", err))
		}
		if _, err := ret.WriteString("\n---\n"); err != nil {
			errs = util.AppendErr(errs, fmt.Errorf("writeString: %s", err))
		}
	}
	return ret.String(), errs.ToError()
}

// applyPatches applies the given patches against the given object. It returns the resulting patched YAML if successful,
// or a list of errors otherwise.
func applyPatches(base *object.K8sObject, patches []*v1alpha2.K8SObjectOverlay_PathValue) (outYAML []byte, errs util.Errors) {
	bo := make(map[interface{}]interface{})
	by, err := base.YAML()
	if err != nil {
		return nil, util.NewErrs(err)
	}
	err = yaml.Unmarshal(by, bo)
	if err != nil {
		return nil, util.NewErrs(err)
	}
	for _, p := range patches {
		scope.Debugf("applying path=%s, value=%s\n", p.Path, p.Value)
		inc, _, err := tpath.GetPathContext(bo, util.PathFromString(p.Path))
		if err != nil {
			errs = util.AppendErr(errs, err)
			continue
		}
		errs = util.AppendErr(errs, tpath.WritePathContext(inc, p.Value))
	}
	oy, err := yaml.Marshal(bo)
	if err != nil {
		return nil, util.AppendErr(errs, err)
	}
	return oy, errs
}

// objectOverrideMap converts oos, a slice of object overlays, into a map of the same overlays where the key is the
// object manifest.Hash.
func objectOverrideMap(oos []*v1alpha2.K8SObjectOverlay, namespace string) map[string][]*v1alpha2.K8SObjectOverlay_PathValue {
	ret := make(map[string][]*v1alpha2.K8SObjectOverlay_PathValue)
	for _, o := range oos {
		ret[object.Hash(o.Kind, namespace, o.Name)] = o.Patches
	}
	return ret
}
<|endoftext|>