<commit_before>package githistory

import (
	"encoding/hex"
	"fmt"
	"strings"

	"github.com/git-lfs/git-lfs/errors"
	"github.com/git-lfs/git-lfs/git"
	"github.com/git-lfs/git-lfs/tasklog"
	"github.com/git-lfs/git-lfs/tools"
	"github.com/git-lfs/gitobj"
)

// refUpdater is a type responsible for moving references from one point in the
// Git object graph to another.
type refUpdater struct {
	// CacheFn is a function that returns the SHA1 transformation from an
	// original hash to a new one. It specifies a "bool" return value
	// signaling whether or not that given "old" SHA1 was migrated.
	CacheFn func(old []byte) ([]byte, bool)
	// Logger logs the progress of reference updating.
	Logger *tasklog.Logger
	// Refs is a set of *git.Ref's to migrate.
	Refs []*git.Ref
	// Root is the given directory on disk in which the repository is
	// located.
	Root string

	db *gitobj.ObjectDatabase
}

// UpdateRefs performs the reference update(s) from existing locations (see:
// Refs) to their respective new locations in the graph (see CacheFn).
//
// It creates reflog entries as well as stderr log entries as it progresses
// through the reference updates.
//
// It returns any error encountered, or nil if the reference update(s) was/were
// successful.
func (r *refUpdater) UpdateRefs() error {
	list := r.Logger.List("migrate: Updating refs")
	defer list.Complete()

	var maxNameLen int
	for _, ref := range r.Refs {
		maxNameLen = tools.MaxInt(maxNameLen, len(ref.Name))
	}

	seen := make(map[string]struct{})
	for _, ref := range r.Refs {
		if err := r.updateOneRef(list, maxNameLen, seen, ref); err != nil {
			return err
		}
	}

	return nil
}

func (r *refUpdater) updateOneRef(list *tasklog.ListTask, maxNameLen int, seen map[string]struct{}, ref *git.Ref) error {
	sha1, err := hex.DecodeString(ref.Sha)
	if err != nil {
		return errors.Wrapf(err, "could not decode: %q", ref.Sha)
	}

	if _, ok := seen[ref.Name]; ok {
		return nil
	}
	seen[ref.Name] = struct{}{}

	to, ok := r.CacheFn(sha1)

	if ref.Type == git.RefTypeLocalTag {
		tag, _ := r.db.Tag(sha1)
		if tag != nil && tag.ObjectType == gitobj.CommitObjectType {
			// Assume that a non-nil error is an indication
			// that the tag is bare (without annotation).

			toObj, okObj := r.CacheFn(tag.Object)
			if !okObj {
				return nil
			}

			newTag, err := r.db.WriteTag(&gitobj.Tag{
				Object:     toObj,
				ObjectType: tag.ObjectType,
				Name:       tag.Name,
				Tagger:     tag.Tagger,

				Message: tag.Message,
			})

			if err != nil {
				return errors.Wrapf(err, "could not rewrite tag: %s", tag.Name)
			}

			to = newTag
			ok = true
		}
	}

	if !ok {
		return nil
	}

	if err := git.UpdateRefIn(r.Root, ref, to, ""); err != nil {
		return err
	}

	namePadding := tools.MaxInt(maxNameLen-len(ref.Name), 0)
	list.Entry(fmt.Sprintf("  %s%s\t%s -> %x", ref.Name, strings.Repeat(" ", namePadding), ref.Sha, to))
	return nil
}
<commit_msg>githistory: split out tag updates into a function<commit_after>package githistory

import (
	"encoding/hex"
	"fmt"
	"strings"

	"github.com/git-lfs/git-lfs/errors"
	"github.com/git-lfs/git-lfs/git"
	"github.com/git-lfs/git-lfs/tasklog"
	"github.com/git-lfs/git-lfs/tools"
	"github.com/git-lfs/gitobj"
)

// refUpdater is a type responsible for moving references from one point in the
// Git object graph to another.
type refUpdater struct {
	// CacheFn is a function that returns the SHA1 transformation from an
	// original hash to a new one. It specifies a "bool" return value
	// signaling whether or not that given "old" SHA1 was migrated.
	CacheFn func(old []byte) ([]byte, bool)
	// Logger logs the progress of reference updating.
	Logger *tasklog.Logger
	// Refs is a set of *git.Ref's to migrate.
	Refs []*git.Ref
	// Root is the given directory on disk in which the repository is
	// located.
	Root string

	db *gitobj.ObjectDatabase
}

// UpdateRefs performs the reference update(s) from existing locations (see:
// Refs) to their respective new locations in the graph (see CacheFn).
//
// It creates reflog entries as well as stderr log entries as it progresses
// through the reference updates.
//
// It returns any error encountered, or nil if the reference update(s) was/were
// successful.
func (r *refUpdater) UpdateRefs() error {
	list := r.Logger.List("migrate: Updating refs")
	defer list.Complete()

	var maxNameLen int
	for _, ref := range r.Refs {
		maxNameLen = tools.MaxInt(maxNameLen, len(ref.Name))
	}

	seen := make(map[string]struct{})
	for _, ref := range r.Refs {
		if err := r.updateOneRef(list, maxNameLen, seen, ref); err != nil {
			return err
		}
	}

	return nil
}

func (r *refUpdater) updateOneTag(tag *gitobj.Tag) ([]byte, error) {
	toObj, okObj := r.CacheFn(tag.Object)
	if !okObj {
		return nil, nil
	}

	newTag, err := r.db.WriteTag(&gitobj.Tag{
		Object:     toObj,
		ObjectType: tag.ObjectType,
		Name:       tag.Name,
		Tagger:     tag.Tagger,

		Message: tag.Message,
	})

	if err != nil {
		return nil, errors.Wrapf(err, "could not rewrite tag: %s", tag.Name)
	}
	return newTag, nil
}

func (r *refUpdater) updateOneRef(list *tasklog.ListTask, maxNameLen int, seen map[string]struct{}, ref *git.Ref) error {
	sha1, err := hex.DecodeString(ref.Sha)
	if err != nil {
		return errors.Wrapf(err, "could not decode: %q", ref.Sha)
	}

	if _, ok := seen[ref.Name]; ok {
		return nil
	}
	seen[ref.Name] = struct{}{}

	to, ok := r.CacheFn(sha1)

	if ref.Type == git.RefTypeLocalTag {
		tag, _ := r.db.Tag(sha1)
		if tag != nil && tag.ObjectType == gitobj.CommitObjectType {
			// Assume that a non-nil error is an indication
			// that the tag is bare (without annotation).

			newTag, err := r.updateOneTag(tag)
			if newTag == nil {
				return err
			}
			to = newTag
			ok = true
		}
	}

	if !ok {
		return nil
	}

	if err := git.UpdateRefIn(r.Root, ref, to, ""); err != nil {
		return err
	}

	namePadding := tools.MaxInt(maxNameLen-len(ref.Name), 0)
	list.Entry(fmt.Sprintf("  %s%s\t%s -> %x", ref.Name, strings.Repeat(" ", namePadding), ref.Sha, to))
	return nil
}
<|endoftext|>


<commit_before>package githistory

import (
	"encoding/hex"
	"fmt"

	"github.com/git-lfs/git-lfs/errors"
	"github.com/git-lfs/git-lfs/git"
	"github.com/git-lfs/git-lfs/git/githistory/log"
)

// refUpdater is a type responsible for moving references from one point in the
// Git object graph to another.
type refUpdater struct {
	// CacheFn is a function that returns the SHA1 transformation from an
	// original hash to a new one. It specifies a "bool" return value
	// signaling whether or not that given "old" SHA1 was migrated.
	CacheFn func(old []byte) ([]byte, bool)
	// Logger logs the progress of reference updating.
	Logger *log.Logger
	// Refs is a set of *git.Ref's to migrate.
	Refs []*git.Ref
	// Root is the given directory on disk in which the repository is
	// located.
	Root string
}

// UpdateRefs performs the reference update(s) from existing locations (see:
// Refs) to their respective new locations in the graph (see CacheFn).
//
// It creates reflog entries as well as stderr log entries as it progresses
// through the reference updates.
//
// It returns any error encountered, or nil if the reference update(s) was/were
// successful.
func (r *refUpdater) UpdateRefs() error {
	list := r.Logger.List("migrate: Updating refs")
	defer list.Complete()

	for _, ref := range r.Refs {
		sha1, err := hex.DecodeString(ref.Sha)
		if err != nil {
			return errors.Wrapf(err, "could not decode: %q", ref.Sha)
		}

		to, ok := r.CacheFn(sha1)
		if !ok {
			continue
		}

		if err := git.UpdateRefIn(r.Root, ref, to, ""); err != nil {
			return err
		}

		list.Entry(fmt.Sprintf("%s\t%s -> %x", ref, ref.Sha, to))
	}

	return nil
}
<commit_msg>git/githistory: fix formatting in *refUpdater<commit_after>package githistory

import (
	"encoding/hex"
	"fmt"

	"github.com/git-lfs/git-lfs/errors"
	"github.com/git-lfs/git-lfs/git"
	"github.com/git-lfs/git-lfs/git/githistory/log"
)

// refUpdater is a type responsible for moving references from one point in the
// Git object graph to another.
type refUpdater struct {
	// CacheFn is a function that returns the SHA1 transformation from an
	// original hash to a new one. It specifies a "bool" return value
	// signaling whether or not that given "old" SHA1 was migrated.
	CacheFn func(old []byte) ([]byte, bool)
	// Logger logs the progress of reference updating.
	Logger *log.Logger
	// Refs is a set of *git.Ref's to migrate.
	Refs []*git.Ref
	// Root is the given directory on disk in which the repository is
	// located.
	Root string
}

// UpdateRefs performs the reference update(s) from existing locations (see:
// Refs) to their respective new locations in the graph (see CacheFn).
//
// It creates reflog entries as well as stderr log entries as it progresses
// through the reference updates.
//
// It returns any error encountered, or nil if the reference update(s) was/were
// successful.
func (r *refUpdater) UpdateRefs() error {
	list := r.Logger.List("migrate: Updating refs")
	defer list.Complete()

	for _, ref := range r.Refs {
		sha1, err := hex.DecodeString(ref.Sha)
		if err != nil {
			return errors.Wrapf(err, "could not decode: %q", ref.Sha)
		}

		to, ok := r.CacheFn(sha1)
		if !ok {
			continue
		}

		if err := git.UpdateRefIn(r.Root, ref, to, ""); err != nil {
			return err
		}

		list.Entry(fmt.Sprintf("  %s\t%s -> %x", ref.Name, ref.Sha, to))
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2013-2014 by Michael Dvorkin. All Rights Reserved.
// Use of this source code is governed by a MIT-style license that can
// be found in the LICENSE file.

package donna

import (
	`bufio`
	`fmt`
	`io`
	`os`
	`strconv`
	`strings`
)

func (e *Engine) uciScore(depth, score, alpha, beta int) *Engine {
	str := fmt.Sprintf("info depth %d score", depth)

	if abs(score) < Checkmate-MaxPly {
		str += fmt.Sprintf(" cp %d", score*100/onePawn)
	} else {
		mate := -Checkmate - score
		if score > 0 {
			mate = Checkmate - score + 1
		}
		str += fmt.Sprintf(" mate %d", mate/2)
	}
	if score <= alpha {
		str += " upperbound"
	} else if score >= beta {
		str += " lowerbound"
	}

	return engine.reply(str + "\n")
}

func (e *Engine) uciMove(move Move, moveno, depth int) *Engine {
	return engine.reply("info depth %d currmove %s currmovenumber %d\n", depth, move.notation(), moveno)
}

func (e *Engine) uciBestMove(move Move, duration int64) *Engine {
	return engine.reply("info nodes %d time %d\nbestmove %s\n", game.nodes + game.qnodes, duration, move.notation())
}

func (e *Engine) uciPrincipal(depth, score int, duration int64) *Engine {
	str := fmt.Sprintf("info depth %d score", depth)

	if abs(score) < Checkmate - MaxPly {
		str += fmt.Sprintf(" cp %d", score * 100 / onePawn)
	} else {
		mate := -Checkmate - score
		if score > 0 {
			mate = Checkmate - score + 1
		}
		str += fmt.Sprintf(" mate %d", mate / 2)
	}
	str += fmt.Sprintf(" nodes %d nps %d time %d pv", game.nodes + game.qnodes, nps(duration), duration)

	for i := 0; i < len(game.rootpv); i++ {
		str += " " + game.rootpv[i].notation()
	}

	return engine.reply(str + "\n")
}

// Brain-damaged universal chess interface (UCI) protocol as described at
// http://wbec-ridderkerk.nl/html/UCIProtocol.html
func (e *Engine) Uci() *Engine {
	var game *Game
	var position *Position

	e.uci = true

	// "uci" command handler.
	doUci := func(args []string) {
		e.reply("Donna v%s Copyright (c) 2014 by Michael Dvorkin. All Rights Reserved.\n", Version)
		e.reply("id name Donna %s\n", Version)
		e.reply("id author Michael Dvorkin\n")
		e.reply("option name Hash type spin default %d min 1 max 1024\n", 64)
		e.reply("option name Mobility type spin default %d min 0 max 100\n", weights[0].midgame)
		e.reply("option name PawnStructure type spin default %d min 0 max 100\n", weights[1].midgame)
		e.reply("option name PassedPawns type spin default %d min 0 max 100\n", weights[2].midgame)
		e.reply("option name KingSafety type spin default %d min 0 max 100\n", weights[3].midgame)
		e.reply("option name EnemyKingSafety type spin default %d min 0 max 100\n", weights[4].midgame)
		e.reply("uciok\n")
	}

	// "ucinewgame" command handler.
	doUciNewGame := func(args []string) {
		game, position = nil, nil
	}

	// "isready" command handler.
	doIsReady := func(args []string) {
		e.reply("readyok\n")
	}

	// "position [startpos | fen ] [ moves ... ]" command handler.
	doPosition := func(args []string) {
		// Make sure we've started the game since "ucinewgame" is optional.
		if game == nil || position == nil {
			game = NewGame()
		}

		switch args[0] {
		case `startpos`:
			args = args[1:]
			position = game.start()
		case `fen`:
			fen := []string{}
			for _, token := range args[1:] {
				args = args[1:] // Shift the token.
				if token == `moves` {
					break
				}
				fen = append(fen, token)
			}
			game.initial = strings.Join(fen, ` `)
			position = game.start()
		default:
			return
		}

		if position != nil && len(args) > 0 && args[0] == `moves` {
			for _, move := range args[1:] {
				args = args[1:] // Shift the move.
				position = position.makeMove(NewMoveFromNotation(position, move))
			}
		}
	}

	// "go [[wtime winc | btime binc ] movestogo] | depth | nodes | movetime"
	doGo := func(args []string) {
		think := true
		options := e.options

		for i, token := range args {
			// Boolen "infinite" and "ponder" commands have no arguments.
			if token == `infinite` {
				options = Options{infinite: true}
			} else if token == `ponder` {
				options = Options{ponder: true}
			} else if token == `test` { // <-- Custom token for use in tests.
				think = false
			} else if len(args) > i+1 {
				switch token {
				case `depth`:
					if n, err := strconv.Atoi(args[i+1]); err == nil {
						options = Options{ maxDepth: n }
					}
				case `nodes`:
					if n, err := strconv.Atoi(args[i+1]); err == nil {
						options = Options{ maxNodes: n }
					}
				case `movetime`:
					if n, err := strconv.Atoi(args[i+1]); err == nil {
						options = Options{ moveTime: int64(n) }
					}
				case `wtime`:
					if position.color == White {
						if n, err := strconv.Atoi(args[i+1]); err == nil {
							options.timeLeft = int64(n)
						}
					}
				case `btime`:
					if position.color == Black {
						if n, err := strconv.Atoi(args[i+1]); err == nil {
							options.timeLeft = int64(n)
						}
					}
				case `winc`:
					if position.color == White {
						if n, err := strconv.Atoi(args[i+1]); err == nil {
							options.timeInc = int64(n)
						}
					}
				case `binc`:
					if position.color == Black {
						if n, err := strconv.Atoi(args[i+1]); err == nil {
							options.timeInc = int64(n)
						}
					}
				case `movestogo`:
					if n, err := strconv.Atoi(args[i+1]); err == nil {
						options.movesToGo = int64(n)
					}
				}
			}
		}
		if options.timeLeft != 0 || options.timeInc != 0 || options.movesToGo != 0 {
			e.varyingLimits(options)
		} else {
			e.fixedLimit(options)
		}

		// Start "thinking" and come up with best move unless when running
		// tests where we verify argument parsing only.
		if think {
			game.Think()
		}
	}

	// Stop calculating as soon as possible.
	doStop := func(args []string) {
		e.clock.halt = true
	}

	var commands = map[string]func([]string){
		`isready`:    doIsReady,
		`uci`:        doUci,
		`ucinewgame`: doUciNewGame,
		`position`:   doPosition,
		`go`:         doGo,
		`stop`:       doStop,
	}

	bio := bufio.NewReader(os.Stdin)
	for {
		command, err := bio.ReadString('\n')
		if err != io.EOF && len(command) > 0 {
			e.debug("> " + command)
			args := strings.Split(strings.Trim(command, " \t\r\n"), ` `)
			if args[0] == `quit` {
				break
			}
			if handler, ok := commands[args[0]]; ok {
				handler(args[1:])
			}
		}
	}
	return e
}
<commit_msg>UCI option setting is not supported yet<commit_after>// Copyright (c) 2013-2014 by Michael Dvorkin. All Rights Reserved.
// Use of this source code is governed by a MIT-style license that can
// be found in the LICENSE file.

package donna

import (
	`bufio`
	`fmt`
	`io`
	`os`
	`strconv`
	`strings`
)

func (e *Engine) uciScore(depth, score, alpha, beta int) *Engine {
	str := fmt.Sprintf("info depth %d score", depth)

	if abs(score) < Checkmate-MaxPly {
		str += fmt.Sprintf(" cp %d", score*100/onePawn)
	} else {
		mate := -Checkmate - score
		if score > 0 {
			mate = Checkmate - score + 1
		}
		str += fmt.Sprintf(" mate %d", mate/2)
	}
	if score <= alpha {
		str += " upperbound"
	} else if score >= beta {
		str += " lowerbound"
	}

	return engine.reply(str + "\n")
}

func (e *Engine) uciMove(move Move, moveno, depth int) *Engine {
	return engine.reply("info depth %d currmove %s currmovenumber %d\n", depth, move.notation(), moveno)
}

func (e *Engine) uciBestMove(move Move, duration int64) *Engine {
	return engine.reply("info nodes %d time %d\nbestmove %s\n", game.nodes + game.qnodes, duration, move.notation())
}

func (e *Engine) uciPrincipal(depth, score int, duration int64) *Engine {
	str := fmt.Sprintf("info depth %d score", depth)

	if abs(score) < Checkmate - MaxPly {
		str += fmt.Sprintf(" cp %d", score * 100 / onePawn)
	} else {
		mate := -Checkmate - score
		if score > 0 {
			mate = Checkmate - score + 1
		}
		str += fmt.Sprintf(" mate %d", mate / 2)
	}
	str += fmt.Sprintf(" nodes %d nps %d time %d pv", game.nodes + game.qnodes, nps(duration), duration)

	for i := 0; i < len(game.rootpv); i++ {
		str += " " + game.rootpv[i].notation()
	}

	return engine.reply(str + "\n")
}

// Brain-damaged universal chess interface (UCI) protocol as described at
// http://wbec-ridderkerk.nl/html/UCIProtocol.html
func (e *Engine) Uci() *Engine {
	var game *Game
	var position *Position

	e.uci = true

	// "uci" command handler.
	doUci := func(args []string) {
		e.reply("Donna v%s Copyright (c) 2014 by Michael Dvorkin. All Rights Reserved.\n", Version)
		e.reply("id name Donna %s\n", Version)
		e.reply("id author Michael Dvorkin\n")
		// e.reply("option name Hash type spin default %d min 1 max 1024\n", 64)
		// e.reply("option name Mobility type spin default %d min 0 max 100\n", weights[0].midgame)
		// e.reply("option name PawnStructure type spin default %d min 0 max 100\n", weights[1].midgame)
		// e.reply("option name PassedPawns type spin default %d min 0 max 100\n", weights[2].midgame)
		// e.reply("option name KingSafety type spin default %d min 0 max 100\n", weights[3].midgame)
		// e.reply("option name EnemyKingSafety type spin default %d min 0 max 100\n", weights[4].midgame)
		e.reply("uciok\n")
	}

	// "ucinewgame" command handler.
	doUciNewGame := func(args []string) {
		game, position = nil, nil
	}

	// "isready" command handler.
	doIsReady := func(args []string) {
		e.reply("readyok\n")
	}

	// "position [startpos | fen ] [ moves ... ]" command handler.
	doPosition := func(args []string) {
		// Make sure we've started the game since "ucinewgame" is optional.
		if game == nil || position == nil {
			game = NewGame()
		}

		switch args[0] {
		case `startpos`:
			args = args[1:]
			position = game.start()
		case `fen`:
			fen := []string{}
			for _, token := range args[1:] {
				args = args[1:] // Shift the token.
				if token == `moves` {
					break
				}
				fen = append(fen, token)
			}
			game.initial = strings.Join(fen, ` `)
			position = game.start()
		default:
			return
		}

		if position != nil && len(args) > 0 && args[0] == `moves` {
			for _, move := range args[1:] {
				args = args[1:] // Shift the move.
				position = position.makeMove(NewMoveFromNotation(position, move))
			}
		}
	}

	// "go [[wtime winc | btime binc ] movestogo] | depth | nodes | movetime"
	doGo := func(args []string) {
		think := true
		options := e.options

		for i, token := range args {
			// Boolen "infinite" and "ponder" commands have no arguments.
			if token == `infinite` {
				options = Options{infinite: true}
			} else if token == `ponder` {
				options = Options{ponder: true}
			} else if token == `test` { // <-- Custom token for use in tests.
				think = false
			} else if len(args) > i+1 {
				switch token {
				case `depth`:
					if n, err := strconv.Atoi(args[i+1]); err == nil {
						options = Options{ maxDepth: n }
					}
				case `nodes`:
					if n, err := strconv.Atoi(args[i+1]); err == nil {
						options = Options{ maxNodes: n }
					}
				case `movetime`:
					if n, err := strconv.Atoi(args[i+1]); err == nil {
						options = Options{ moveTime: int64(n) }
					}
				case `wtime`:
					if position.color == White {
						if n, err := strconv.Atoi(args[i+1]); err == nil {
							options.timeLeft = int64(n)
						}
					}
				case `btime`:
					if position.color == Black {
						if n, err := strconv.Atoi(args[i+1]); err == nil {
							options.timeLeft = int64(n)
						}
					}
				case `winc`:
					if position.color == White {
						if n, err := strconv.Atoi(args[i+1]); err == nil {
							options.timeInc = int64(n)
						}
					}
				case `binc`:
					if position.color == Black {
						if n, err := strconv.Atoi(args[i+1]); err == nil {
							options.timeInc = int64(n)
						}
					}
				case `movestogo`:
					if n, err := strconv.Atoi(args[i+1]); err == nil {
						options.movesToGo = int64(n)
					}
				}
			}
		}
		if options.timeLeft != 0 || options.timeInc != 0 || options.movesToGo != 0 {
			e.varyingLimits(options)
		} else {
			e.fixedLimit(options)
		}

		// Start "thinking" and come up with best move unless when running
		// tests where we verify argument parsing only.
		if think {
			game.Think()
		}
	}

	// Stop calculating as soon as possible.
	doStop := func(args []string) {
		e.clock.halt = true
	}

	var commands = map[string]func([]string){
		`isready`:    doIsReady,
		`uci`:        doUci,
		`ucinewgame`: doUciNewGame,
		`position`:   doPosition,
		`go`:         doGo,
		`stop`:       doStop,
	}

	bio := bufio.NewReader(os.Stdin)
	for {
		command, err := bio.ReadString('\n')
		if err != io.EOF && len(command) > 0 {
			e.debug("> " + command)
			args := strings.Split(strings.Trim(command, " \t\r\n"), ` `)
			if args[0] == `quit` {
				break
			}
			if handler, ok := commands[args[0]]; ok {
				handler(args[1:])
			}
		}
	}
	return e
}
<|endoftext|>


<commit_before>package test

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"path/filepath"
	"reflect"
	"runtime"
	"testing"

	"github.com/Sirupsen/logrus"
)

// Assert fails the test if the condition is false.
func Assert(tb testing.TB, condition bool, v ...interface{}) {
	if !condition {
		_, file, line, _ := runtime.Caller(1)
		fmt.Printf("%s:%d:\n\n", append([]interface{}{filepath.Base(file), line}, v...)...)
		tb.FailNow()
	}
}

// ErrorNil fails the test if an err is not nil.
func ErrorNil(tb testing.TB, err error) {
	if err != nil {
		_, file, line, _ := runtime.Caller(1)
		fmt.Printf("%s:%d: unexpected error: %s\n\n", filepath.Base(file), line, err.Error())
		tb.FailNow()
	}
}

// ErrorNotNil fails the test if an err is not nil.
func ErrorNotNil(tb testing.TB, err error) {
	if err == nil {
		_, file, line, _ := runtime.Caller(1)
		fmt.Printf("%s:%d: expected error but got none\n\n", filepath.Base(file), line)
		tb.FailNow()
	}
}

// Equals fails the test if expected is not equal to actual.
func Equals(tb testing.TB, expected, actual interface{}) {
	if !reflect.DeepEqual(expected, actual) {
		_, file, line, _ := runtime.Caller(1)
		fmt.Printf("%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\n\n", filepath.Base(file), line, expected, actual)
		tb.FailNow()
	}
}

// LogEntryContainsField fails if the log entry does not contain a field
// with the given key.
func LogEntryContainsField(tb testing.TB, key string, expectedValue string, logEntry *logrus.Entry) {
	if logEntry == nil {
		tb.Fatal("log entry expected")
	}

	actual := fmt.Sprintf("%v", logEntry.Data[key])
	if actual != expectedValue {
		tb.Fatalf("log entry expected: '%s' but got: '%s'", expectedValue, actual)
	}
}

// MakeGetRequest performs a GET request using a given handler.
func MakeGetRequest(handler http.Handler, uri string) (resp *httptest.ResponseRecorder) {
	req, _ := http.NewRequest(http.MethodGet, uri, nil)
	resp = httptest.NewRecorder()
	handler.ServeHTTP(resp, req)

	return
}
<commit_msg>Removed log test from common<commit_after>package test

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"path/filepath"
	"reflect"
	"runtime"
	"testing"
)

// Assert fails the test if the condition is false.
func Assert(tb testing.TB, condition bool, v ...interface{}) {
	if !condition {
		_, file, line, _ := runtime.Caller(1)
		fmt.Printf("%s:%d:\n\n", append([]interface{}{filepath.Base(file), line}, v...)...)
		tb.FailNow()
	}
}

// ErrorNil fails the test if an err is not nil.
func ErrorNil(tb testing.TB, err error) {
	if err != nil {
		_, file, line, _ := runtime.Caller(1)
		fmt.Printf("%s:%d: unexpected error: %s\n\n", filepath.Base(file), line, err.Error())
		tb.FailNow()
	}
}

// ErrorNotNil fails the test if an err is not nil.
func ErrorNotNil(tb testing.TB, err error) {
	if err == nil {
		_, file, line, _ := runtime.Caller(1)
		fmt.Printf("%s:%d: expected error but got none\n\n", filepath.Base(file), line)
		tb.FailNow()
	}
}

// Equals fails the test if expected is not equal to actual.
func Equals(tb testing.TB, expected, actual interface{}) {
	if !reflect.DeepEqual(expected, actual) {
		_, file, line, _ := runtime.Caller(1)
		fmt.Printf("%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\n\n", filepath.Base(file), line, expected, actual)
		tb.FailNow()
	}
}

// MakeGetRequest performs a GET request using a given handler.
func MakeGetRequest(handler http.Handler, uri string) (resp *httptest.ResponseRecorder) {
	req, _ := http.NewRequest(http.MethodGet, uri, nil)
	resp = httptest.NewRecorder()
	handler.ServeHTTP(resp, req)

	return
}
<|endoftext|>


<commit_before>package main

import (
	//"database/sql"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"os"
	"path/filepath"
  "sync"
  "time"
)

const SQL_CREATE_SHARE_TABLE = "create table if not exists 'shares' ('id' INTEGER PRIMARY KEY AUTOINCREMENT,'name' VARCHAR(128) NULL,path VARCHAR(4096) NULL)"

func checkErr(err error) {
	if err != nil {
		panic(err)
	}
}
func ShareWorker(StopChan chan bool, WakeEvent *sync.Cond) {
  var lcvrun bool = true
  for lcvrun {
    select {
    case x, ok := <-StopChan:
        if ok {
            fmt.Println("Value was read:", x)
            if x {
              break
            }
        } else {
            fmt.Println("Channel closed!")
            lcvrun = false
            break
        }
    default:
        fmt.Println("No value ready, Waiting on Event.")
        WakeEvent.L.Lock()
        WakeEvent.Wait()
        WakeEvent.L.Unlock()
    }
  }
  return
}

func processWalk(path string, info os.FileInfo, err error) error {
  var clean_path string
  clean_path = filepath.Clean(path)
	fmt.Println(clean_path, info)
	return nil
}

func main() {
	fmt.Println("Taco")
  var lock sync.Mutex
  var stopchan chan bool = make(chan bool)
  wakeevent := sync.NewCond(&lock)
  go ShareWorker(stopchan,wakeevent)
  stopchan <- false
  fmt.Println("Sleeping in main")
  time.Sleep(3 * time.Second)
  close(stopchan)
  wakeevent.Broadcast()
  time.Sleep(3 * time.Second)
	//db, err := sql.Open("sqlite3", "./foo.db")
	//checkErr(err)

	//res, err := db.Exec(SQL_CREATE_SHARE_TABLE)
	//_ = res
	//checkErr(err)
	//filepath.Walk("/home/scott/", processWalk)

	//db.Close()
	fmt.Println("Taco2")
}
<commit_msg>started some goroutine logic<commit_after>package main

import (
	//"database/sql"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"os"
	"path/filepath"
  "sync"
  "time"
)

const SQL_CREATE_SHARE_TABLE = "create table if not exists 'shares' ('id' INTEGER PRIMARY KEY AUTOINCREMENT,'name' VARCHAR(128) NULL,path VARCHAR(4096) NULL)"

func checkErr(err error) {
	if err != nil {
		panic(err)
	}
}
func ShareWorker(StopChan chan bool, WakeEvent *sync.Cond) {
  var lcvrun bool = true
  for lcvrun {
    x, ok := <-StopChan
    if ok {
        fmt.Println("Value was read:", x)
        if x {
          break
        }
    } else {
        fmt.Println("Channel closed!")
        lcvrun = false
        break
    }
    fmt.Println("No value ready, Waiting on Event.")
    WakeEvent.L.Lock()
    WakeEvent.Wait()
    WakeEvent.L.Unlock()
  }
  fmt.Println("ShareWorker GR Exit")
  return
}

func processWalk(path string, info os.FileInfo, err error) error {
  var clean_path string
  clean_path = filepath.Clean(path)
	fmt.Println(clean_path, info)
	return nil
}

func main() {
	fmt.Println("Taco")
  var lock sync.Mutex
  var stopchan chan bool = make(chan bool,10)
  wakeevent := sync.NewCond(&lock)
  go ShareWorker(stopchan,wakeevent)
  stopchan <- false
  fmt.Println("Sleeping in main")
  time.Sleep(3 * time.Second)
  stopchan <- true
  wakeevent.Broadcast()
  time.Sleep(3 * time.Second)
	//db, err := sql.Open("sqlite3", "./foo.db")
	//checkErr(err)

	//res, err := db.Exec(SQL_CREATE_SHARE_TABLE)
	//_ = res
	//checkErr(err)
	//filepath.Walk("/home/scott/", processWalk)

	//db.Close()
	fmt.Println("Taco2")
}
<|endoftext|>


<commit_before>package agent

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"testing"

	"github.com/hashicorp/consul/testutil"
	"github.com/mitchellh/cli"
)

func TestCommand_implements(t *testing.T) {
	var _ cli.Command = new(Command)
}

func TestValidDatacenter(t *testing.T) {
	shouldMatch := []string{
		"dc1",
		"east-aws-001",
		"PROD_aws01-small",
	}
	noMatch := []string{
		"east.aws",
		"east!aws",
		"first,second",
	}
	for _, m := range shouldMatch {
		if !validDatacenter.MatchString(m) {
			t.Fatalf("expected match: %s", m)
		}
	}
	for _, m := range noMatch {
		if validDatacenter.MatchString(m) {
			t.Fatalf("expected no match: %s", m)
		}
	}
}

func TestRetryJoin(t *testing.T) {
	dir, agent := makeAgent(t, nextConfig())
	defer os.RemoveAll(dir)
	defer agent.Shutdown()

	conf2 := nextConfig()
	tmpDir, err := ioutil.TempDir("", "consul")
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	defer os.RemoveAll(tmpDir)

	doneCh := make(chan struct{})
	shutdownCh := make(chan struct{})

	defer func() {
		close(shutdownCh)
		<-doneCh
	}()

	cmd := &Command{
		ShutdownCh: shutdownCh,
		Ui:         new(cli.MockUi),
	}

	serfAddr := fmt.Sprintf(
		"%s:%d",
		agent.config.BindAddr,
		agent.config.Ports.SerfLan)

	serfWanAddr := fmt.Sprintf(
		"%s:%d",
		agent.config.BindAddr,
		agent.config.Ports.SerfWan)

	args := []string{
		"-server",
		"-data-dir", tmpDir,
		"-node", fmt.Sprintf(`"%s"`, conf2.NodeName),
		"-retry-join", serfAddr,
		"-retry-interval", "1s",
		"-retry-join-wan", serfWanAddr,
		"-retry-interval-wan", "1s",
	}

	go func() {
		if code := cmd.Run(args); code != 0 {
			log.Printf("bad: %d", code)
		}
		close(doneCh)
	}()

	testutil.WaitForResult(func() (bool, error) {
		mem := agent.LANMembers()
		if len(mem) != 2 {
			return false, fmt.Errorf("bad: %#v", mem)
		}
		mem = agent.WANMembers()
		if len(mem) != 2 {
			return false, fmt.Errorf("bad (wan): %#v", mem)
		}
		return true, nil
	}, func(err error) {
		t.Fatalf(err.Error())
	})
}

func TestRetryJoinFail(t *testing.T) {
	conf := nextConfig()
	tmpDir, err := ioutil.TempDir("", "consul")
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	defer os.RemoveAll(tmpDir)

	shutdownCh := make(chan struct{})
	defer close(shutdownCh)

	cmd := &Command{
		ShutdownCh: shutdownCh,
		Ui:         new(cli.MockUi),
	}

	serfAddr := fmt.Sprintf("%s:%d", conf.BindAddr, conf.Ports.SerfLan)

	args := []string{
		"-data-dir", tmpDir,
		"-retry-join", serfAddr,
		"-retry-max", "1",
	}

	if code := cmd.Run(args); code == 0 {
		t.Fatalf("bad: %d", code)
	}
}

func TestRetryJoinWanFail(t *testing.T) {
	conf := nextConfig()
	tmpDir, err := ioutil.TempDir("", "consul")
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	defer os.RemoveAll(tmpDir)

	shutdownCh := make(chan struct{})
	defer close(shutdownCh)

	cmd := &Command{
		ShutdownCh: shutdownCh,
		Ui:         new(cli.MockUi),
	}

	serfAddr := fmt.Sprintf("%s:%d", conf.BindAddr, conf.Ports.SerfWan)

	args := []string{
		"-server",
		"-data-dir", tmpDir,
		"-retry-join-wan", serfAddr,
		"-retry-max-wan", "1",
	}

	if code := cmd.Run(args); code == 0 {
		t.Fatalf("bad: %d", code)
	}
}
<commit_msg>agent: test agent rpc unix socket create failure<commit_after>package agent

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"testing"

	"github.com/hashicorp/consul/testutil"
	"github.com/mitchellh/cli"
)

func TestCommand_implements(t *testing.T) {
	var _ cli.Command = new(Command)
}

func TestValidDatacenter(t *testing.T) {
	shouldMatch := []string{
		"dc1",
		"east-aws-001",
		"PROD_aws01-small",
	}
	noMatch := []string{
		"east.aws",
		"east!aws",
		"first,second",
	}
	for _, m := range shouldMatch {
		if !validDatacenter.MatchString(m) {
			t.Fatalf("expected match: %s", m)
		}
	}
	for _, m := range noMatch {
		if validDatacenter.MatchString(m) {
			t.Fatalf("expected no match: %s", m)
		}
	}
}

func TestRetryJoin(t *testing.T) {
	dir, agent := makeAgent(t, nextConfig())
	defer os.RemoveAll(dir)
	defer agent.Shutdown()

	conf2 := nextConfig()
	tmpDir, err := ioutil.TempDir("", "consul")
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	defer os.RemoveAll(tmpDir)

	doneCh := make(chan struct{})
	shutdownCh := make(chan struct{})

	defer func() {
		close(shutdownCh)
		<-doneCh
	}()

	cmd := &Command{
		ShutdownCh: shutdownCh,
		Ui:         new(cli.MockUi),
	}

	serfAddr := fmt.Sprintf(
		"%s:%d",
		agent.config.BindAddr,
		agent.config.Ports.SerfLan)

	serfWanAddr := fmt.Sprintf(
		"%s:%d",
		agent.config.BindAddr,
		agent.config.Ports.SerfWan)

	args := []string{
		"-server",
		"-data-dir", tmpDir,
		"-node", fmt.Sprintf(`"%s"`, conf2.NodeName),
		"-retry-join", serfAddr,
		"-retry-interval", "1s",
		"-retry-join-wan", serfWanAddr,
		"-retry-interval-wan", "1s",
	}

	go func() {
		if code := cmd.Run(args); code != 0 {
			log.Printf("bad: %d", code)
		}
		close(doneCh)
	}()

	testutil.WaitForResult(func() (bool, error) {
		mem := agent.LANMembers()
		if len(mem) != 2 {
			return false, fmt.Errorf("bad: %#v", mem)
		}
		mem = agent.WANMembers()
		if len(mem) != 2 {
			return false, fmt.Errorf("bad (wan): %#v", mem)
		}
		return true, nil
	}, func(err error) {
		t.Fatalf(err.Error())
	})
}

func TestRetryJoinFail(t *testing.T) {
	conf := nextConfig()
	tmpDir, err := ioutil.TempDir("", "consul")
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	defer os.RemoveAll(tmpDir)

	shutdownCh := make(chan struct{})
	defer close(shutdownCh)

	cmd := &Command{
		ShutdownCh: shutdownCh,
		Ui:         new(cli.MockUi),
	}

	serfAddr := fmt.Sprintf("%s:%d", conf.BindAddr, conf.Ports.SerfLan)

	args := []string{
		"-data-dir", tmpDir,
		"-retry-join", serfAddr,
		"-retry-max", "1",
	}

	if code := cmd.Run(args); code == 0 {
		t.Fatalf("bad: %d", code)
	}
}

func TestRetryJoinWanFail(t *testing.T) {
	conf := nextConfig()
	tmpDir, err := ioutil.TempDir("", "consul")
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	defer os.RemoveAll(tmpDir)

	shutdownCh := make(chan struct{})
	defer close(shutdownCh)

	cmd := &Command{
		ShutdownCh: shutdownCh,
		Ui:         new(cli.MockUi),
	}

	serfAddr := fmt.Sprintf("%s:%d", conf.BindAddr, conf.Ports.SerfWan)

	args := []string{
		"-server",
		"-data-dir", tmpDir,
		"-retry-join-wan", serfAddr,
		"-retry-max-wan", "1",
	}

	if code := cmd.Run(args); code == 0 {
		t.Fatalf("bad: %d", code)
	}
}

func TestSetupAgent_UnixSocket_Fails(t *testing.T) {
	conf := nextConfig()
	tmpDir, err := ioutil.TempDir("", "consul")
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	defer os.RemoveAll(tmpDir)

	tmpFile, err := ioutil.TempFile("", "consul")
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	defer os.Remove(tmpFile.Name())
	socketPath := tmpFile.Name()

	conf.DataDir = tmpDir
	conf.Server = true
	conf.Bootstrap = true

	// Set socket address to an existing file. Consul should fail to
	// start and return an error.
	conf.Addresses.RPC = "unix://" + socketPath

	shutdownCh := make(chan struct{})
	defer close(shutdownCh)

	cmd := &Command{
		ShutdownCh: shutdownCh,
		Ui:         new(cli.MockUi),
	}

	logWriter := NewLogWriter(512)
	logOutput := new(bytes.Buffer)

	// Ensure we got an error mentioning the socket file
	err = cmd.setupAgent(conf, logOutput, logWriter)
	if err == nil {
		t.Fatalf("should have failed")
	}
	if !strings.Contains(err.Error(), socketPath) {
		t.Fatalf("expected socket file error, got: %q", err)
	}
}
<|endoftext|>


<commit_before>// +build envtest

/*
 Copyright 2021 Crunchy Data Solutions, Inc.
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package postgrescluster

import (
	"context"
	"io"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/onsi/gomega"
	"github.com/onsi/gomega/gstruct"
	"go.opentelemetry.io/otel"
	"gotest.tools/v3/assert"
	appsv1 "k8s.io/api/apps/v1"
	v1 "k8s.io/api/core/v1"
	rbacv1 "k8s.io/api/rbac/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/client-go/kubernetes/scheme"
	"k8s.io/client-go/tools/record"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/envtest"
	"sigs.k8s.io/controller-runtime/pkg/reconcile"
	"sigs.k8s.io/yaml"

	"github.com/crunchydata/postgres-operator/internal/patroni"
	"github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1alpha1"
)

func TestReconcilerHandleDelete(t *testing.T) {
	if !strings.EqualFold(os.Getenv("USE_EXISTING_CLUSTER"), "true") {
		t.Skip("requires a running garbage collection controller")
	}
	t.Parallel()

	ctx := context.Background()
	env := &envtest.Environment{
		CRDDirectoryPaths: []string{
			filepath.Join("..", "..", "..", "config", "crd", "bases"),
		},
	}

	options := client.Options{}
	options.Scheme = runtime.NewScheme()
	assert.NilError(t, scheme.AddToScheme(options.Scheme))
	assert.NilError(t, v1alpha1.AddToScheme(options.Scheme))

	config, err := env.Start()
	assert.NilError(t, err)
	t.Cleanup(func() { assert.Check(t, env.Stop()) })

	cc, err := client.New(config, options)
	assert.NilError(t, err)

	ns := &v1.Namespace{}
	ns.GenerateName = "postgres-operator-test-"
	ns.Labels = labels.Set{"postgres-operator-test": t.Name()}
	assert.NilError(t, cc.Create(ctx, ns))
	t.Cleanup(func() { assert.Check(t, cc.Delete(ctx, ns)) })

	// TODO(cbandy): namespace rbac
	assert.NilError(t, cc.Create(ctx, &v1.ServiceAccount{
		ObjectMeta: metav1.ObjectMeta{Namespace: ns.Name, Name: "postgres-operator"},
	}))
	assert.NilError(t, cc.Create(ctx, &rbacv1.RoleBinding{
		ObjectMeta: metav1.ObjectMeta{Namespace: ns.Name, Name: "postgres-operator"},
		RoleRef: rbacv1.RoleRef{
			Kind: "ClusterRole", Name: "postgres-operator-role",
		},
		Subjects: []rbacv1.Subject{{
			Kind: "ServiceAccount", Namespace: ns.Name, Name: "postgres-operator",
		}},
	}))

	reconciler := Reconciler{
		Client:   cc,
		Owner:    client.FieldOwner(t.Name()),
		Recorder: new(record.FakeRecorder),
		Tracer:   otel.Tracer(t.Name()),
	}

	reconciler.PodExec, err = newPodExecutor(config)
	assert.NilError(t, err)

	mustReconcile := func(t *testing.T, cluster *v1alpha1.PostgresCluster) reconcile.Result {
		t.Helper()
		key := client.ObjectKeyFromObject(cluster)
		request := reconcile.Request{NamespacedName: key}
		result, err := reconciler.Reconcile(ctx, request)
		assert.NilError(t, err, "%+v", err)
		return result
	}

	for _, test := range []struct {
		name         string
		beforeDelete func(*testing.T, *v1alpha1.PostgresCluster)
		propagation  metav1.DeletionPropagation
	}{
		{
			name:        "Background",
			propagation: metav1.DeletePropagationBackground,
		},
		// TODO(cbandy): metav1.DeletePropagationForeground
		{
			name:        "AfterFailover",
			propagation: metav1.DeletePropagationBackground,
			beforeDelete: func(t *testing.T, cluster *v1alpha1.PostgresCluster) {
				list := v1.PodList{}
				selector, err := labels.Parse(strings.Join([]string{
					"postgres-operator.crunchydata.com/cluster=" + cluster.Name,
					"postgres-operator.crunchydata.com/instance",
				}, ","))
				assert.NilError(t, err)
				assert.NilError(t, cc.List(ctx, &list,
					client.InNamespace(cluster.Namespace),
					client.MatchingLabelsSelector{Selector: selector}))

				var primary *v1.Pod
				var replica *v1.Pod
				for i := range list.Items {
					if list.Items[i].Labels["postgres-operator.crunchydata.com/role"] == "replica" {
						replica = &list.Items[i]
					} else {
						primary = &list.Items[i]
					}
				}
				assert.Assert(t, primary != nil, "expected to find a primary in %+v", list.Items)
				assert.Assert(t, replica != nil, "expected to find a replica in %+v", list.Items)

				assert.NilError(t, patroni.Executor(func(_ context.Context, stdin io.Reader, stdout, stderr io.Writer, command ...string) error {
					return reconciler.PodExec(replica.Namespace, replica.Name, "database", stdin, stdout, stderr, command...)
				}).ChangePrimary(ctx, primary.Name, replica.Name))
			},
		},
	} {
		t.Run(test.name, func(t *testing.T) {
			g := gomega.NewWithT(t)

			cluster := &v1alpha1.PostgresCluster{}
			assert.NilError(t, yaml.Unmarshal([]byte(`{
				spec: {
					postgresVersion: 12,
					instances: [
						{ replicas: 2 },
					],
				},
			}`), cluster))

			cluster.Namespace = ns.Name
			cluster.Name = strings.ToLower(test.name)
			assert.NilError(t, cc.Create(ctx, cluster))

			t.Cleanup(func() {
				// Remove finalizers, if any, so the namespace can terminate.
				assert.Check(t, client.IgnoreNotFound(
					cc.Patch(ctx, cluster, client.RawPatch(
						client.Merge.Type(), []byte(`{"metadata":{"finalizers":[]}}`)))))
			})

			// Start cluster.
			mustReconcile(t, cluster)

			assert.NilError(t,
				cc.Get(ctx, client.ObjectKeyFromObject(cluster), cluster))
			g.Expect(cluster.Finalizers).To(
				gomega.ContainElement("postgres-operator.crunchydata.com/finalizer"),
				"cluster should immediately have a finalizer")

			// Continue until instances are healthy.
			g.Eventually(func() (instances []appsv1.StatefulSet) {
				mustReconcile(t, cluster)

				list := appsv1.StatefulSetList{}
				selector, err := labels.Parse(strings.Join([]string{
					"postgres-operator.crunchydata.com/cluster=" + cluster.Name,
					"postgres-operator.crunchydata.com/instance",
				}, ","))
				assert.NilError(t, err)
				assert.NilError(t, cc.List(ctx, &list,
					client.InNamespace(cluster.Namespace),
					client.MatchingLabelsSelector{Selector: selector}))
				return list.Items
			},
				"30s", // timeout
				"1s",  // interval
			).Should(gstruct.MatchElements(
				func(interface{}) string { return "each" },
				gstruct.AllowDuplicates,
				gstruct.Elements{
					"each": gomega.WithTransform(
						func(sts appsv1.StatefulSet) int32 {
							return sts.Status.ReadyReplicas
						},
						gomega.BeEquivalentTo(1),
					),
				},
			))

			if test.beforeDelete != nil {
				test.beforeDelete(t, cluster)
			}

			switch test.propagation {
			case metav1.DeletePropagationBackground:
				// Background deletion is the default for custom resources.
				// - https://issue.k8s.io/81628
				assert.NilError(t, cc.Delete(ctx, cluster))
			default:
				assert.NilError(t, cc.Delete(ctx, cluster,
					client.PropagationPolicy(test.propagation)))
			}

			// Stop cluster.
			result := mustReconcile(t, cluster)

			// Replicas should stop first, leaving just the one primary.
			g.Eventually(func() (instances []v1.Pod) {
				if result.Requeue {
					result = mustReconcile(t, cluster)
				}

				list := v1.PodList{}
				selector, err := labels.Parse(strings.Join([]string{
					"postgres-operator.crunchydata.com/cluster=" + cluster.Name,
					"postgres-operator.crunchydata.com/instance",
				}, ","))
				assert.NilError(t, err)
				assert.NilError(t, cc.List(ctx, &list,
					client.InNamespace(cluster.Namespace),
					client.MatchingLabelsSelector{Selector: selector}))
				return list.Items
			},
				"30s", // timeout
				"1s",  // interval
			).Should(gomega.ConsistOf(
				gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
					"ObjectMeta": gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
						"Labels": gstruct.MatchKeys(gstruct.IgnoreExtras, gstruct.Keys{
							// Patroni doesn't use "primary" to identify the primary.
							"postgres-operator.crunchydata.com/role": gomega.Equal("master"),
						}),
					}),
				}),
			), "expected one instance")

			// Patroni DCS objects should not be deleted yet.
			{
				list := v1.EndpointsList{}
				selector, err := labels.Parse(strings.Join([]string{
					"postgres-operator.crunchydata.com/cluster=" + cluster.Name,
					"postgres-operator.crunchydata.com/patroni",
				}, ","))
				assert.NilError(t, err)
				assert.NilError(t, cc.List(ctx, &list,
					client.InNamespace(cluster.Namespace),
					client.MatchingLabelsSelector{Selector: selector}))

				assert.Assert(t, len(list.Items) >= 2, // config + leader
					"expected Patroni DCS objects to remain, there are %v",
					len(list.Items))

				// Endpoints are deleted differently than other resources, and
				// Patroni might have recreated them to stay alive. Check that
				// they are all from before the cluster delete operation.
				// - https://issue.k8s.io/99407
				assert.NilError(t,
					cc.Get(ctx, client.ObjectKeyFromObject(cluster), cluster))
				g.Expect(list.Items).To(gstruct.MatchElements(
					func(interface{}) string { return "each" },
					gstruct.AllowDuplicates,
					gstruct.Elements{
						"each": gomega.WithTransform(
							func(ep v1.Endpoints) time.Time {
								return ep.CreationTimestamp.Time
							},
							gomega.BeTemporally("<", cluster.DeletionTimestamp.Time),
						),
					},
				))
			}

			// Continue until cluster is gone.
			g.Eventually(func() error {
				mustReconcile(t, cluster)

				return cc.Get(ctx, client.ObjectKeyFromObject(cluster), cluster)
			},
				"30s", // timeout
				"1s",  // interval
			).Should(
				gomega.SatisfyAll(
					// https://github.com/onsi/gomega/issues/420
					gomega.Not(gomega.BeNil()),
					gomega.WithTransform(apierrors.IsNotFound, gomega.BeTrue()),
				),
			)

			g.Eventually(func() []v1.Endpoints {
				list := v1.EndpointsList{}
				selector, err := labels.Parse(strings.Join([]string{
					"postgres-operator.crunchydata.com/cluster=" + cluster.Name,
					"postgres-operator.crunchydata.com/patroni",
				}, ","))
				assert.NilError(t, err)
				assert.NilError(t, cc.List(ctx, &list,
					client.InNamespace(cluster.Namespace),
					client.MatchingLabelsSelector{Selector: selector}))
				return list.Items
			},
				"10s", // timeout
				"1s",  // interval
			).Should(gomega.BeEmpty(), "Patroni DCS objects should be gone")
		})
	}
}
<commit_msg>Set Operator Namespace for test<commit_after>// +build envtest

/*
 Copyright 2021 Crunchy Data Solutions, Inc.
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package postgrescluster

import (
	"context"
	"io"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/onsi/gomega"
	"github.com/onsi/gomega/gstruct"
	"go.opentelemetry.io/otel"
	"gotest.tools/v3/assert"
	appsv1 "k8s.io/api/apps/v1"
	v1 "k8s.io/api/core/v1"
	rbacv1 "k8s.io/api/rbac/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/client-go/kubernetes/scheme"
	"k8s.io/client-go/tools/record"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/envtest"
	"sigs.k8s.io/controller-runtime/pkg/reconcile"
	"sigs.k8s.io/yaml"

	"github.com/crunchydata/postgres-operator/internal/naming"
	"github.com/crunchydata/postgres-operator/internal/patroni"
	"github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1alpha1"
)

func TestReconcilerHandleDelete(t *testing.T) {
	if !strings.EqualFold(os.Getenv("USE_EXISTING_CLUSTER"), "true") {
		t.Skip("requires a running garbage collection controller")
	}
	t.Parallel()

	ctx := context.Background()
	env := &envtest.Environment{
		CRDDirectoryPaths: []string{
			filepath.Join("..", "..", "..", "config", "crd", "bases"),
		},
	}

	options := client.Options{}
	options.Scheme = runtime.NewScheme()
	assert.NilError(t, scheme.AddToScheme(options.Scheme))
	assert.NilError(t, v1alpha1.AddToScheme(options.Scheme))

	config, err := env.Start()
	assert.NilError(t, err)
	t.Cleanup(func() { assert.Check(t, env.Stop()) })

	cc, err := client.New(config, options)
	assert.NilError(t, err)

	ns := &v1.Namespace{}
	ns.GenerateName = "postgres-operator-test-"
	ns.Labels = labels.Set{"postgres-operator-test": t.Name()}
	assert.NilError(t, cc.Create(ctx, ns))
	t.Cleanup(func() { assert.Check(t, cc.Delete(ctx, ns)) })

	// set the operator namespace variable
	naming.PostgresOperatorNamespace = ns.Name

	// TODO(cbandy): namespace rbac
	assert.NilError(t, cc.Create(ctx, &v1.ServiceAccount{
		ObjectMeta: metav1.ObjectMeta{Namespace: ns.Name, Name: "postgres-operator"},
	}))
	assert.NilError(t, cc.Create(ctx, &rbacv1.RoleBinding{
		ObjectMeta: metav1.ObjectMeta{Namespace: ns.Name, Name: "postgres-operator"},
		RoleRef: rbacv1.RoleRef{
			Kind: "ClusterRole", Name: "postgres-operator-role",
		},
		Subjects: []rbacv1.Subject{{
			Kind: "ServiceAccount", Namespace: ns.Name, Name: "postgres-operator",
		}},
	}))

	reconciler := Reconciler{
		Client:   cc,
		Owner:    client.FieldOwner(t.Name()),
		Recorder: new(record.FakeRecorder),
		Tracer:   otel.Tracer(t.Name()),
	}

	reconciler.PodExec, err = newPodExecutor(config)
	assert.NilError(t, err)

	mustReconcile := func(t *testing.T, cluster *v1alpha1.PostgresCluster) reconcile.Result {
		t.Helper()
		key := client.ObjectKeyFromObject(cluster)
		request := reconcile.Request{NamespacedName: key}
		result, err := reconciler.Reconcile(ctx, request)
		assert.NilError(t, err, "%+v", err)
		return result
	}

	for _, test := range []struct {
		name         string
		beforeDelete func(*testing.T, *v1alpha1.PostgresCluster)
		propagation  metav1.DeletionPropagation
	}{
		{
			name:        "Background",
			propagation: metav1.DeletePropagationBackground,
		},
		// TODO(cbandy): metav1.DeletePropagationForeground
		{
			name:        "AfterFailover",
			propagation: metav1.DeletePropagationBackground,
			beforeDelete: func(t *testing.T, cluster *v1alpha1.PostgresCluster) {
				list := v1.PodList{}
				selector, err := labels.Parse(strings.Join([]string{
					"postgres-operator.crunchydata.com/cluster=" + cluster.Name,
					"postgres-operator.crunchydata.com/instance",
				}, ","))
				assert.NilError(t, err)
				assert.NilError(t, cc.List(ctx, &list,
					client.InNamespace(cluster.Namespace),
					client.MatchingLabelsSelector{Selector: selector}))

				var primary *v1.Pod
				var replica *v1.Pod
				for i := range list.Items {
					if list.Items[i].Labels["postgres-operator.crunchydata.com/role"] == "replica" {
						replica = &list.Items[i]
					} else {
						primary = &list.Items[i]
					}
				}
				assert.Assert(t, primary != nil, "expected to find a primary in %+v", list.Items)
				assert.Assert(t, replica != nil, "expected to find a replica in %+v", list.Items)

				assert.NilError(t, patroni.Executor(func(_ context.Context, stdin io.Reader, stdout, stderr io.Writer, command ...string) error {
					return reconciler.PodExec(replica.Namespace, replica.Name, "database", stdin, stdout, stderr, command...)
				}).ChangePrimary(ctx, primary.Name, replica.Name))
			},
		},
	} {
		t.Run(test.name, func(t *testing.T) {
			g := gomega.NewWithT(t)

			cluster := &v1alpha1.PostgresCluster{}
			assert.NilError(t, yaml.Unmarshal([]byte(`{
				spec: {
					postgresVersion: 12,
					instances: [
						{ replicas: 2 },
					],
				},
			}`), cluster))

			cluster.Namespace = ns.Name
			cluster.Name = strings.ToLower(test.name)
			assert.NilError(t, cc.Create(ctx, cluster))

			t.Cleanup(func() {
				// Remove finalizers, if any, so the namespace can terminate.
				assert.Check(t, client.IgnoreNotFound(
					cc.Patch(ctx, cluster, client.RawPatch(
						client.Merge.Type(), []byte(`{"metadata":{"finalizers":[]}}`)))))
			})

			// Start cluster.
			mustReconcile(t, cluster)

			assert.NilError(t,
				cc.Get(ctx, client.ObjectKeyFromObject(cluster), cluster))
			g.Expect(cluster.Finalizers).To(
				gomega.ContainElement("postgres-operator.crunchydata.com/finalizer"),
				"cluster should immediately have a finalizer")

			// Continue until instances are healthy.
			g.Eventually(func() (instances []appsv1.StatefulSet) {
				mustReconcile(t, cluster)

				list := appsv1.StatefulSetList{}
				selector, err := labels.Parse(strings.Join([]string{
					"postgres-operator.crunchydata.com/cluster=" + cluster.Name,
					"postgres-operator.crunchydata.com/instance",
				}, ","))
				assert.NilError(t, err)
				assert.NilError(t, cc.List(ctx, &list,
					client.InNamespace(cluster.Namespace),
					client.MatchingLabelsSelector{Selector: selector}))
				return list.Items
			},
				"30s", // timeout
				"1s",  // interval
			).Should(gstruct.MatchElements(
				func(interface{}) string { return "each" },
				gstruct.AllowDuplicates,
				gstruct.Elements{
					"each": gomega.WithTransform(
						func(sts appsv1.StatefulSet) int32 {
							return sts.Status.ReadyReplicas
						},
						gomega.BeEquivalentTo(1),
					),
				},
			))

			if test.beforeDelete != nil {
				test.beforeDelete(t, cluster)
			}

			switch test.propagation {
			case metav1.DeletePropagationBackground:
				// Background deletion is the default for custom resources.
				// - https://issue.k8s.io/81628
				assert.NilError(t, cc.Delete(ctx, cluster))
			default:
				assert.NilError(t, cc.Delete(ctx, cluster,
					client.PropagationPolicy(test.propagation)))
			}

			// Stop cluster.
			result := mustReconcile(t, cluster)

			// Replicas should stop first, leaving just the one primary.
			g.Eventually(func() (instances []v1.Pod) {
				if result.Requeue {
					result = mustReconcile(t, cluster)
				}

				list := v1.PodList{}
				selector, err := labels.Parse(strings.Join([]string{
					"postgres-operator.crunchydata.com/cluster=" + cluster.Name,
					"postgres-operator.crunchydata.com/instance",
				}, ","))
				assert.NilError(t, err)
				assert.NilError(t, cc.List(ctx, &list,
					client.InNamespace(cluster.Namespace),
					client.MatchingLabelsSelector{Selector: selector}))
				return list.Items
			},
				"30s", // timeout
				"1s",  // interval
			).Should(gomega.ConsistOf(
				gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
					"ObjectMeta": gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
						"Labels": gstruct.MatchKeys(gstruct.IgnoreExtras, gstruct.Keys{
							// Patroni doesn't use "primary" to identify the primary.
							"postgres-operator.crunchydata.com/role": gomega.Equal("master"),
						}),
					}),
				}),
			), "expected one instance")

			// Patroni DCS objects should not be deleted yet.
			{
				list := v1.EndpointsList{}
				selector, err := labels.Parse(strings.Join([]string{
					"postgres-operator.crunchydata.com/cluster=" + cluster.Name,
					"postgres-operator.crunchydata.com/patroni",
				}, ","))
				assert.NilError(t, err)
				assert.NilError(t, cc.List(ctx, &list,
					client.InNamespace(cluster.Namespace),
					client.MatchingLabelsSelector{Selector: selector}))

				assert.Assert(t, len(list.Items) >= 2, // config + leader
					"expected Patroni DCS objects to remain, there are %v",
					len(list.Items))

				// Endpoints are deleted differently than other resources, and
				// Patroni might have recreated them to stay alive. Check that
				// they are all from before the cluster delete operation.
				// - https://issue.k8s.io/99407
				assert.NilError(t,
					cc.Get(ctx, client.ObjectKeyFromObject(cluster), cluster))
				g.Expect(list.Items).To(gstruct.MatchElements(
					func(interface{}) string { return "each" },
					gstruct.AllowDuplicates,
					gstruct.Elements{
						"each": gomega.WithTransform(
							func(ep v1.Endpoints) time.Time {
								return ep.CreationTimestamp.Time
							},
							gomega.BeTemporally("<", cluster.DeletionTimestamp.Time),
						),
					},
				))
			}

			// Continue until cluster is gone.
			g.Eventually(func() error {
				mustReconcile(t, cluster)

				return cc.Get(ctx, client.ObjectKeyFromObject(cluster), cluster)
			},
				"30s", // timeout
				"1s",  // interval
			).Should(
				gomega.SatisfyAll(
					// https://github.com/onsi/gomega/issues/420
					gomega.Not(gomega.BeNil()),
					gomega.WithTransform(apierrors.IsNotFound, gomega.BeTrue()),
				),
			)

			g.Eventually(func() []v1.Endpoints {
				list := v1.EndpointsList{}
				selector, err := labels.Parse(strings.Join([]string{
					"postgres-operator.crunchydata.com/cluster=" + cluster.Name,
					"postgres-operator.crunchydata.com/patroni",
				}, ","))
				assert.NilError(t, err)
				assert.NilError(t, cc.List(ctx, &list,
					client.InNamespace(cluster.Namespace),
					client.MatchingLabelsSelector{Selector: selector}))
				return list.Items
			},
				"10s", // timeout
				"1s",  // interval
			).Should(gomega.BeEmpty(), "Patroni DCS objects should be gone")
		})
	}
}
<|endoftext|>


<commit_before>/*
Copyright The Helm Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package registry // import "helm.sh/helm/v3/internal/experimental/registry"

import (
	"encoding/json"
	"fmt"

	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
	"oras.land/oras-go/pkg/content"
	"oras.land/oras-go/pkg/oras"

	"helm.sh/helm/v3/pkg/chart"
)

// Pull downloads a chart from a registry
func (c *Client) Pull(ref string, options ...PullOption) (*PullResult, error) {
	operation := &pullOperation{
		withChart: true, // By default, always download the chart layer
	}
	for _, option := range options {
		option(operation)
	}
	if !operation.withChart && !operation.withProv {
		return nil, errors.New(
			"must specify at least one layer to pull (chart/prov)")
	}
	store := content.NewMemoryStore()
	allowedMediaTypes := []string{
		ConfigMediaType,
	}
	minNumDescriptors := 1 // 1 for the config
	if operation.withChart {
		minNumDescriptors++
		allowedMediaTypes = append(allowedMediaTypes, ChartLayerMediaType)
	}
	if operation.withProv {
		if !operation.ignoreMissingProv {
			minNumDescriptors++
		}
		allowedMediaTypes = append(allowedMediaTypes, ProvLayerMediaType)
	}
	manifest, descriptors, err := oras.Pull(ctx(c.out, c.debug), c.resolver, ref, store,
		oras.WithPullEmptyNameAllowed(),
		oras.WithAllowedMediaTypes(allowedMediaTypes))
	if err != nil {
		return nil, err
	}
	numDescriptors := len(descriptors)
	if numDescriptors < minNumDescriptors {
		return nil, errors.New(
			fmt.Sprintf("manifest does not contain minimum number of descriptors (%d), descriptors found: %d",
				minNumDescriptors, numDescriptors))
	}
	var configDescriptor *ocispec.Descriptor
	var chartDescriptor *ocispec.Descriptor
	var provDescriptor *ocispec.Descriptor
	for _, descriptor := range descriptors {
		d := descriptor
		switch d.MediaType {
		case ConfigMediaType:
			configDescriptor = &d
		case ChartLayerMediaType:
			chartDescriptor = &d
		case ProvLayerMediaType:
			provDescriptor = &d
		}
	}
	if configDescriptor == nil {
		return nil, errors.New(
			fmt.Sprintf("could not load config with mediatype %s", ConfigMediaType))
	}
	if operation.withChart && chartDescriptor == nil {
		return nil, errors.New(
			fmt.Sprintf("manifest does not contain a layer with mediatype %s",
				ChartLayerMediaType))
	}
	var provMissing bool
	if operation.withProv && provDescriptor == nil {
		if operation.ignoreMissingProv {
			provMissing = true
		} else {
			return nil, errors.New(
				fmt.Sprintf("manifest does not contain a layer with mediatype %s",
					ProvLayerMediaType))
		}
	}
	_, manifestData, ok := store.Get(manifest)
	if !ok {
		return nil, errors.Errorf("Unable to retrieve blob with digest %s", manifest.Digest)
	}
	_, configData, ok := store.Get(*configDescriptor)
	if !ok {
		return nil, errors.Errorf("Unable to retrieve blob with digest %s", configDescriptor.Digest)
	}
	var meta *chart.Metadata
	if err := json.Unmarshal(configData, &meta); err != nil {
		return nil, err
	}
	var chartData []byte
	if operation.withChart {
		var ok bool
		_, chartData, ok = store.Get(*chartDescriptor)
		if !ok {
			return nil, errors.Errorf("Unable to retrieve blob with digest %s", chartDescriptor.Digest)
		}
	}
	var provData []byte
	if operation.withProv && !provMissing {
		var ok bool
		_, provData, ok = store.Get(*provDescriptor)
		if !ok {
			return nil, errors.Errorf("Unable to retrieve blob with digest %s", provDescriptor.Digest)
		}
	}
	result := &PullResult{
		Manifest: &descriptorPullSummary{
			Data:   manifestData,
			Digest: manifest.Digest.String(),
			Size:   manifest.Size,
		},
		Config: &descriptorPullSummary{
			Data:   configData,
			Digest: configDescriptor.Digest.String(),
			Size:   configDescriptor.Size,
		},
		Chart: &descriptorPullSummaryWithMeta{
			Meta: meta,
		},
		Prov: &descriptorPullSummary{}, // prevent nil references
		Ref:  ref,
	}
	if chartData != nil {
		result.Chart.Data = chartData
		result.Chart.Digest = chartDescriptor.Digest.String()
		result.Chart.Size = chartDescriptor.Size
	}
	if provData != nil {
		result.Prov = &descriptorPullSummary{
			Data:   provData,
			Digest: provDescriptor.Digest.String(),
			Size:   provDescriptor.Size,
		}
	}
	fmt.Fprintf(c.out, "Pulled: %s\n", result.Ref)
	fmt.Fprintf(c.out, "Digest: %s\n", result.Manifest.Digest)
	return result, nil
}
<commit_msg>inline data fetching for pull result<commit_after>/*
Copyright The Helm Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package registry // import "helm.sh/helm/v3/internal/experimental/registry"

import (
	"encoding/json"
	"fmt"

	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
	"oras.land/oras-go/pkg/content"
	"oras.land/oras-go/pkg/oras"

	"helm.sh/helm/v3/pkg/chart"
)

// Pull downloads a chart from a registry
func (c *Client) Pull(ref string, options ...PullOption) (*PullResult, error) {
	operation := &pullOperation{
		withChart: true, // By default, always download the chart layer
	}
	for _, option := range options {
		option(operation)
	}
	if !operation.withChart && !operation.withProv {
		return nil, errors.New(
			"must specify at least one layer to pull (chart/prov)")
	}
	store := content.NewMemoryStore()
	allowedMediaTypes := []string{
		ConfigMediaType,
	}
	minNumDescriptors := 1 // 1 for the config
	if operation.withChart {
		minNumDescriptors++
		allowedMediaTypes = append(allowedMediaTypes, ChartLayerMediaType)
	}
	if operation.withProv {
		if !operation.ignoreMissingProv {
			minNumDescriptors++
		}
		allowedMediaTypes = append(allowedMediaTypes, ProvLayerMediaType)
	}
	manifest, descriptors, err := oras.Pull(ctx(c.out, c.debug), c.resolver, ref, store,
		oras.WithPullEmptyNameAllowed(),
		oras.WithAllowedMediaTypes(allowedMediaTypes))
	if err != nil {
		return nil, err
	}
	numDescriptors := len(descriptors)
	if numDescriptors < minNumDescriptors {
		return nil, errors.New(
			fmt.Sprintf("manifest does not contain minimum number of descriptors (%d), descriptors found: %d",
				minNumDescriptors, numDescriptors))
	}
	var configDescriptor *ocispec.Descriptor
	var chartDescriptor *ocispec.Descriptor
	var provDescriptor *ocispec.Descriptor
	for _, descriptor := range descriptors {
		d := descriptor
		switch d.MediaType {
		case ConfigMediaType:
			configDescriptor = &d
		case ChartLayerMediaType:
			chartDescriptor = &d
		case ProvLayerMediaType:
			provDescriptor = &d
		}
	}
	if configDescriptor == nil {
		return nil, errors.New(
			fmt.Sprintf("could not load config with mediatype %s", ConfigMediaType))
	}
	if operation.withChart && chartDescriptor == nil {
		return nil, errors.New(
			fmt.Sprintf("manifest does not contain a layer with mediatype %s",
				ChartLayerMediaType))
	}
	var provMissing bool
	if operation.withProv && provDescriptor == nil {
		if operation.ignoreMissingProv {
			provMissing = true
		} else {
			return nil, errors.New(
				fmt.Sprintf("manifest does not contain a layer with mediatype %s",
					ProvLayerMediaType))
		}
	}
	result := &PullResult{
		Manifest: &descriptorPullSummary{
			Digest: manifest.Digest.String(),
			Size:   manifest.Size,
		},
		Config: &descriptorPullSummary{
			Digest: configDescriptor.Digest.String(),
			Size:   configDescriptor.Size,
		},
		Chart: &descriptorPullSummaryWithMeta{},
		Prov: &descriptorPullSummary{},
		Ref:  ref,
	}
	if _, manifestData, ok := store.Get(manifest); !ok {
		return nil, errors.Errorf("Unable to retrieve blob with digest %s", manifest.Digest)
	} else {
		result.Manifest.Data = manifestData
	}
	if _, configData, ok := store.Get(*configDescriptor); !ok {
		return nil, errors.Errorf("Unable to retrieve blob with digest %s", configDescriptor.Digest)
	} else {
		result.Config.Data = configData
		var meta *chart.Metadata
		if err := json.Unmarshal(configData, &meta); err != nil {
			return nil, err
		}
		result.Chart.Meta = meta
	}
	if operation.withChart {
		if _, chartData, ok := store.Get(*chartDescriptor); !ok {
			return nil, errors.Errorf("Unable to retrieve blob with digest %s", chartDescriptor.Digest)
		} else {
			result.Chart.Data = chartData
			result.Chart.Digest = chartDescriptor.Digest.String()
			result.Chart.Size = chartDescriptor.Size
		}
	}
	if operation.withProv && !provMissing {
		if _, provData, ok := store.Get(*provDescriptor); !ok {
			return nil, errors.Errorf("Unable to retrieve blob with digest %s", provDescriptor.Digest)
		} else {
			result.Prov.Data = provData
			result.Prov.Digest = provDescriptor.Digest.String()
			result.Prov.Size = provDescriptor.Size
		}
	}
	fmt.Fprintf(c.out, "Pulled: %s\n", result.Ref)
	fmt.Fprintf(c.out, "Digest: %s\n", result.Manifest.Digest)
	return result, nil
}
<|endoftext|>


<commit_before>/*
 * Copyright 2018 Google Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Package extractor provides a library for running kythe extraction on a single
// repository.
package config

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

// kytheConfigFileName The name of the Kythe extraction config
const kytheExtractionConfigFile = ".kythe-extraction-config"

// ExtractRepo extracts a given code repository and outputs kindex files.
//
// It makes a local clone of the repository. It optionally uses a passed
// extraction config, otherwise it attempts to find a Kythe config named
// ".kythe-extraction-config".
//
// It builds a one-off customized Docker image for extraction, and then runs it,
// generating kindex files (format defined here:
// http://kythe.io/docs/kythe-index-pack.html).
//
// This function requires both Git and Docker to be in $PATH during execution.
func ExtractRepo(repoURI, outputPath, configPath string) error {
	if err := verifyRequiredTools(); err != nil {
		return fmt.Errorf("ExtractRepo requires git and docker to be in $PATH: %v", err)
	}

	// create a temporary directory for the repo clone
	repoDir, err := ioutil.TempDir("", "repoDir")
	if err != nil {
		return fmt.Errorf("creating tmp repo dir: %v", err)
	}
	defer os.RemoveAll(repoDir)

	// create a temporary directory for the extraction output
	tmpOutDir, err := ioutil.TempDir("", "tmpOutDir")
	if err != nil {
		return fmt.Errorf("creating tmp out dir: %v", err)
	}
	defer os.RemoveAll(tmpOutDir)

	// clone the repo
	cmd := exec.Command("git", "clone", repoURI, repoDir)
	err = cmd.Run()
	if err != nil {
		return fmt.Errorf("cloning repo: %v", err)
	}

	// if a config was passed as a arg, use the specified config
	if configPath == "" {
		// otherwise, use a Kythe config within the repo (if it exists)
		configPath = filepath.Join(repoDir, kytheExtractionConfigFile)
	}

	log.Printf("Using configuration file: %q\n", configPath)
	extractionDockerFile, err := ioutil.TempFile(tmpOutDir, "extractionDockerFile")
	if err != nil {
		return fmt.Errorf("creating tmp Dockerfile: %v", err)
	}

	// generate an extraction image from the config
	configFile, err := os.Open(configPath)
	if err != nil {
		return fmt.Errorf("opening config file: %v", err)
	}

	extractionConfig, err := Load(configFile)
	if err != nil {
		return fmt.Errorf("loading extraction config: %v", err)
	}

	err = CreateImage(extractionDockerFile.Name(), extractionConfig)
	if err != nil {
		return fmt.Errorf("creating extraction image: %v", err)
	}

	// use Docker to build the extraction image
	imageTag := strings.ToLower(filepath.Base(extractionDockerFile.Name()))
	output, err := exec.Command("docker", "build", "-f", extractionDockerFile.Name(), "-t", imageTag, tmpOutDir).CombinedOutput()
	defer mustCleanUpImage(imageTag)
	if err != nil {
		return fmt.Errorf("building docker image: %v\nCommand output %s", err, string(output))
	}

	// run the extraction
	output, err = exec.Command("docker", "run", "--rm", "-v", fmt.Sprintf("%s:%s", repoDir, DefaultRepoVolume), "-v", fmt.Sprintf("%s:%s", outputPath, DefaultOutputVolume), "-t", imageTag).CombinedOutput()
	if err != nil {
		return fmt.Errorf("extracting repo: %v\nCommand output: %s", err, string(output))
	}

	return nil
}

func verifyRequiredTools() error {
	if _, err := exec.LookPath("git"); err != nil {
		return err
	}

	if _, err := exec.LookPath("docker"); err != nil {
		return err
	}
	return nil
}

func mustCleanUpImage(tmpImageTag string) {
	cmd := exec.Command("docker", "image", "rm", tmpImageTag)
	err := cmd.Run()
	if err != nil {
		log.Printf("Failed to clean up docker image: %v", err)
	}
}
<commit_msg>Fix golint error only caught downstream.<commit_after>/*
 * Copyright 2018 Google Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package config

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

// kytheConfigFileName The name of the Kythe extraction config
const kytheExtractionConfigFile = ".kythe-extraction-config"

// ExtractRepo extracts a given code repository and outputs kindex files.
//
// It makes a local clone of the repository. It optionally uses a passed
// extraction config, otherwise it attempts to find a Kythe config named
// ".kythe-extraction-config".
//
// It builds a one-off customized Docker image for extraction, and then runs it,
// generating kindex files (format defined here:
// http://kythe.io/docs/kythe-index-pack.html).
//
// This function requires both Git and Docker to be in $PATH during execution.
func ExtractRepo(repoURI, outputPath, configPath string) error {
	if err := verifyRequiredTools(); err != nil {
		return fmt.Errorf("ExtractRepo requires git and docker to be in $PATH: %v", err)
	}

	// create a temporary directory for the repo clone
	repoDir, err := ioutil.TempDir("", "repoDir")
	if err != nil {
		return fmt.Errorf("creating tmp repo dir: %v", err)
	}
	defer os.RemoveAll(repoDir)

	// create a temporary directory for the extraction output
	tmpOutDir, err := ioutil.TempDir("", "tmpOutDir")
	if err != nil {
		return fmt.Errorf("creating tmp out dir: %v", err)
	}
	defer os.RemoveAll(tmpOutDir)

	// clone the repo
	cmd := exec.Command("git", "clone", repoURI, repoDir)
	err = cmd.Run()
	if err != nil {
		return fmt.Errorf("cloning repo: %v", err)
	}

	// if a config was passed as a arg, use the specified config
	if configPath == "" {
		// otherwise, use a Kythe config within the repo (if it exists)
		configPath = filepath.Join(repoDir, kytheExtractionConfigFile)
	}

	log.Printf("Using configuration file: %q\n", configPath)
	extractionDockerFile, err := ioutil.TempFile(tmpOutDir, "extractionDockerFile")
	if err != nil {
		return fmt.Errorf("creating tmp Dockerfile: %v", err)
	}

	// generate an extraction image from the config
	configFile, err := os.Open(configPath)
	if err != nil {
		return fmt.Errorf("opening config file: %v", err)
	}

	extractionConfig, err := Load(configFile)
	if err != nil {
		return fmt.Errorf("loading extraction config: %v", err)
	}

	err = CreateImage(extractionDockerFile.Name(), extractionConfig)
	if err != nil {
		return fmt.Errorf("creating extraction image: %v", err)
	}

	// use Docker to build the extraction image
	imageTag := strings.ToLower(filepath.Base(extractionDockerFile.Name()))
	output, err := exec.Command("docker", "build", "-f", extractionDockerFile.Name(), "-t", imageTag, tmpOutDir).CombinedOutput()
	defer mustCleanUpImage(imageTag)
	if err != nil {
		return fmt.Errorf("building docker image: %v\nCommand output %s", err, string(output))
	}

	// run the extraction
	output, err = exec.Command("docker", "run", "--rm", "-v", fmt.Sprintf("%s:%s", repoDir, DefaultRepoVolume), "-v", fmt.Sprintf("%s:%s", outputPath, DefaultOutputVolume), "-t", imageTag).CombinedOutput()
	if err != nil {
		return fmt.Errorf("extracting repo: %v\nCommand output: %s", err, string(output))
	}

	return nil
}

func verifyRequiredTools() error {
	if _, err := exec.LookPath("git"); err != nil {
		return err
	}

	if _, err := exec.LookPath("docker"); err != nil {
		return err
	}
	return nil
}

func mustCleanUpImage(tmpImageTag string) {
	cmd := exec.Command("docker", "image", "rm", tmpImageTag)
	err := cmd.Run()
	if err != nil {
		log.Printf("Failed to clean up docker image: %v", err)
	}
}
<|endoftext|>


<commit_before>/*
 * Copyright 2014 The Kythe Authors. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package vnameutil

import (
	"encoding/json"
	"regexp"
	"strconv"
	"strings"
	"testing"

	"github.com/golang/protobuf/proto"
	"github.com/google/go-cmp/cmp"

	spb "kythe.io/kythe/proto/storage_go_proto"
)

// Copied exactly from kythe/javatests/com/google/devtools/kythe/extractors/shared/FileVNamesTest.java
// This could be prettier in Go but a copy ensures better compatibility between
// the libraries.
var testConfig = strings.Join([]string{
	"[",
	"  {",
	"    \"pattern\": \"static/path\",",
	"    \"vname\": {",
	"      \"root\": \"root\",",
	"      \"corpus\": \"static\"",
	"    }",
	"  },",
	"  {",
	"    \"pattern\": \"dup/path\",",
	"    \"vname\": {",
	"      \"corpus\": \"first\"",
	"    }",
	"  },",
	"  {",
	"    \"pattern\": \"dup/path2\",",
	"    \"vname\": {",
	"      \"corpus\": \"second\"",
	"    }",
	"  },",
	"  {",
	"    \"pattern\": \"(grp1)/(\\\\d+)/(.*)\",",
	"    \"vname\": {",
	"      \"root\": \"@2@\",",
	"      \"corpus\": \"@1@/@3@\"",
	"    }",
	"  },",
	"  {",
	"    \"pattern\": \"bazel-bin/([^/]+)/java/.*[.]jar!/.*\",",
	"    \"vname\": {",
	"      \"root\": \"java\",",
	"      \"corpus\": \"@1@\"",
	"    }",
	"  },",
	"  {",
	"    \"pattern\": \"third_party/([^/]+)/.*[.]jar!/.*\",",
	"    \"vname\": {",
	"      \"root\": \"@1@\",",
	"      \"corpus\": \"third_party\"",
	"    }",
	"  },",
	"  {",
	"    \"pattern\": \"([^/]+)/java/.*\",",
	"    \"vname\": {",
	"      \"root\": \"java\",",
	"      \"corpus\": \"@1@\"",
	"    }",
	"  },",
	"  {",
	"    \"pattern\": \"([^/]+)/.*\",",
	"    \"vname\": {",
	"      \"corpus\": \"@1@\"",
	"    }",
	"  }",
	"]"}, "\n")

// Verify that parsing in the Go implementation is consistent with the Java
// implementation.
func TestParseConsistency(t *testing.T) {
	r, err := ParseRules([]byte(testConfig))
	if err != nil {
		t.Error(err)
	} else if len(r) == 0 {
		t.Error("empty rules")
	}
}

func TestRoundtripJSON(t *testing.T) {
	tests := []string{
		"[]",
		`[{"pattern": "p(.)", "vname": {"corpus": "$@1@"}}]`,
		`[{
  "pattern": "(?P<corpus>.+)/(?P<root>.+)::(?P<path>.+)",
  "vname": {"corpus": "@corpus@", "root": "@root@", "path": "@path@"}
}]`,
		testConfig,
	}

	for i, test := range tests {
		t.Run(strconv.Itoa(i), func(t *testing.T) {
			expected, err := ReadRules(strings.NewReader(test))
			if err != nil {
				t.Fatal(err)
			}
			t.Logf("Rules: %s", expected)

			rec, err := json.Marshal(expected)
			if err != nil {
				t.Fatalf("Error marshaling rules %+v: %v", expected, err)
			}

			r, err := ParseRules(rec)
			if err != nil {
				t.Fatalf("Error parsing rules %q: %v", rec, err)
			}

			if diff := cmp.Diff(r, expected, transformRegexp); diff != "" {
				t.Errorf("Unexpected diff (- found; + expected):\n%s", diff)
			}
		})
	}
}

func TestRoundtripProto(t *testing.T) {
	tests := []string{
		"[]",
		`[{"pattern": "p(.)", "vname": {"corpus": "$@1@"}}]`,
		`[{
  "pattern": "(?P<corpus>.+)/(?P<root>.+)::(?P<path>.+)",
  "vname": {"corpus": "@corpus@", "root": "@root@", "path": "@path@"}
}]`,
		testConfig,
	}

	for i, test := range tests {
		t.Run(strconv.Itoa(i), func(t *testing.T) {
			expected, err := ReadRules(strings.NewReader(test))
			if err != nil {
				t.Fatal(err)
			}
			t.Logf("Rules: %s", expected)

			rec, err := expected.Marshal()
			if err != nil {
				t.Fatalf("Error marshaling rules %+v: %v", expected, err)
			}

			r, err := ParseProtoRules(rec)
			if err != nil {
				t.Fatalf("Error parsing rules %q: %v", rec, err)
			}

			if diff := cmp.Diff(r, expected, transformRegexp); diff != "" {
				t.Errorf("Unexpected diff (- found; + expected):\n%s", diff)
			}
		})
	}
}

func TestCornerCases(t *testing.T) {
	testRule1 := Rule{regexp.MustCompile(`(?P<first>\w+)(?:/(?P<second>\w+))?`), V{Corpus: "${first}", Path: "${second}"}.pb()}
	testRule2 := Rule{regexp.MustCompile(`x/(?P<sig>\w+)/y/(?P<tail>.+)$`), V{Path: "${tail}", Sig: "|${sig}|"}.pb()}
	tests := []struct {
		rule  Rule
		input string
		want  *spb.VName
	}{
		// Optional portions of the pattern should be handled correctly.
		{testRule1, "alpha/bravo", V{Corpus: "alpha", Path: "bravo"}.pb()},
		{testRule1, "alpha", V{Corpus: "alpha"}.pb()},

		// Substitution of signature fields should work.
		{testRule2, "x/kanga/y/roo.txt", V{Path: "roo.txt", Sig: "|kanga|"}.pb()},
	}
	for _, test := range tests {
		got, ok := test.rule.Apply(test.input)
		if !ok {
			t.Errorf("Apply %v failed", test.rule)
		} else if !proto.Equal(got, test.want) {
			t.Errorf("Apply %v: got {%+v}, want {%+v}", test.rule, got, test.want)
		} else {
			t.Logf("Apply %v properly returned {%+v}", test.rule, got)
		}
	}
}

func TestFileRewrites(t *testing.T) {
	tests := []struct {
		path string
		want *spb.VName
	}{
		// static
		{"static/path", V{Corpus: "static", Root: "root"}.pb()},

		// ordered
		{"dup/path", V{Corpus: "first"}.pb()},
		{"dup/path2", V{Corpus: "second"}.pb()},

		// groups
		{"corpus/some/path/here", V{Corpus: "corpus"}.pb()},
		{"grp1/12345/endingGroup", V{Corpus: "grp1/endingGroup", Root: "12345"}.pb()},
		{"bazel-bin/kythe/java/some/path/A.jar!/some/path/A.class", V{Corpus: "kythe", Root: "java"}.pb()},
		{"kythe/java/com/google/devtools/kythe/util/KytheURI.java", V{Corpus: "kythe", Root: "java"}.pb()},
		{"otherCorpus/java/com/google/devtools/kythe/util/KytheURI.java", V{Corpus: "otherCorpus", Root: "java"}.pb()},
	}

	r, err := ParseRules([]byte(testConfig))
	if err != nil {
		t.Fatalf("Broken test rules: %v", err)
	}

	for _, test := range tests {
		got, ok := r.Apply(test.path)
		if !ok {
			t.Errorf("Apply(%q): no match", test.path)
		} else if !proto.Equal(got, test.want) {
			t.Errorf("Apply(%q): got {%+v}, want {%+v}", test.path, got, test.want)
		}
	}
}

type V struct {
	Corpus, Root, Path, Sig, Lang string
}

func (v V) pb() *spb.VName {
	return &spb.VName{
		Corpus:    v.Corpus,
		Root:      v.Root,
		Path:      v.Path,
		Signature: v.Sig,
		Language:  v.Lang,
	}
}

var (
	transformRegexp = cmp.Transformer("Regexp", func(r *regexp.Regexp) string { return r.String() })
)
<commit_msg>test: ensure protos are compared properly (#4350)<commit_after>/*
 * Copyright 2014 The Kythe Authors. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package vnameutil

import (
	"encoding/json"
	"regexp"
	"strconv"
	"strings"
	"testing"

	"github.com/golang/protobuf/proto"
	"github.com/google/go-cmp/cmp"

	spb "kythe.io/kythe/proto/storage_go_proto"
)

// Copied exactly from kythe/javatests/com/google/devtools/kythe/extractors/shared/FileVNamesTest.java
// This could be prettier in Go but a copy ensures better compatibility between
// the libraries.
var testConfig = strings.Join([]string{
	"[",
	"  {",
	"    \"pattern\": \"static/path\",",
	"    \"vname\": {",
	"      \"root\": \"root\",",
	"      \"corpus\": \"static\"",
	"    }",
	"  },",
	"  {",
	"    \"pattern\": \"dup/path\",",
	"    \"vname\": {",
	"      \"corpus\": \"first\"",
	"    }",
	"  },",
	"  {",
	"    \"pattern\": \"dup/path2\",",
	"    \"vname\": {",
	"      \"corpus\": \"second\"",
	"    }",
	"  },",
	"  {",
	"    \"pattern\": \"(grp1)/(\\\\d+)/(.*)\",",
	"    \"vname\": {",
	"      \"root\": \"@2@\",",
	"      \"corpus\": \"@1@/@3@\"",
	"    }",
	"  },",
	"  {",
	"    \"pattern\": \"bazel-bin/([^/]+)/java/.*[.]jar!/.*\",",
	"    \"vname\": {",
	"      \"root\": \"java\",",
	"      \"corpus\": \"@1@\"",
	"    }",
	"  },",
	"  {",
	"    \"pattern\": \"third_party/([^/]+)/.*[.]jar!/.*\",",
	"    \"vname\": {",
	"      \"root\": \"@1@\",",
	"      \"corpus\": \"third_party\"",
	"    }",
	"  },",
	"  {",
	"    \"pattern\": \"([^/]+)/java/.*\",",
	"    \"vname\": {",
	"      \"root\": \"java\",",
	"      \"corpus\": \"@1@\"",
	"    }",
	"  },",
	"  {",
	"    \"pattern\": \"([^/]+)/.*\",",
	"    \"vname\": {",
	"      \"corpus\": \"@1@\"",
	"    }",
	"  }",
	"]"}, "\n")

// Verify that parsing in the Go implementation is consistent with the Java
// implementation.
func TestParseConsistency(t *testing.T) {
	r, err := ParseRules([]byte(testConfig))
	if err != nil {
		t.Error(err)
	} else if len(r) == 0 {
		t.Error("empty rules")
	}
}

func TestRoundtripJSON(t *testing.T) {
	tests := []string{
		"[]",
		`[{"pattern": "p(.)", "vname": {"corpus": "$@1@"}}]`,
		`[{
  "pattern": "(?P<corpus>.+)/(?P<root>.+)::(?P<path>.+)",
  "vname": {"corpus": "@corpus@", "root": "@root@", "path": "@path@"}
}]`,
		testConfig,
	}

	for i, test := range tests {
		t.Run(strconv.Itoa(i), func(t *testing.T) {
			expected, err := ReadRules(strings.NewReader(test))
			if err != nil {
				t.Fatal(err)
			}
			t.Logf("Rules: %s", expected)

			rec, err := json.Marshal(expected)
			if err != nil {
				t.Fatalf("Error marshaling rules %+v: %v", expected, err)
			}

			r, err := ParseRules(rec)
			if err != nil {
				t.Fatalf("Error parsing rules %q: %v", rec, err)
			}

			if diff := cmp.Diff(r, expected, transformRegexp, compareVNames); diff != "" {
				t.Errorf("Unexpected diff (- found; + expected):\n%s", diff)
			}
		})
	}
}

func TestRoundtripProto(t *testing.T) {
	tests := []string{
		"[]",
		`[{"pattern": "p(.)", "vname": {"corpus": "$@1@"}}]`,
		`[{
  "pattern": "(?P<corpus>.+)/(?P<root>.+)::(?P<path>.+)",
  "vname": {"corpus": "@corpus@", "root": "@root@", "path": "@path@"}
}]`,
		testConfig,
	}

	for i, test := range tests {
		t.Run(strconv.Itoa(i), func(t *testing.T) {
			expected, err := ReadRules(strings.NewReader(test))
			if err != nil {
				t.Fatal(err)
			}
			t.Logf("Rules: %s", expected)

			rec, err := expected.Marshal()
			if err != nil {
				t.Fatalf("Error marshaling rules %+v: %v", expected, err)
			}

			r, err := ParseProtoRules(rec)
			if err != nil {
				t.Fatalf("Error parsing rules %q: %v", rec, err)
			}

			if diff := cmp.Diff(r, expected, transformRegexp, compareVNames); diff != "" {
				t.Errorf("Unexpected diff (- found; + expected):\n%s", diff)
			}
		})
	}
}

func TestCornerCases(t *testing.T) {
	testRule1 := Rule{regexp.MustCompile(`(?P<first>\w+)(?:/(?P<second>\w+))?`), V{Corpus: "${first}", Path: "${second}"}.pb()}
	testRule2 := Rule{regexp.MustCompile(`x/(?P<sig>\w+)/y/(?P<tail>.+)$`), V{Path: "${tail}", Sig: "|${sig}|"}.pb()}
	tests := []struct {
		rule  Rule
		input string
		want  *spb.VName
	}{
		// Optional portions of the pattern should be handled correctly.
		{testRule1, "alpha/bravo", V{Corpus: "alpha", Path: "bravo"}.pb()},
		{testRule1, "alpha", V{Corpus: "alpha"}.pb()},

		// Substitution of signature fields should work.
		{testRule2, "x/kanga/y/roo.txt", V{Path: "roo.txt", Sig: "|kanga|"}.pb()},
	}
	for _, test := range tests {
		got, ok := test.rule.Apply(test.input)
		if !ok {
			t.Errorf("Apply %v failed", test.rule)
		} else if !proto.Equal(got, test.want) {
			t.Errorf("Apply %v: got {%+v}, want {%+v}", test.rule, got, test.want)
		} else {
			t.Logf("Apply %v properly returned {%+v}", test.rule, got)
		}
	}
}

func TestFileRewrites(t *testing.T) {
	tests := []struct {
		path string
		want *spb.VName
	}{
		// static
		{"static/path", V{Corpus: "static", Root: "root"}.pb()},

		// ordered
		{"dup/path", V{Corpus: "first"}.pb()},
		{"dup/path2", V{Corpus: "second"}.pb()},

		// groups
		{"corpus/some/path/here", V{Corpus: "corpus"}.pb()},
		{"grp1/12345/endingGroup", V{Corpus: "grp1/endingGroup", Root: "12345"}.pb()},
		{"bazel-bin/kythe/java/some/path/A.jar!/some/path/A.class", V{Corpus: "kythe", Root: "java"}.pb()},
		{"kythe/java/com/google/devtools/kythe/util/KytheURI.java", V{Corpus: "kythe", Root: "java"}.pb()},
		{"otherCorpus/java/com/google/devtools/kythe/util/KytheURI.java", V{Corpus: "otherCorpus", Root: "java"}.pb()},
	}

	r, err := ParseRules([]byte(testConfig))
	if err != nil {
		t.Fatalf("Broken test rules: %v", err)
	}

	for _, test := range tests {
		got, ok := r.Apply(test.path)
		if !ok {
			t.Errorf("Apply(%q): no match", test.path)
		} else if !proto.Equal(got, test.want) {
			t.Errorf("Apply(%q): got {%+v}, want {%+v}", test.path, got, test.want)
		}
	}
}

type V struct {
	Corpus, Root, Path, Sig, Lang string
}

func (v V) pb() *spb.VName {
	return &spb.VName{
		Corpus:    v.Corpus,
		Root:      v.Root,
		Path:      v.Path,
		Signature: v.Sig,
		Language:  v.Lang,
	}
}

var (
	transformRegexp = cmp.Transformer("Regexp", func(r *regexp.Regexp) string { return r.String() })
	compareVNames   = cmp.Comparer(func(a, b *spb.VName) bool { return proto.Equal(a, b) })
)
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"github.com/phzfi/RIC/server/config"
	"github.com/phzfi/RIC/server/logging"
	"github.com/phzfi/RIC/server/ops"
	"github.com/valyala/fasthttp"
	"path/filepath"
	"strings"
)

func ExtToFormat(ext string) string {
	ext = strings.ToUpper(strings.TrimLeft(ext, "."))
	if ext == "JPG" {
		return "JPEG"
	}
	if ext == "TIF" {
		return "TIFF"
	}
	return ext
}

func ParseURI(uri *fasthttp.URI, source ops.ImageSource, marker ops.Watermarker, conf config.Conf) (operations []ops.Operation, ext string, err, invalid error) {
	filename := string(uri.Path())
	w, h, mode, invalid := getParams(uri.QueryArgs())
	ow, oh, err := source.ImageSize(filename)
	if invalid != nil {
		return
	}
	if err != nil {
		return
	}

	operations = []ops.Operation{source.LoadImageOp(filename)}

	adjustWidth := func() {
		w = roundedIntegerDivision(h*ow, oh)
	}

	adjustHeight := func() {
		h = roundedIntegerDivision(w*oh, ow)
	}

	adjustSize := func() {
		if h == 0 && w != 0 {
			adjustHeight()
		} else if h != 0 && w == 0 {
			adjustWidth()
		} else if w == 0 && h == 0 {
			w, h = ow, oh
		}
	}

	denyUpscale := func() {
		h0 := h
		w0 := w
		if w > ow {
			h = roundedIntegerDivision(ow*h0, w0)
			w = ow
		}
		if h > oh || h > h0 {
			w = roundedIntegerDivision(oh*w0, h0)
			h = oh
		}
	}

	resize := func() {
		denyUpscale()
		adjustSize()
		operations = append(operations, ops.Resize{w, h})
	}

	liquid := func() {
		denyUpscale()
		adjustSize()
		operations = append(operations, ops.LiquidRescale{w, h})
	}

	fit := func() {
		if w > ow {
			w = ow
		}
		if h > oh {
			h = oh
		}
		if w != 0 && h != 0 {
			if ow*h > w*oh {
				adjustHeight()
			} else {
				adjustWidth()
			}
			operations = append(operations, ops.Resize{w, h})
		} else {
			resize()
		}
	}

	watermark := func() {
		heightOK := h > marker.Minheight && h < marker.Maxheight
		widthOK := w > marker.Minwidth && w < marker.Maxwidth
		if marker.AddMark && heightOK && widthOK {
			logging.Debug("Adding watermarkOp")
			operations = append(operations, ops.WatermarkOp(marker.WatermarkImage, marker.Horizontal, marker.Vertical))
		}
	}

	switch mode {
	case resizeMode:
		resize()
	case fitMode:
		fit()
	case liquidMode:
		liquid()
	}
	watermark()

	ext = filepath.Ext(filename)
	if ext == "" {
		ext = ".jpg"
	}
	operations = append(operations, ops.Convert{ExtToFormat(ext)})

	return
}

func roundedIntegerDivision(n, m int) int {
	if (n < 0) == (m < 0) {
		return (n + m/2) / m
	} else { // -5 / 6 should round to -1
		return (n - m/2) / m
	}
}

var stringToMode = map[string]mode{
	"":       resizeMode,
	"resize": resizeMode,
	"fit":    fitMode,
	"crop":   cropMode,
	"liquid": liquidMode,
}

type mode int

const (
	fitMode = mode(1 + iota)
	cropMode
	liquidMode
	resizeMode
)

// returns validated parameters from request and error if invalid
func getParams(a *fasthttp.Args) (w int, h int, mode mode, err error) {
	if strings.Contains(a.String(), "%") {
		err = errors.New("Invalid characters in request!")
		return
	}

	w, err = a.GetUint("width")
	if isParseError(err) {
		return
	}

	h, err = a.GetUint("height")
	if isParseError(err) {
		return
	}

	mode = stringToMode[string(a.Peek("mode"))]
	if mode == 0 {
		err = errors.New("Invalid mode!")
		return
	}

	a.Del("width")
	a.Del("height")
	a.Del("mode")
	if a.Len() != 0 {
		err = errors.New("Invalid parameter " + a.String())
		return
	}
	return
}

func isParseError(err error) bool {
	return err != nil && err != fasthttp.ErrNoArgValue
}
<commit_msg>finish refactoring urlparser for now<commit_after>package main

import (
	"errors"
	"github.com/phzfi/RIC/server/config"
	"github.com/phzfi/RIC/server/logging"
	"github.com/phzfi/RIC/server/ops"
	"github.com/valyala/fasthttp"
	"path/filepath"
	"strings"
)

func ExtToFormat(ext string) string {
	ext = strings.ToUpper(strings.TrimLeft(ext, "."))
	if ext == "JPG" {
		return "JPEG"
	}
	if ext == "TIF" {
		return "TIFF"
	}
	return ext
}

func ParseURI(uri *fasthttp.URI, source ops.ImageSource, marker ops.Watermarker, conf config.Conf) (operations []ops.Operation, ext string, err, invalid error) {
	filename := string(uri.Path())
	w, h, mode, invalid := getParams(uri.QueryArgs())
	ow, oh, err := source.ImageSize(filename)
	if invalid != nil {
		return
	}
	if err != nil {
		return
	}

	operations = []ops.Operation{source.LoadImageOp(filename)}

	adjustWidth := func() {
		w = roundedIntegerDivision(h*ow, oh)
	}

	adjustHeight := func() {
		h = roundedIntegerDivision(w*oh, ow)
	}

	adjustSize := func() {
		if h == -1 && w != -1 {
			adjustHeight()
		} else if h != -1 && w == -1 {
			adjustWidth()
		} else if w == -1 && h == -1 {
			w, h = ow, oh
		}
	}

	denyUpscale := func() {
		h0 := h
		w0 := w
		if w > ow {
			h = roundedIntegerDivision(ow*h0, w0)
			w = ow
		}
		if h > oh || h > h0 {
			w = roundedIntegerDivision(oh*w0, h0)
			h = oh
		}
	}

	resize := func() {
		denyUpscale()
		adjustSize()
		operations = append(operations, ops.Resize{w, h})
	}

	liquid := func() {
		denyUpscale()
		adjustSize()
		operations = append(operations, ops.LiquidRescale{w, h})
	}

	fit := func() {
		if w > ow {
			w = ow
		}
		if h > oh {
			h = oh
		}
		if w != -1 && h != -1 {
			if ow*h > w*oh {
				adjustHeight()
			} else {
				adjustWidth()
			}
			operations = append(operations, ops.Resize{w, h})
		} else {
			resize()
		}
	}

	watermark := func() {
		heightOK := h > marker.Minheight && h < marker.Maxheight
		widthOK := w > marker.Minwidth && w < marker.Maxwidth
		if marker.AddMark && heightOK && widthOK {
			logging.Debug("Adding watermarkOp")
			operations = append(operations, ops.WatermarkOp(marker.WatermarkImage, marker.Horizontal, marker.Vertical))
		}
	}

	switch mode {
	case resizeMode:
		resize()
	case fitMode:
		fit()
	case liquidMode:
		liquid()
	}
	watermark()

	ext = filepath.Ext(filename)
	if ext == "" {
		ext = ".jpg"
	}
	operations = append(operations, ops.Convert{ExtToFormat(ext)})

	return
}

func roundedIntegerDivision(n, m int) int {
	if (n < 0) == (m < 0) {
		return (n + m/2) / m
	} else { // -5 / 6 should round to -1
		return (n - m/2) / m
	}
}

var stringToMode = map[string]mode{
	"":       resizeMode,
	"resize": resizeMode,
	"fit":    fitMode,
	"crop":   cropMode,
	"liquid": liquidMode,
}

type mode int

const (
	fitMode = mode(1 + iota)
	cropMode
	liquidMode
	resizeMode
)

// returns validated parameters from request and error if invalid
func getParams(a *fasthttp.Args) (w int, h int, mode mode, err error) {
	if strings.Contains(a.String(), "%") {
		err = errors.New("Invalid characters in request!")
		return
	}

	w, err = a.GetUint("width")
	if isParseError(err) {
		return
	}

	h, err = a.GetUint("height")
	if isParseError(err) {
		return
	}

	mode = stringToMode[string(a.Peek("mode"))]
	if mode == 0 {
		err = errors.New("Invalid mode!")
		return
	}

	a.Del("width")
	a.Del("height")
	a.Del("mode")
	if a.Len() != 0 {
		err = errors.New("Invalid parameter " + a.String())
		return
	}

	err = nil
	return
}

func isParseError(err error) bool {
	return err != nil && err != fasthttp.ErrNoArgValue
}
<|endoftext|>