<commit_before>// Copyright (c) 2013-2014 by Michael Dvorkin. All Rights Reserved.
// Use of this source code is governed by a MIT-style license that can
// be found in the LICENSE file.

package donna

import (
	`bytes`
)

var tree [1024]Position
var node, rootNode int

type Position struct {
	game       *Game
	enpassant  int         // En-passant square caused by previous move.
	color      int         // Side to make next move.
	reversible bool        // Is this position reversible?
	castles    uint8       // Castle rights mask.
	hash       uint64      // Polyglot hash value.
	board      Bitmask     // Bitmask of all pieces on the board.
	king       [2]int      // King's square for both colors.
	count      [16]int     // Counts of each piece on the board, ex. white pawns: 6, etc.
	pieces     [64]Piece   // Array of 64 squares with pieces on them.
	outposts   [14]Bitmask // Bitmasks of each piece on the board; [0] all white, [1] all black.
}

func NewPosition(game *Game, pieces [64]Piece, color int) *Position {
	tree[node] = Position{game: game, pieces: pieces, color: color}
	p := &tree[node]

	p.castles = castleKingside[White] | castleQueenside[White] |
		castleKingside[Black] | castleQueenside[Black]

	if p.pieces[E1] != King || p.pieces[H1] != Rook {
		p.castles &= ^castleKingside[White]
	}
	if p.pieces[E1] != King || p.pieces[A1] != Rook {
		p.castles &= ^castleQueenside[White]
	}

	if p.pieces[E8] != BlackKing || p.pieces[H8] != BlackRook {
		p.castles &= ^castleKingside[Black]
	}
	if p.pieces[E8] != BlackKing || p.pieces[A8] != BlackRook {
		p.castles &= ^castleQueenside[Black]
	}

	for square, piece := range p.pieces {
		if piece != 0 {
			p.outposts[piece].set(square)
			p.outposts[piece.color()].set(square)
			p.count[piece]++
			if piece.isKing() {
				p.king[piece.color()] = square
			}
		}
	}

	p.reversible = true
	p.hash = p.polyglot()
	p.board = p.outposts[White] | p.outposts[Black]

	return p
}

func (p *Position) movePiece(piece Piece, from, to int) *Position {
	p.pieces[from], p.pieces[to] = 0, piece
	p.outposts[piece] ^= bit[from] | bit[to]
	p.outposts[piece.color()] ^= bit[from] | bit[to]

	return p
}

func (p *Position) promotePawn(piece Piece, from, to int, promo Piece) *Position {
	p.pieces[from], p.pieces[to] = 0, promo
	p.outposts[piece] ^= bit[from]
	p.outposts[promo] ^= bit[to]
	p.outposts[piece.color()] ^= bit[from] | bit[to]
	p.count[piece]--
	p.count[promo]++

	return p
}

func (p *Position) capturePiece(capture Piece, from, to int) *Position {
	p.outposts[capture] ^= bit[to]
	p.outposts[capture.color()] ^= bit[to]
	p.count[capture]--

	return p
}

func (p *Position) captureEnpassant(capture Piece, from, to int) *Position {
	enpassant := to - eight[capture.color()^1]

	p.pieces[enpassant] = 0
	p.outposts[capture] ^= bit[enpassant]
	p.outposts[capture.color()] ^= bit[enpassant]
	p.count[capture]--

	return p
}

func (p *Position) MakeMove(move Move) *Position {
	color := move.color()
	from, to, piece, capture := move.split()
	//
	// Copy over the contents of previous tree node to the current one.
	//
	node++
	tree[node] = *p // => tree[node] = tree[node - 1]
	pp := &tree[node]

	pp.enpassant, pp.reversible = 0, true
	//
	// Castle rights for current node are based on the castle rights from
	// the previous node.
	//
	pp.castles &= castleRights[from] & castleRights[to]

	if capture != 0 {
		pp.reversible = false
		if to != 0 && to == p.enpassant {
			pp.hash ^= polyglotRandom[64 * pawn(color^1).polyglot() + to - eight[color]]
			pp.captureEnpassant(pawn(color^1), from, to)
		} else {
			pp.hash ^= polyglotRandom[64 * p.pieces[to].polyglot() + to]
			pp.capturePiece(capture, from, to)
		}
	}

	if promo := move.promo(); promo == 0 {
		poly := 64 * p.pieces[from].polyglot()
		pp.hash ^= polyglotRandom[poly+from] ^ polyglotRandom[poly+to]
		pp.movePiece(piece, from, to)

		if piece.isKing() {
			pp.king[color] = to
			if move.isCastle() {
				pp.reversible = false
				switch to {
				case G1:
					poly = 64 * Piece(Rook).polyglot()
					pp.hash ^= polyglotRandom[poly+H1] ^ polyglotRandom[poly+F1]
					pp.movePiece(Rook, H1, F1)
				case C1:
					poly = 64 * Piece(Rook).polyglot()
					pp.hash ^= polyglotRandom[poly+A1] ^ polyglotRandom[poly+D1]
					pp.movePiece(Rook, A1, D1)
				case G8:
					poly = 64 * Piece(BlackRook).polyglot()
					pp.hash ^= polyglotRandom[poly+H8] ^ polyglotRandom[poly+F8]
					pp.movePiece(BlackRook, H8, F8)
				case C8:
					poly = 64 * Piece(BlackRook).polyglot()
					pp.hash ^= polyglotRandom[poly+A8] ^ polyglotRandom[poly+D8]
					pp.movePiece(BlackRook, A8, D8)
				}
			}
		} else if piece.isPawn() {
			pp.reversible = false
			if move.isEnpassant() {
				pp.enpassant = from + eight[color] // Save the en-passant square.
				pp.hash ^= hashEnpassant[Col(pp.enpassant)]
			}
		}
	} else {
		pp.reversible = false
		pp.hash ^= polyglotRandom[64 * pawn(color).polyglot() + from]
		pp.hash ^= polyglotRandom[64 * promo.polyglot() + to]
		pp.promotePawn(piece, from, to, promo)
	}

	pp.board = pp.outposts[White] | pp.outposts[Black]
	if pp.isInCheck(color) {
		node--
		return nil
	}

	pp.hash ^= hashCastle[pp.castles]
	if pp.enpassant != 0 {
		pp.hash ^= hashEnpassant[Col(pp.enpassant)]
	}

	if color == White {
		pp.hash ^= polyglotRandomWhite
	}
	pp.color = color ^ 1

	return pp // => &tree[node]
}

// Makes "null" move by copying over previous node position (i.e. preserving all pieces
// intact) and flipping the color.
func (p *Position) MakeNullMove() *Position {
	node++
	tree[node] = *p // => tree[node] = tree[node - 1]
	pp := &tree[node]

	// Flipping side to move obviously invalidates the enpassant square.
	if pp.enpassant != 0 {
		pp.hash ^= hashEnpassant[Col(pp.enpassant)]
		pp.enpassant = 0
	}

	if pp.color == White {
		pp.hash ^= polyglotRandomWhite
	}
	pp.color ^= 1

	return pp // => &tree[node]
}

// Restores previous position effectively taking back the last move made.
func (p *Position) TakeBack(move Move) *Position {
	node--
	return &tree[node]
}

func (p *Position) TakeBackNullMove() *Position {
	p.color ^= 1
	if p.color == White {
		p.hash ^= polyglotRandomWhite
	}
	return p.TakeBack(Move(0))
}

func (p *Position) isInCheck(color int) bool {
	return p.isAttacked(p.king[color], color^1)
}

func (p *Position) isNull() bool {
	return node > 0 && tree[node].board == tree[node-1].board
}

func (p *Position) isRepetition() bool {
	if !p.reversible {
		return false
	}

	for reps, prevNode := 1, node-1; prevNode >= 0; prevNode-- {
		if !tree[prevNode].reversible {
			return false
		}
		if tree[prevNode].color == p.color && tree[prevNode].hash == p.hash {
			reps++
			if reps == 3 {
				return true
			}
		}
	}
	return false
}

func (p *Position) isInsufficient() bool {
	return false
}

func (p *Position) canCastle(color int) (kingside, queenside bool) {
	attacks := p.attacks(color ^ 1)
	kingside = p.castles & castleKingside[color] != 0 &&
		(gapKing[color] & p.board == 0) &&
		(castleKing[color] & attacks == 0)

	queenside = p.castles&castleQueenside[color] != 0 &&
		(gapQueen[color] & p.board == 0) &&
		(castleQueen[color] & attacks == 0)
	return
}

// Reports game status for current position or after the given move. The status
// help to determine whether to continue with search or if the game is over.
func (p *Position) status(move Move, score int) int {
	if move != Move(0) {
		p = p.MakeMove(move)
		defer func() { p = p.TakeBack(move) }()
	}

	switch ply, score := Ply(), Abs(score); score {
	case 0:
		if ply == 1 {
			if p.isRepetition() {
				return Repetition
			} else if p.isInsufficient() {
				return Insufficient
			}
		}
		if !NewGen(p, ply+1).generateMoves().anyValid(p) {
			return Stalemate
		}
	case Checkmate - ply:
		if p.isInCheck(p.color) {
			if p.color == White {
				return BlackWon
			}
			return WhiteWon
		}
		return Stalemate
	default:
		if score > Checkmate-MaxDepth && (score+ply)/2 > 0 {
			if p.color == White {
				return BlackWinning
			}
			return WhiteWinning
		}
	}
	return InProgress
}

// Calculates position stage based on what pieces are on the board (256 for
// the initial position, 0 for bare kings).
func (p *Position) stage() int {
	return 2 * (p.count[Pawn] + p.count[BlackPawn]) +
		6 * (p.count[Knight] + p.count[BlackKnight]) +
		12 * (p.count[Bishop] + p.count[BlackBishop]) +
		16 * (p.count[Rook] + p.count[BlackRook]) +
		44 * (p.count[Queen] + p.count[BlackQueen])
}

// Calculates normalized position score based on position stage and given
// midgame/endgame values.
func (p *Position) score(midgame, endgame int) int {
	stage := p.stage()
	return (midgame * stage + endgame * (256-stage)) / 256
}

// Compute position's polyglot hash.
func (p *Position) polyglot() (key uint64) {
	board := p.board
	for board != 0 {
		square := board.pop() // Inline polyhash() is at least 10% faster.
		key ^= polyglotRandom[64 * p.pieces[square].polyglot() + square]
	}

	key ^= hashCastle[p.castles]

	if p.enpassant != 0 {
		key ^= hashEnpassant[Col(p.enpassant)]
	}
	if p.color == White {
		key ^= polyglotRandomWhite
	}

	return
}

func (p *Position) polyhash(square int) uint64 {
	return polyglotRandom[64 * p.pieces[square].polyglot() + square]
}

func (p *Position) String() string {
	buffer := bytes.NewBufferString("  a b c d e f g h")
	if !p.isInCheck(p.color) {
		buffer.WriteString("\n")
	} else {
		buffer.WriteString("  Check to " + C(p.color) + "\n")
	}
	for row := 7; row >= 0; row-- {
		buffer.WriteByte('1' + byte(row))
		for col := 0; col <= 7; col++ {
			square := Square(row, col)
			buffer.WriteByte(' ')
			if piece := p.pieces[square]; piece != 0 {
				buffer.WriteString(piece.String())
			} else {
				buffer.WriteString("\u22C5")
			}
		}
		buffer.WriteByte('\n')
	}
	return buffer.String()
}
<commit_msg>Less code<commit_after>// Copyright (c) 2013-2014 by Michael Dvorkin. All Rights Reserved.
// Use of this source code is governed by a MIT-style license that can
// be found in the LICENSE file.

package donna

import (
	`bytes`
)

var tree [1024]Position
var node, rootNode int

type Position struct {
	game       *Game
	enpassant  int         // En-passant square caused by previous move.
	color      int         // Side to make next move.
	reversible bool        // Is this position reversible?
	castles    uint8       // Castle rights mask.
	hash       uint64      // Polyglot hash value.
	board      Bitmask     // Bitmask of all pieces on the board.
	king       [2]int      // King's square for both colors.
	count      [16]int     // Counts of each piece on the board, ex. white pawns: 6, etc.
	pieces     [64]Piece   // Array of 64 squares with pieces on them.
	outposts   [14]Bitmask // Bitmasks of each piece on the board; [0] all white, [1] all black.
}

func NewPosition(game *Game, pieces [64]Piece, color int) *Position {
	tree[node] = Position{game: game, pieces: pieces, color: color}
	p := &tree[node]

	p.castles = castleKingside[White] | castleQueenside[White] |
		castleKingside[Black] | castleQueenside[Black]

	if p.pieces[E1] != King || p.pieces[H1] != Rook {
		p.castles &= ^castleKingside[White]
	}
	if p.pieces[E1] != King || p.pieces[A1] != Rook {
		p.castles &= ^castleQueenside[White]
	}

	if p.pieces[E8] != BlackKing || p.pieces[H8] != BlackRook {
		p.castles &= ^castleKingside[Black]
	}
	if p.pieces[E8] != BlackKing || p.pieces[A8] != BlackRook {
		p.castles &= ^castleQueenside[Black]
	}

	for square, piece := range p.pieces {
		if piece != 0 {
			p.outposts[piece].set(square)
			p.outposts[piece.color()].set(square)
			p.count[piece]++
			if piece.isKing() {
				p.king[piece.color()] = square
			}
		}
	}

	p.reversible = true
	p.hash = p.polyglot()
	p.board = p.outposts[White] | p.outposts[Black]

	return p
}

func (p *Position) movePiece(piece Piece, from, to int) *Position {
	p.pieces[from], p.pieces[to] = 0, piece
	p.outposts[piece] ^= bit[from] | bit[to]
	p.outposts[piece.color()] ^= bit[from] | bit[to]

	// Update position's hash.
	poly := 64 * piece.polyglot()
	p.hash ^= polyglotRandom[poly + from] ^ polyglotRandom[poly + to]

	return p
}

func (p *Position) promotePawn(piece Piece, from, to int, promo Piece) *Position {
	p.pieces[from], p.pieces[to] = 0, promo
	p.outposts[piece] ^= bit[from]
	p.outposts[promo] ^= bit[to]
	p.outposts[piece.color()] ^= bit[from] | bit[to]
	p.count[piece]--
	p.count[promo]++

	return p
}

func (p *Position) capturePiece(capture Piece, from, to int) *Position {
	p.outposts[capture] ^= bit[to]
	p.outposts[capture.color()] ^= bit[to]
	p.count[capture]--

	return p
}

func (p *Position) captureEnpassant(capture Piece, from, to int) *Position {
	enpassant := to - eight[capture.color()^1]

	p.pieces[enpassant] = 0
	p.outposts[capture] ^= bit[enpassant]
	p.outposts[capture.color()] ^= bit[enpassant]
	p.count[capture]--

	return p
}

func (p *Position) MakeMove(move Move) *Position {
	color := move.color()
	from, to, piece, capture := move.split()
	//
	// Copy over the contents of previous tree node to the current one.
	//
	node++
	tree[node] = *p // => tree[node] = tree[node - 1]
	pp := &tree[node]

	pp.enpassant, pp.reversible = 0, true
	//
	// Castle rights for current node are based on the castle rights from
	// the previous node.
	//
	pp.castles &= castleRights[from] & castleRights[to]

	if capture != 0 {
		pp.reversible = false
		if to != 0 && to == p.enpassant {
			pp.hash ^= polyglotRandom[64 * pawn(color^1).polyglot() + to - eight[color]]
			pp.captureEnpassant(pawn(color^1), from, to)
		} else {
			pp.hash ^= polyglotRandom[64 * p.pieces[to].polyglot() + to]
			pp.capturePiece(capture, from, to)
		}
	}

	if promo := move.promo(); promo == 0 {
		pp.movePiece(piece, from, to)

		if piece.isKing() {
			pp.king[color] = to
			if move.isCastle() {
				pp.reversible = false
				switch to {
				case G1:
					pp.movePiece(Rook, H1, F1)
				case C1:
					pp.movePiece(Rook, A1, D1)
				case G8:
					pp.movePiece(BlackRook, H8, F8)
				case C8:
					pp.movePiece(BlackRook, A8, D8)
				}
			}
		} else if piece.isPawn() {
			pp.reversible = false
			if move.isEnpassant() {
				pp.enpassant = from + eight[color] // Save the en-passant square.
				pp.hash ^= hashEnpassant[Col(pp.enpassant)]
			}
		}
	} else {
		pp.reversible = false
		pp.hash ^= polyglotRandom[64 * pawn(color).polyglot() + from]
		pp.hash ^= polyglotRandom[64 * promo.polyglot() + to]
		pp.promotePawn(piece, from, to, promo)
	}

	pp.board = pp.outposts[White] | pp.outposts[Black]
	if pp.isInCheck(color) {
		node--
		return nil
	}

	pp.hash ^= hashCastle[pp.castles]
	if pp.enpassant != 0 {
		pp.hash ^= hashEnpassant[Col(pp.enpassant)]
	}

	if color == White {
		pp.hash ^= polyglotRandomWhite
	}
	pp.color = color ^ 1

	return pp // => &tree[node]
}

// Makes "null" move by copying over previous node position (i.e. preserving all pieces
// intact) and flipping the color.
func (p *Position) MakeNullMove() *Position {
	node++
	tree[node] = *p // => tree[node] = tree[node - 1]
	pp := &tree[node]

	// Flipping side to move obviously invalidates the enpassant square.
	if pp.enpassant != 0 {
		pp.hash ^= hashEnpassant[Col(pp.enpassant)]
		pp.enpassant = 0
	}

	if pp.color == White {
		pp.hash ^= polyglotRandomWhite
	}
	pp.color ^= 1

	return pp // => &tree[node]
}

// Restores previous position effectively taking back the last move made.
func (p *Position) TakeBack(move Move) *Position {
	node--
	return &tree[node]
}

func (p *Position) TakeBackNullMove() *Position {
	p.color ^= 1
	if p.color == White {
		p.hash ^= polyglotRandomWhite
	}
	return p.TakeBack(Move(0))
}

func (p *Position) isInCheck(color int) bool {
	return p.isAttacked(p.king[color], color^1)
}

func (p *Position) isNull() bool {
	return node > 0 && tree[node].board == tree[node-1].board
}

func (p *Position) isRepetition() bool {
	if !p.reversible {
		return false
	}

	for reps, prevNode := 1, node-1; prevNode >= 0; prevNode-- {
		if !tree[prevNode].reversible {
			return false
		}
		if tree[prevNode].color == p.color && tree[prevNode].hash == p.hash {
			reps++
			if reps == 3 {
				return true
			}
		}
	}
	return false
}

func (p *Position) isInsufficient() bool {
	return false
}

func (p *Position) canCastle(color int) (kingside, queenside bool) {
	attacks := p.attacks(color ^ 1)
	kingside = p.castles & castleKingside[color] != 0 &&
		(gapKing[color] & p.board == 0) &&
		(castleKing[color] & attacks == 0)

	queenside = p.castles&castleQueenside[color] != 0 &&
		(gapQueen[color] & p.board == 0) &&
		(castleQueen[color] & attacks == 0)
	return
}

// Reports game status for current position or after the given move. The status
// help to determine whether to continue with search or if the game is over.
func (p *Position) status(move Move, score int) int {
	if move != Move(0) {
		p = p.MakeMove(move)
		defer func() { p = p.TakeBack(move) }()
	}

	switch ply, score := Ply(), Abs(score); score {
	case 0:
		if ply == 1 {
			if p.isRepetition() {
				return Repetition
			} else if p.isInsufficient() {
				return Insufficient
			}
		}
		if !NewGen(p, ply+1).generateMoves().anyValid(p) {
			return Stalemate
		}
	case Checkmate - ply:
		if p.isInCheck(p.color) {
			if p.color == White {
				return BlackWon
			}
			return WhiteWon
		}
		return Stalemate
	default:
		if score > Checkmate-MaxDepth && (score+ply)/2 > 0 {
			if p.color == White {
				return BlackWinning
			}
			return WhiteWinning
		}
	}
	return InProgress
}

// Calculates position stage based on what pieces are on the board (256 for
// the initial position, 0 for bare kings).
func (p *Position) stage() int {
	return 2 * (p.count[Pawn] + p.count[BlackPawn]) +
		6 * (p.count[Knight] + p.count[BlackKnight]) +
		12 * (p.count[Bishop] + p.count[BlackBishop]) +
		16 * (p.count[Rook] + p.count[BlackRook]) +
		44 * (p.count[Queen] + p.count[BlackQueen])
}

// Calculates normalized position score based on position stage and given
// midgame/endgame values.
func (p *Position) score(midgame, endgame int) int {
	stage := p.stage()
	return (midgame * stage + endgame * (256-stage)) / 256
}

// Compute position's polyglot hash.
func (p *Position) polyglot() (key uint64) {
	board := p.board
	for board != 0 {
		square := board.pop()
		key ^= polyglotRandom[64 * p.pieces[square].polyglot() + square]
	}

	key ^= hashCastle[p.castles]

	if p.enpassant != 0 {
		key ^= hashEnpassant[Col(p.enpassant)]
	}
	if p.color == White {
		key ^= polyglotRandomWhite
	}

	return
}

func (p *Position) String() string {
	buffer := bytes.NewBufferString("  a b c d e f g h")
	if !p.isInCheck(p.color) {
		buffer.WriteString("\n")
	} else {
		buffer.WriteString("  Check to " + C(p.color) + "\n")
	}
	for row := 7; row >= 0; row-- {
		buffer.WriteByte('1' + byte(row))
		for col := 0; col <= 7; col++ {
			square := Square(row, col)
			buffer.WriteByte(' ')
			if piece := p.pieces[square]; piece != 0 {
				buffer.WriteString(piece.String())
			} else {
				buffer.WriteString("\u22C5")
			}
		}
		buffer.WriteByte('\n')
	}
	return buffer.String()
}
<|endoftext|>


<commit_before>package gorm

import (
	"database/sql"
	"database/sql/driver"
	"fmt"
	"reflect"
	"time"

	"github.com/lib/pq/hstore"
)

type postgres struct {
}

func (s *postgres) BinVar(i int) string {
	return fmt.Sprintf("$%v", i)
}

func (s *postgres) SupportLastInsertId() bool {
	return false
}

func (s *postgres) HasTop() bool {
	return false
}

func (s *postgres) SqlTag(value reflect.Value, size int) string {
	switch value.Kind() {
	case reflect.Bool:
		return "boolean"
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uintptr:
		return "integer"
	case reflect.Int64, reflect.Uint64:
		return "bigint"
	case reflect.Float32, reflect.Float64:
		return "numeric"
	case reflect.String:
		if size > 0 && size < 65532 {
			return fmt.Sprintf("varchar(%d)", size)
		}
		return "text"
	case reflect.Struct:
		if _, ok := value.Interface().(time.Time); ok {
			return "timestamp with time zone"
		}
	case reflect.Map:
		if value.Type() == hstoreType {
			return "hstore"
		}
	default:
		if _, ok := value.Interface().([]byte); ok {
			return "bytea"
		}
	}
	panic(fmt.Sprintf("invalid sql type %s (%s) for postgres", value.Type().Name(), value.Kind().String()))
}

func (s *postgres) PrimaryKeyTag(value reflect.Value, size int) string {
	switch value.Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uintptr:
		return "serial PRIMARY KEY"
	case reflect.Int64, reflect.Uint64:
		return "bigserial PRIMARY KEY"
	case reflect.Array:
		if value.Type().Elem().Kind() == reflect.Uint8 {
			return "bytea PRIMARY KEY" // for Uid/[]byte Type
		}
		fallthrough
	default:
		panic("Invalid primary key type")
	}
}

func (s *postgres) ReturningStr(tableName, key string) string {
	return fmt.Sprintf("RETURNING %v.%v", s.Quote(tableName), key)
}

func (s *postgres) SelectFromDummyTable() string {
	return ""
}

func (s *postgres) Quote(key string) string {
	return fmt.Sprintf("\"%s\"", key)
}

func (s *postgres) HasTable(scope *Scope, tableName string) bool {
	var count int
	newScope := scope.New(nil)
	newScope.Raw(fmt.Sprintf("SELECT count(*) FROM INFORMATION_SCHEMA.tables where table_name = %v and table_type = 'BASE TABLE'", newScope.AddToVars(tableName)))
	newScope.SqlDB().QueryRow(newScope.Sql, newScope.SqlVars...).Scan(&count)
	return count > 0
}

func (s *postgres) HasColumn(scope *Scope, tableName string, columnName string) bool {
	var count int
	newScope := scope.New(nil)
	newScope.Raw(fmt.Sprintf("SELECT count(*) FROM information_schema.columns WHERE table_name = %v AND column_name = %v",
		newScope.AddToVars(tableName),
		newScope.AddToVars(columnName),
	))
	newScope.SqlDB().QueryRow(newScope.Sql, newScope.SqlVars...).Scan(&count)
	return count > 0
}

func (s *postgres) RemoveIndex(scope *Scope, indexName string) {
	scope.Raw(fmt.Sprintf("DROP INDEX %v", s.Quote(indexName))).Exec()
}

var hstoreType = reflect.TypeOf(Hstore{})

type Hstore map[string]*string

func (h Hstore) Value() (driver.Value, error) {
	hstore := hstore.Hstore{Map: map[string]sql.NullString{}}
	if len(h) == 0 {
		return nil, nil
	}

	for key, value := range h {
		hstore.Map[key] = sql.NullString{String: *value, Valid: true}
	}
	return hstore.Value()
}

func (h *Hstore) Scan(value interface{}) error {
	hstore := hstore.Hstore{}

	if err := hstore.Scan(value); err != nil {
		return err
	}

	if len(hstore.Map) == 0 {
		return nil
	}

	*h = Hstore{}
	for k := range hstore.Map {
		if hstore.Map[k].Valid {
			s := hstore.Map[k].String
			(*h)[k] = &s
		} else {
			(*h)[k] = nil
		}
	}

	return nil
}
<commit_msg>Add support of []byte for postgres<commit_after>package gorm

import (
	"database/sql"
	"database/sql/driver"
	"fmt"
	"reflect"
	"time"

	"github.com/lib/pq/hstore"
)

type postgres struct {
}

func (s *postgres) BinVar(i int) string {
	return fmt.Sprintf("$%v", i)
}

func (s *postgres) SupportLastInsertId() bool {
	return false
}

func (s *postgres) HasTop() bool {
	return false
}

func (s *postgres) SqlTag(value reflect.Value, size int) string {
	switch value.Kind() {
	case reflect.Bool:
		return "boolean"
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uintptr:
		return "integer"
	case reflect.Int64, reflect.Uint64:
		return "bigint"
	case reflect.Float32, reflect.Float64:
		return "numeric"
	case reflect.String:
		if size > 0 && size < 65532 {
			return fmt.Sprintf("varchar(%d)", size)
		}
		return "text"
	case reflect.Struct:
		if _, ok := value.Interface().(time.Time); ok {
			return "timestamp with time zone"
		}
	case reflect.Map:
		if value.Type() == hstoreType {
			return "hstore"
		}
	case reflect.Array:
		if value.Type().Elem().Kind() == reflect.Uint8 {
			return "bytea"
		}
	default:
		if _, ok := value.Interface().([]byte); ok {
			return "bytea"
		}
	}
	panic(fmt.Sprintf("invalid sql type %s (%s) for postgres", value.Type().Name(), value.Kind().String()))
}

func (s *postgres) PrimaryKeyTag(value reflect.Value, size int) string {
	switch value.Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uintptr:
		return "serial PRIMARY KEY"
	case reflect.Int64, reflect.Uint64:
		return "bigserial PRIMARY KEY"
	case reflect.Array:
		if value.Type().Elem().Kind() == reflect.Uint8 {
			return "bytea PRIMARY KEY" // for Uid/[]byte Type
		}
		fallthrough
	default:
		panic("Invalid primary key type")
	}
}

func (s *postgres) ReturningStr(tableName, key string) string {
	return fmt.Sprintf("RETURNING %v.%v", s.Quote(tableName), key)
}

func (s *postgres) SelectFromDummyTable() string {
	return ""
}

func (s *postgres) Quote(key string) string {
	return fmt.Sprintf("\"%s\"", key)
}

func (s *postgres) HasTable(scope *Scope, tableName string) bool {
	var count int
	newScope := scope.New(nil)
	newScope.Raw(fmt.Sprintf("SELECT count(*) FROM INFORMATION_SCHEMA.tables where table_name = %v and table_type = 'BASE TABLE'", newScope.AddToVars(tableName)))
	newScope.SqlDB().QueryRow(newScope.Sql, newScope.SqlVars...).Scan(&count)
	return count > 0
}

func (s *postgres) HasColumn(scope *Scope, tableName string, columnName string) bool {
	var count int
	newScope := scope.New(nil)
	newScope.Raw(fmt.Sprintf("SELECT count(*) FROM information_schema.columns WHERE table_name = %v AND column_name = %v",
		newScope.AddToVars(tableName),
		newScope.AddToVars(columnName),
	))
	newScope.SqlDB().QueryRow(newScope.Sql, newScope.SqlVars...).Scan(&count)
	return count > 0
}

func (s *postgres) RemoveIndex(scope *Scope, indexName string) {
	scope.Raw(fmt.Sprintf("DROP INDEX %v", s.Quote(indexName))).Exec()
}

var hstoreType = reflect.TypeOf(Hstore{})

type Hstore map[string]*string

func (h Hstore) Value() (driver.Value, error) {
	hstore := hstore.Hstore{Map: map[string]sql.NullString{}}
	if len(h) == 0 {
		return nil, nil
	}

	for key, value := range h {
		hstore.Map[key] = sql.NullString{String: *value, Valid: true}
	}
	return hstore.Value()
}

func (h *Hstore) Scan(value interface{}) error {
	hstore := hstore.Hstore{}

	if err := hstore.Scan(value); err != nil {
		return err
	}

	if len(hstore.Map) == 0 {
		return nil
	}

	*h = Hstore{}
	for k := range hstore.Map {
		if hstore.Map[k].Valid {
			s := hstore.Map[k].String
			(*h)[k] = &s
		} else {
			(*h)[k] = nil
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// Code generated by protoc-gen-go.
// source: carbonserver.proto
// DO NOT EDIT!

package carbonserverpb

import proto "code.google.com/p/goprotobuf/proto"
import json "encoding/json"
import math "math"

// Reference proto, json, and math imports to suppress error if they are not otherwise used.
var _ = proto.Marshal
var _ = &json.SyntaxError{}
var _ = math.Inf

type FetchResponse struct {
	Name             *string   `protobuf:"bytes,1,req,name=name" json:"name,omitempty"`
	StartTime        *int32    `protobuf:"varint,2,req,name=startTime" json:"startTime,omitempty"`
	StopTime         *int32    `protobuf:"varint,3,req,name=stopTime" json:"stopTime,omitempty"`
	StepTime         *int32    `protobuf:"varint,4,req,name=stepTime" json:"stepTime,omitempty"`
	Values           []float64 `protobuf:"fixed64,5,rep,name=values" json:"values,omitempty"`
	IsAbsent         []bool    `protobuf:"varint,6,rep,name=isAbsent" json:"isAbsent,omitempty"`
	XXX_unrecognized []byte    `json:"-"`
}

func (m *FetchResponse) Reset()         { *m = FetchResponse{} }
func (m *FetchResponse) String() string { return proto.CompactTextString(m) }
func (*FetchResponse) ProtoMessage()    {}

func (m *FetchResponse) GetName() string {
	if m != nil && m.Name != nil {
		return *m.Name
	}
	return ""
}

func (m *FetchResponse) GetStartTime() int32 {
	if m != nil && m.StartTime != nil {
		return *m.StartTime
	}
	return 0
}

func (m *FetchResponse) GetStopTime() int32 {
	if m != nil && m.StopTime != nil {
		return *m.StopTime
	}
	return 0
}

func (m *FetchResponse) GetStepTime() int32 {
	if m != nil && m.StepTime != nil {
		return *m.StepTime
	}
	return 0
}

func (m *FetchResponse) GetValues() []float64 {
	if m != nil {
		return m.Values
	}
	return nil
}

func (m *FetchResponse) GetIsAbsent() []bool {
	if m != nil {
		return m.IsAbsent
	}
	return nil
}

type GlobMatch struct {
	Path             *string `protobuf:"bytes,1,req,name=path" json:"path,omitempty"`
	IsLeaf           *bool   `protobuf:"varint,2,req,name=isLeaf" json:"isLeaf,omitempty"`
	XXX_unrecognized []byte  `json:"-"`
}

func (m *GlobMatch) Reset()         { *m = GlobMatch{} }
func (m *GlobMatch) String() string { return proto.CompactTextString(m) }
func (*GlobMatch) ProtoMessage()    {}

func (m *GlobMatch) GetPath() string {
	if m != nil && m.Path != nil {
		return *m.Path
	}
	return ""
}

func (m *GlobMatch) GetIsLeaf() bool {
	if m != nil && m.IsLeaf != nil {
		return *m.IsLeaf
	}
	return false
}

type GlobResponse struct {
	Name             *string      `protobuf:"bytes,1,req,name=name" json:"name,omitempty"`
	Matches          []*GlobMatch `protobuf:"bytes,2,rep,name=matches" json:"matches,omitempty"`
	XXX_unrecognized []byte       `json:"-"`
}

func (m *GlobResponse) Reset()         { *m = GlobResponse{} }
func (m *GlobResponse) String() string { return proto.CompactTextString(m) }
func (*GlobResponse) ProtoMessage()    {}

func (m *GlobResponse) GetName() string {
	if m != nil && m.Name != nil {
		return *m.Name
	}
	return ""
}

func (m *GlobResponse) GetMatches() []*GlobMatch {
	if m != nil {
		return m.Matches
	}
	return nil
}

type Retention struct {
	SecondsPerPoint  *int32 `protobuf:"varint,1,req,name=secondsPerPoint" json:"secondsPerPoint,omitempty"`
	NumberOfPoints   *int32 `protobuf:"varint,2,req,name=numberOfPoints" json:"numberOfPoints,omitempty"`
	XXX_unrecognized []byte `json:"-"`
}

func (m *Retention) Reset()         { *m = Retention{} }
func (m *Retention) String() string { return proto.CompactTextString(m) }
func (*Retention) ProtoMessage()    {}

func (m *Retention) GetSecondsPerPoint() int32 {
	if m != nil && m.SecondsPerPoint != nil {
		return *m.SecondsPerPoint
	}
	return 0
}

func (m *Retention) GetNumberOfPoints() int32 {
	if m != nil && m.NumberOfPoints != nil {
		return *m.NumberOfPoints
	}
	return 0
}

type InfoResponse struct {
	Name              *string      `protobuf:"bytes,1,req,name=name" json:"name,omitempty"`
	AggregationMethod *string      `protobuf:"bytes,2,req,name=aggregationMethod" json:"aggregationMethod,omitempty"`
	MaxRetention      *int32       `protobuf:"varint,3,req,name=maxRetention" json:"maxRetention,omitempty"`
	XFilesFactor      *float32     `protobuf:"fixed32,4,req,name=xFilesFactor" json:"xFilesFactor,omitempty"`
	Retentions        []*Retention `protobuf:"bytes,5,rep,name=retentions" json:"retentions,omitempty"`
	XXX_unrecognized  []byte       `json:"-"`
}

func (m *InfoResponse) Reset()         { *m = InfoResponse{} }
func (m *InfoResponse) String() string { return proto.CompactTextString(m) }
func (*InfoResponse) ProtoMessage()    {}

func (m *InfoResponse) GetName() string {
	if m != nil && m.Name != nil {
		return *m.Name
	}
	return ""
}

func (m *InfoResponse) GetAggregationMethod() string {
	if m != nil && m.AggregationMethod != nil {
		return *m.AggregationMethod
	}
	return ""
}

func (m *InfoResponse) GetMaxRetention() int32 {
	if m != nil && m.MaxRetention != nil {
		return *m.MaxRetention
	}
	return 0
}

func (m *InfoResponse) GetXFilesFactor() float32 {
	if m != nil && m.XFilesFactor != nil {
		return *m.XFilesFactor
	}
	return 0
}

func (m *InfoResponse) GetRetentions() []*Retention {
	if m != nil {
		return m.Retentions
	}
	return nil
}

func init() {
}
<commit_msg>carbonserverpb: regenerate with gogoprotobuf<commit_after>// Code generated by protoc-gen-gogo.
// source: carbonserver.proto
// DO NOT EDIT!

/*
Package carbonserverpb is a generated protocol buffer package.

It is generated from these files:
	carbonserver.proto

It has these top-level messages:
	FetchResponse
	GlobMatch
	GlobResponse
	Retention
	InfoResponse
*/
package carbonserverpb

import proto "code.google.com/p/gogoprotobuf/proto"
import json "encoding/json"
import math "math"

// Reference proto, json, and math imports to suppress error if they are not otherwise used.
var _ = proto.Marshal
var _ = &json.SyntaxError{}
var _ = math.Inf

type FetchResponse struct {
	Name             *string   `protobuf:"bytes,1,req,name=name" json:"name,omitempty"`
	StartTime        *int32    `protobuf:"varint,2,req,name=startTime" json:"startTime,omitempty"`
	StopTime         *int32    `protobuf:"varint,3,req,name=stopTime" json:"stopTime,omitempty"`
	StepTime         *int32    `protobuf:"varint,4,req,name=stepTime" json:"stepTime,omitempty"`
	Values           []float64 `protobuf:"fixed64,5,rep,name=values" json:"values,omitempty"`
	IsAbsent         []bool    `protobuf:"varint,6,rep,name=isAbsent" json:"isAbsent,omitempty"`
	XXX_unrecognized []byte    `json:"-"`
}

func (m *FetchResponse) Reset()         { *m = FetchResponse{} }
func (m *FetchResponse) String() string { return proto.CompactTextString(m) }
func (*FetchResponse) ProtoMessage()    {}

func (m *FetchResponse) GetName() string {
	if m != nil && m.Name != nil {
		return *m.Name
	}
	return ""
}

func (m *FetchResponse) GetStartTime() int32 {
	if m != nil && m.StartTime != nil {
		return *m.StartTime
	}
	return 0
}

func (m *FetchResponse) GetStopTime() int32 {
	if m != nil && m.StopTime != nil {
		return *m.StopTime
	}
	return 0
}

func (m *FetchResponse) GetStepTime() int32 {
	if m != nil && m.StepTime != nil {
		return *m.StepTime
	}
	return 0
}

func (m *FetchResponse) GetValues() []float64 {
	if m != nil {
		return m.Values
	}
	return nil
}

func (m *FetchResponse) GetIsAbsent() []bool {
	if m != nil {
		return m.IsAbsent
	}
	return nil
}

type GlobMatch struct {
	Path             *string `protobuf:"bytes,1,req,name=path" json:"path,omitempty"`
	IsLeaf           *bool   `protobuf:"varint,2,req,name=isLeaf" json:"isLeaf,omitempty"`
	XXX_unrecognized []byte  `json:"-"`
}

func (m *GlobMatch) Reset()         { *m = GlobMatch{} }
func (m *GlobMatch) String() string { return proto.CompactTextString(m) }
func (*GlobMatch) ProtoMessage()    {}

func (m *GlobMatch) GetPath() string {
	if m != nil && m.Path != nil {
		return *m.Path
	}
	return ""
}

func (m *GlobMatch) GetIsLeaf() bool {
	if m != nil && m.IsLeaf != nil {
		return *m.IsLeaf
	}
	return false
}

type GlobResponse struct {
	Name             *string      `protobuf:"bytes,1,req,name=name" json:"name,omitempty"`
	Matches          []*GlobMatch `protobuf:"bytes,2,rep,name=matches" json:"matches,omitempty"`
	XXX_unrecognized []byte       `json:"-"`
}

func (m *GlobResponse) Reset()         { *m = GlobResponse{} }
func (m *GlobResponse) String() string { return proto.CompactTextString(m) }
func (*GlobResponse) ProtoMessage()    {}

func (m *GlobResponse) GetName() string {
	if m != nil && m.Name != nil {
		return *m.Name
	}
	return ""
}

func (m *GlobResponse) GetMatches() []*GlobMatch {
	if m != nil {
		return m.Matches
	}
	return nil
}

type Retention struct {
	SecondsPerPoint  *int32 `protobuf:"varint,1,req,name=secondsPerPoint" json:"secondsPerPoint,omitempty"`
	NumberOfPoints   *int32 `protobuf:"varint,2,req,name=numberOfPoints" json:"numberOfPoints,omitempty"`
	XXX_unrecognized []byte `json:"-"`
}

func (m *Retention) Reset()         { *m = Retention{} }
func (m *Retention) String() string { return proto.CompactTextString(m) }
func (*Retention) ProtoMessage()    {}

func (m *Retention) GetSecondsPerPoint() int32 {
	if m != nil && m.SecondsPerPoint != nil {
		return *m.SecondsPerPoint
	}
	return 0
}

func (m *Retention) GetNumberOfPoints() int32 {
	if m != nil && m.NumberOfPoints != nil {
		return *m.NumberOfPoints
	}
	return 0
}

type InfoResponse struct {
	Name              *string      `protobuf:"bytes,1,req,name=name" json:"name,omitempty"`
	AggregationMethod *string      `protobuf:"bytes,2,req,name=aggregationMethod" json:"aggregationMethod,omitempty"`
	MaxRetention      *int32       `protobuf:"varint,3,req,name=maxRetention" json:"maxRetention,omitempty"`
	XFilesFactor      *float32     `protobuf:"fixed32,4,req,name=xFilesFactor" json:"xFilesFactor,omitempty"`
	Retentions        []*Retention `protobuf:"bytes,5,rep,name=retentions" json:"retentions,omitempty"`
	XXX_unrecognized  []byte       `json:"-"`
}

func (m *InfoResponse) Reset()         { *m = InfoResponse{} }
func (m *InfoResponse) String() string { return proto.CompactTextString(m) }
func (*InfoResponse) ProtoMessage()    {}

func (m *InfoResponse) GetName() string {
	if m != nil && m.Name != nil {
		return *m.Name
	}
	return ""
}

func (m *InfoResponse) GetAggregationMethod() string {
	if m != nil && m.AggregationMethod != nil {
		return *m.AggregationMethod
	}
	return ""
}

func (m *InfoResponse) GetMaxRetention() int32 {
	if m != nil && m.MaxRetention != nil {
		return *m.MaxRetention
	}
	return 0
}

func (m *InfoResponse) GetXFilesFactor() float32 {
	if m != nil && m.XFilesFactor != nil {
		return *m.XFilesFactor
	}
	return 0
}

func (m *InfoResponse) GetRetentions() []*Retention {
	if m != nil {
		return m.Retentions
	}
	return nil
}

func init() {
}
<|endoftext|>


<commit_before>package disque

import (
	"errors"
	"log"
	"math/rand"
	"sync"
	"time"

	"github.com/garyburd/redigo/redis"
)

// NodeList is a list of nodes with search
type nodeList []Node

func (l nodeList) contains(n Node) bool {
	for _, node := range l {
		if n.Addr == node.Addr {
			return true
		}
	}

	return false
}

// Node describes a node in the cluster, received from Hello
type Node struct {
	Id       string
	Addr     string
	Priority int
}

// IsNull checks if a node is empty or not
func (n Node) IsNull() bool {
	return n.Addr == ""
}

// DialFunc is a redis dialer function that should be supplied to the pool
type DialFunc func(string) (redis.Conn, error)

// Pool is a client pool that keeps track of the available servers in the cluster, and retrieves
// clients to random nodes in the cluster. Pooled connections should be closed to automatically
// be returned to the pool
type Pool struct {
	mutx     sync.Mutex
	nodes    nodeList
	pools    map[string]*redis.Pool
	dialFunc DialFunc
	// for borrow tests
	numBorrowed int
}

func (p *Pool) Size() int {
	defer scopedLock(&p.mutx)()
	n := 0
	for _, node := range p.nodes {
		if node.Priority <= maxPriority {
			n++
		}
	}
	return n
}

// NewPool creates a new client pool, with a given redis dial function, and an initial list of ip:port addresses
// to try connecting to. You should call RefreshNodes after creating the pool to update the list of all
// nodes in the pool, and optionally call RunRefreshLoop to let the queue do this periodically in the background
func NewPool(f DialFunc, addrs ...string) *Pool {

	rand.Seed(time.Now().UnixNano())
	nodes := nodeList{}
	for _, addr := range addrs {
		nodes = append(nodes, Node{Addr: addr, Priority: 1})
	}

	return &Pool{
		mutx:     sync.Mutex{},
		nodes:    nodes,
		dialFunc: f,
		pools:    make(map[string]*redis.Pool),
	}
}

func scopedLock(m *sync.Mutex) func() {

	m.Lock()
	return func() {
		m.Unlock()
	}

}

//TMP - we only handle nodes with priority 1 right now
const maxPriority = 1

// selectNode select a valid node by random. Currently only nodes with priority 1 are selected
func (p *Pool) selectNode(selected nodeList) (Node, error) {

	defer scopedLock(&p.mutx)()

	if len(p.nodes) == 0 {
		return Node{}, errors.New("disque: no nodes in pool")
	}

	nodes := nodeList{}
	for _, node := range p.nodes {
		if node.Priority <= maxPriority && !selected.contains(node) {
			nodes = append(nodes, node)
		}
	}
	if len(nodes) == 0 {
		return Node{}, errors.New("disque: no nodes left to select from")
	}

	return nodes[rand.Intn(len(nodes))], nil

}

const (
	maxIdle              = 3
	refreshFrequency     = time.Minute
	testOnBorrowInterval = time.Second
)

// getPool returns a redis connection pool for a given address
func (p *Pool) getPool(addr string) *redis.Pool {

	defer scopedLock(&p.mutx)()

	pool, found := p.pools[addr]
	if !found {
		pool = redis.NewPool(func() (redis.Conn, error) {
			return p.dialFunc(addr)
		}, maxIdle)

		pool.TestOnBorrow = func(c redis.Conn, t time.Time) error {

			// for testing - count how many borrows we did
			p.numBorrowed++
			if time.Since(t) > testOnBorrowInterval {
				_, err := c.Do("PING")
				return err
			}
			return nil
		}

		p.pools[addr] = pool
	}

	return pool

}

// Get returns a client, or an error if we could not init one
func (p *Pool) Get() (Client, error) {

	selected := nodeList{}
	var node Node
	var err error
	// select node to connect to
	for {
		node, err = p.selectNode(selected)
		if err != nil {
			return nil, err
		}

		conn := p.getPool(node.Addr).Get()
		if conn.Err() != nil {
			selected = append(selected, node)
			continue
		}

		return &RedisClient{
			conn: conn,
			node: node,
		}, nil
	}

}

// UpdateNodes explicitly sets the nodes of the pool
func (p *Pool) UpdateNodes(nodes nodeList) {
	defer scopedLock(&p.mutx)()
	p.nodes = nodes
}

// RefreshNodes uses a HELLO call to refresh the node list in the cluster
func (p *Pool) RefreshNodes() error {
	client, err := p.Get()
	if err != nil {
		return err
	}
	defer client.Close()

	resp, err := client.Hello()
	if err != nil {
		return err
	}

	// update the node list based on the hello response
	p.UpdateNodes(resp.Nodes)
	return nil
}

// RunRefreshLoop starts a goroutine that periodically refreshes the node list using HELLO
func (p *Pool) RunRefreshLoop() {

	go func() {
		for range time.Tick(refreshFrequency) {

			err := p.RefreshNodes()
			if err != nil {
				log.Println("disque pool: could not select client for refreshing")

			}

		}
	}()
}
<commit_msg>Increment numBorrowed atomically to avoid races<commit_after>package disque

import (
	"errors"
	"log"
	"math/rand"
	"sync"
	"sync/atomic"
	"time"

	"github.com/garyburd/redigo/redis"
)

// NodeList is a list of nodes with search
type nodeList []Node

func (l nodeList) contains(n Node) bool {
	for _, node := range l {
		if n.Addr == node.Addr {
			return true
		}
	}

	return false
}

// Node describes a node in the cluster, received from Hello
type Node struct {
	Id       string
	Addr     string
	Priority int
}

// IsNull checks if a node is empty or not
func (n Node) IsNull() bool {
	return n.Addr == ""
}

// DialFunc is a redis dialer function that should be supplied to the pool
type DialFunc func(string) (redis.Conn, error)

// Pool is a client pool that keeps track of the available servers in the cluster, and retrieves
// clients to random nodes in the cluster. Pooled connections should be closed to automatically
// be returned to the pool
type Pool struct {
	mutx     sync.Mutex
	nodes    nodeList
	pools    map[string]*redis.Pool
	dialFunc DialFunc
	// for borrow tests
	numBorrowed int64
}

func (p *Pool) Size() int {
	defer scopedLock(&p.mutx)()
	n := 0
	for _, node := range p.nodes {
		if node.Priority <= maxPriority {
			n++
		}
	}
	return n
}

// NewPool creates a new client pool, with a given redis dial function, and an initial list of ip:port addresses
// to try connecting to. You should call RefreshNodes after creating the pool to update the list of all
// nodes in the pool, and optionally call RunRefreshLoop to let the queue do this periodically in the background
func NewPool(f DialFunc, addrs ...string) *Pool {

	rand.Seed(time.Now().UnixNano())
	nodes := nodeList{}
	for _, addr := range addrs {
		nodes = append(nodes, Node{Addr: addr, Priority: 1})
	}

	return &Pool{
		mutx:     sync.Mutex{},
		nodes:    nodes,
		dialFunc: f,
		pools:    make(map[string]*redis.Pool),
	}
}

func scopedLock(m *sync.Mutex) func() {

	m.Lock()
	return func() {
		m.Unlock()
	}

}

//TMP - we only handle nodes with priority 1 right now
const maxPriority = 1

// selectNode select a valid node by random. Currently only nodes with priority 1 are selected
func (p *Pool) selectNode(selected nodeList) (Node, error) {

	defer scopedLock(&p.mutx)()

	if len(p.nodes) == 0 {
		return Node{}, errors.New("disque: no nodes in pool")
	}

	nodes := nodeList{}
	for _, node := range p.nodes {
		if node.Priority <= maxPriority && !selected.contains(node) {
			nodes = append(nodes, node)
		}
	}
	if len(nodes) == 0 {
		return Node{}, errors.New("disque: no nodes left to select from")
	}

	return nodes[rand.Intn(len(nodes))], nil

}

const (
	maxIdle              = 3
	refreshFrequency     = time.Minute
	testOnBorrowInterval = time.Second
)

// getPool returns a redis connection pool for a given address
func (p *Pool) getPool(addr string) *redis.Pool {

	defer scopedLock(&p.mutx)()

	pool, found := p.pools[addr]
	if !found {
		pool = redis.NewPool(func() (redis.Conn, error) {
			return p.dialFunc(addr)
		}, maxIdle)

		pool.TestOnBorrow = func(c redis.Conn, t time.Time) error {

			// for testing - count how many borrows we did
			atomic.AddInt64(&p.numBorrowed, 1)
			if time.Since(t) > testOnBorrowInterval {
				_, err := c.Do("PING")
				return err
			}
			return nil
		}

		p.pools[addr] = pool
	}

	return pool

}

// Get returns a client, or an error if we could not init one
func (p *Pool) Get() (Client, error) {

	selected := nodeList{}
	var node Node
	var err error
	// select node to connect to
	for {
		node, err = p.selectNode(selected)
		if err != nil {
			return nil, err
		}

		conn := p.getPool(node.Addr).Get()
		if conn.Err() != nil {
			selected = append(selected, node)
			continue
		}

		return &RedisClient{
			conn: conn,
			node: node,
		}, nil
	}

}

// UpdateNodes explicitly sets the nodes of the pool
func (p *Pool) UpdateNodes(nodes nodeList) {
	defer scopedLock(&p.mutx)()
	p.nodes = nodes
}

// RefreshNodes uses a HELLO call to refresh the node list in the cluster
func (p *Pool) RefreshNodes() error {
	client, err := p.Get()
	if err != nil {
		return err
	}
	defer client.Close()

	resp, err := client.Hello()
	if err != nil {
		return err
	}

	// update the node list based on the hello response
	p.UpdateNodes(resp.Nodes)
	return nil
}

// RunRefreshLoop starts a goroutine that periodically refreshes the node list using HELLO
func (p *Pool) RunRefreshLoop() {

	go func() {
		for range time.Tick(refreshFrequency) {

			err := p.RefreshNodes()
			if err != nil {
				log.Println("disque pool: could not select client for refreshing")

			}

		}
	}()
}
<|endoftext|>


<commit_before>package proc

const cacheEnabled = true

type memoryReadWriter interface {
	readMemory(addr uintptr, size int) (data []byte, err error)
	writeMemory(addr uintptr, data []byte) (written int, err error)
}

type memCache struct {
	cacheAddr uintptr
	cache     []byte
	mem       memoryReadWriter
}

func (m *memCache) contains(addr uintptr, size int) bool {
	return addr >= m.cacheAddr && (addr+uintptr(size)) <= (m.cacheAddr+uintptr(len(m.cache)))
}

func (m *memCache) readMemory(addr uintptr, size int) (data []byte, err error) {
	if m.contains(addr, size) {
		d := make([]byte, size)
		copy(d, m.cache[addr-m.cacheAddr:])
		return d, nil
	}

	return m.mem.readMemory(addr, size)
}

func (m *memCache) writeMemory(addr uintptr, data []byte) (written int, err error) {
	return m.mem.writeMemory(addr, data)
}

func cacheMemory(mem memoryReadWriter, addr uintptr, size int) memoryReadWriter {
	if !cacheEnabled {
		return mem
	}
	if cacheMem, isCache := mem.(*memCache); isCache {
		if cacheMem.contains(addr, size) {
			return mem
		} else {
			cache, err := cacheMem.mem.readMemory(addr, size)
			if err != nil {
				return mem
			}
			return &memCache{addr, cache, mem}
		}
	}
	cache, err := mem.readMemory(addr, size)
	if err != nil {
		return mem
	}
	return &memCache{addr, cache, mem}
}
<commit_msg>proc: Possible panic while reading uninitialised memory<commit_after>package proc

const cacheEnabled = true

type memoryReadWriter interface {
	readMemory(addr uintptr, size int) (data []byte, err error)
	writeMemory(addr uintptr, data []byte) (written int, err error)
}

type memCache struct {
	cacheAddr uintptr
	cache     []byte
	mem       memoryReadWriter
}

func (m *memCache) contains(addr uintptr, size int) bool {
	return addr >= m.cacheAddr && (addr+uintptr(size)) <= (m.cacheAddr+uintptr(len(m.cache)))
}

func (m *memCache) readMemory(addr uintptr, size int) (data []byte, err error) {
	if m.contains(addr, size) {
		d := make([]byte, size)
		copy(d, m.cache[addr-m.cacheAddr:])
		return d, nil
	}

	return m.mem.readMemory(addr, size)
}

func (m *memCache) writeMemory(addr uintptr, data []byte) (written int, err error) {
	return m.mem.writeMemory(addr, data)
}

func cacheMemory(mem memoryReadWriter, addr uintptr, size int) memoryReadWriter {
	if !cacheEnabled {
		return mem
	}
	if size <= 0 {
		return mem
	}
	if cacheMem, isCache := mem.(*memCache); isCache {
		if cacheMem.contains(addr, size) {
			return mem
		} else {
			cache, err := cacheMem.mem.readMemory(addr, size)
			if err != nil {
				return mem
			}
			return &memCache{addr, cache, mem}
		}
	}
	cache, err := mem.readMemory(addr, size)
	if err != nil {
		return mem
	}
	return &memCache{addr, cache, mem}
}
<|endoftext|>


<commit_before>package command

import (
	"flag"
	"fmt"
	"math/rand"
	"strings"
	"time"

	"github.com/navy/memberid/registry"
)

type ListCommand struct {
	Group   string
	To      string
	Shuffle bool
}

func (c *ListCommand) ConfigureFlags(fs *flag.FlagSet) {
	fs.StringVar(&c.Group, "g", "", "group name")
	fs.StringVar(&c.To, "to", "", "id-type to")
	fs.BoolVar(&c.Shuffle, "shuffle", false, "Shuffle ids")
}

func (c *ListCommand) Help() string {
	return "[-g <GROUP>] [-to <TO>]"
}

func (c *ListCommand) Run(fs *flag.FlagSet, r registry.Registry) {
	var ids []string

	if c.Group != "" {
		ids = r.Ids(strings.Split(c.Group, ",")...)
	} else {
		ids = r.Ids()
	}

	if c.Shuffle {
		ids = shuffle(ids)
	}

	if c.To != "" {
		for i, id := range ids {
			ids[i] = r.ResolveId(id, "", c.To)
		}
	}

	fmt.Printf("%v\n", ids)
}

func shuffle(ids []string) []string {
	rand.Seed(time.Now().UTC().UnixNano())
	for i, _ := range ids {
		j := rand.Intn(i + 1)
		ids[i], ids[j] = ids[j], ids[i]
	}

	return ids
}
<commit_msg>Fix list format<commit_after>package command

import (
	"flag"
	"fmt"
	"math/rand"
	"strings"
	"time"

	"github.com/navy/memberid/registry"
)

type ListCommand struct {
	Group   string
	To      string
	Shuffle bool
}

func (c *ListCommand) ConfigureFlags(fs *flag.FlagSet) {
	fs.StringVar(&c.Group, "g", "", "group name")
	fs.StringVar(&c.To, "to", "", "id-type to")
	fs.BoolVar(&c.Shuffle, "shuffle", false, "Shuffle ids")
}

func (c *ListCommand) Help() string {
	return "[-g <GROUP>] [-to <TO>]"
}

func (c *ListCommand) Run(fs *flag.FlagSet, r registry.Registry) {
	var ids []string

	if c.Group != "" {
		ids = r.Ids(strings.Split(c.Group, ",")...)
	} else {
		ids = r.Ids()
	}

	if c.Shuffle {
		ids = shuffle(ids)
	}

	if c.To != "" {
		for i, id := range ids {
			ids[i] = r.ResolveId(id, "", c.To)
		}
	}

	fmt.Printf("%s\n", strings.Join(ids, " "))
}

func shuffle(ids []string) []string {
	rand.Seed(time.Now().UTC().UnixNano())
	for i, _ := range ids {
		j := rand.Intn(i + 1)
		ids[i], ids[j] = ids[j], ids[i]
	}

	return ids
}
<|endoftext|>


<commit_before>package command

import (
	"fmt"
	"strings"
)

type StopCommand struct {
	Meta
}

func (c *StopCommand) Help() string {
	helpText := `
Usage: nomad stop [options] <job>

  Stop an existing job. This command is used to signal allocations
  to shut down for the given job ID. Upon successful deregistraion,
  an interactive monitor session will start to display log lines as
  the job unwinds its allocations and completes shutting down. It
  is safe to exit the monitor early using ctrl+c.

General Options:

  ` + generalOptionsUsage() + `

Stop Options:

  -detach
    Return immediately instead of entering monitor mode. After the
    deregister command is submitted, a new evaluation ID is printed
    to the screen, which can be used to call up a monitor later if
    needed using the eval-monitor command.
`
	return strings.TrimSpace(helpText)
}

func (c *StopCommand) Synopsis() string {
	return "Stop a running job"
}

func (c *StopCommand) Run(args []string) int {
	var detach bool

	flags := c.Meta.FlagSet("stop", FlagSetClient)
	flags.Usage = func() { c.Ui.Output(c.Help()) }
	flags.BoolVar(&detach, "detach", false, "")

	if err := flags.Parse(args); err != nil {
		return 1
	}

	// Check that we got exactly one job
	args = flags.Args()
	if len(args) != 1 {
		c.Ui.Error(c.Help())
		return 1
	}
	jobID := args[0]

	// Get the HTTP client
	client, err := c.Meta.Client()
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err))
		return 1
	}

	// Check if the job exists
	if _, _, err := client.Jobs().Info(jobID, nil); err != nil {
		c.Ui.Error(fmt.Sprintf("Error deregistering job: %s", err))
		return 1
	}

	// Invoke the stop
	evalID, _, err := client.Jobs().Deregister(jobID, nil)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error deregistering job: %s", err))
		return 1
	}

	if detach {
		c.Ui.Output(evalID)
		return 0
	}

	// Start monitoring the stop eval
	mon := newMonitor(c.Ui, client)
	return mon.monitor(evalID)
}
<commit_msg>Allow short job identifiers for stop command<commit_after>package command

import (
	"fmt"
	"strings"
)

type StopCommand struct {
	Meta
}

func (c *StopCommand) Help() string {
	helpText := `
Usage: nomad stop [options] <job>

  Stop an existing job. This command is used to signal allocations
  to shut down for the given job ID. Upon successful deregistraion,
  an interactive monitor session will start to display log lines as
  the job unwinds its allocations and completes shutting down. It
  is safe to exit the monitor early using ctrl+c.

General Options:

  ` + generalOptionsUsage() + `

Stop Options:

  -detach
    Return immediately instead of entering monitor mode. After the
    deregister command is submitted, a new evaluation ID is printed
    to the screen, which can be used to call up a monitor later if
    needed using the eval-monitor command.
`
	return strings.TrimSpace(helpText)
}

func (c *StopCommand) Synopsis() string {
	return "Stop a running job"
}

func (c *StopCommand) Run(args []string) int {
	var detach bool

	flags := c.Meta.FlagSet("stop", FlagSetClient)
	flags.Usage = func() { c.Ui.Output(c.Help()) }
	flags.BoolVar(&detach, "detach", false, "")

	if err := flags.Parse(args); err != nil {
		return 1
	}

	// Check that we got exactly one job
	args = flags.Args()
	if len(args) != 1 {
		c.Ui.Error(c.Help())
		return 1
	}
	jobID := args[0]

	// Get the HTTP client
	client, err := c.Meta.Client()
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err))
		return 1
	}

	// Check if the job exists
	job, _, err := client.Jobs().Info(jobID, nil)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error deregistering job: %s", err))
		return 1
	}

	// Invoke the stop
	evalID, _, err := client.Jobs().Deregister(job.ID, nil)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error deregistering job: %s", err))
		return 1
	}

	if detach {
		c.Ui.Output(evalID)
		return 0
	}

	// Start monitoring the stop eval
	mon := newMonitor(c.Ui, client)
	return mon.monitor(evalID)
}
<|endoftext|>


<commit_before>package cli

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"strings"
	"testing"
)

func TestCommandFlagParsing(t *testing.T) {
	cases := []struct {
		testArgs               []string
		skipFlagParsing        bool
		useShortOptionHandling bool
		expectedErr            error
	}{
		// Test normal "not ignoring flags" flow
		{testArgs: []string{"test-cmd", "-break", "blah", "blah"}, skipFlagParsing: false, useShortOptionHandling: false, expectedErr: errors.New("flag provided but not defined: -break")},
		{[]string{"test-cmd", "blah", "blah", "-break"}, false, false, nil, false},
		// Test no arg reorder
		{[]string{"test-cmd", "blah", "blah", "-break"}, false, true, nil, false},
		{[]string{"test-cmd", "blah", "blah", "-break", "ls", "-l"}, false, true, nil, true},
		{[]string{"test-cmd", "blah", "blah"}, true, false, nil, false},                                                            // Test SkipFlagParsing without any args that look like flags
		{[]string{"test-cmd", "blah", "-break"}, true, false, nil, false},                                                          // Test SkipFlagParsing with random flag arg
		{[]string{"test-cmd", "blah", "-help"}, true, false, nil, false},                                                           // Test SkipFlagParsing with "special" help flag arg
		{[]string{"test-cmd", "blah"}, false, false, nil, true},                                                                    // Test UseShortOptionHandling
		{testArgs: []string{"test-cmd", "blah", "blah"}, skipFlagParsing: true, useShortOptionHandling: false, expectedErr: nil},   // Test SkipFlagParsing without any args that look like flags
		{testArgs: []string{"test-cmd", "blah", "-break"}, skipFlagParsing: true, useShortOptionHandling: false, expectedErr: nil}, // Test SkipFlagParsing with random flag arg
		{testArgs: []string{"test-cmd", "blah", "-help"}, skipFlagParsing: true, useShortOptionHandling: false, expectedErr: nil},  // Test SkipFlagParsing with "special" help flag arg
		{testArgs: []string{"test-cmd", "blah", "-h"}, skipFlagParsing: false, useShortOptionHandling: true, expectedErr: nil},     // Test UseShortOptionHandling
	}

	for _, c := range cases {
		app := &App{Writer: ioutil.Discard}
		set := flag.NewFlagSet("test", 0)
		_ = set.Parse(c.testArgs)

		context := NewContext(app, set, nil)

		command := Command{
			Name:            "test-cmd",
			Aliases:         []string{"tc"},
			Usage:           "this is for testing",
			Description:     "testing",
			Action:          func(_ *Context) error { return nil },
			SkipFlagParsing: c.skipFlagParsing,
		}

		err := command.Run(context)

		expect(t, err, c.expectedErr)
		expect(t, context.Args().Slice(), c.testArgs)
	}
}

func TestParseAndRunShortOpts(t *testing.T) {
	cases := []struct {
		testArgs     args
		expectedErr  error
		expectedArgs Args
	}{
		{testArgs: args{"foo", "test", "-a"}, expectedErr: nil, expectedArgs: &args{}},
		{testArgs: args{"foo", "test", "-c", "arg1", "arg2"}, expectedErr: nil, expectedArgs: &args{"arg1", "arg2"}},
		{testArgs: args{"foo", "test", "-f"}, expectedErr: nil, expectedArgs: &args{}},
		{testArgs: args{"foo", "test", "-ac", "--fgh"}, expectedErr: nil, expectedArgs: &args{}},
		{testArgs: args{"foo", "test", "-af"}, expectedErr: nil, expectedArgs: &args{}},
		{testArgs: args{"foo", "test", "-cf"}, expectedErr: nil, expectedArgs: &args{}},
		{testArgs: args{"foo", "test", "-acf"}, expectedErr: nil, expectedArgs: &args{}},
		{testArgs: args{"foo", "test", "--acf"}, expectedErr: errors.New("flag provided but not defined: -acf"), expectedArgs: nil},
		{testArgs: args{"foo", "test", "-invalid"}, expectedErr: errors.New("flag provided but not defined: -invalid"), expectedArgs: nil},
		{testArgs: args{"foo", "test", "-acf", "-invalid"}, expectedErr: errors.New("flag provided but not defined: -invalid"), expectedArgs: nil},
		{testArgs: args{"foo", "test", "--invalid"}, expectedErr: errors.New("flag provided but not defined: -invalid"), expectedArgs: nil},
		{testArgs: args{"foo", "test", "-acf", "--invalid"}, expectedErr: errors.New("flag provided but not defined: -invalid"), expectedArgs: nil},
		{testArgs: args{"foo", "test", "-acf", "arg1", "-invalid"}, expectedErr: nil, expectedArgs: &args{"arg1", "-invalid"}},
		{testArgs: args{"foo", "test", "-acf", "arg1", "--invalid"}, expectedErr: nil, expectedArgs: &args{"arg1", "--invalid"}},
		{testArgs: args{"foo", "test", "-acfi", "not-arg", "arg1", "-invalid"}, expectedErr: nil, expectedArgs: &args{"arg1", "-invalid"}},
		{testArgs: args{"foo", "test", "-i", "ivalue"}, expectedErr: nil, expectedArgs: &args{}},
		{testArgs: args{"foo", "test", "-i", "ivalue", "arg1"}, expectedErr: nil, expectedArgs: &args{"arg1"}},
		{testArgs: args{"foo", "test", "-i"}, expectedErr: errors.New("flag needs an argument: -i"), expectedArgs: nil},
	}

	for _, c := range cases {
		var args Args
		cmd := &Command{
			Name:        "test",
			Usage:       "this is for testing",
			Description: "testing",
			Action: func(c *Context) error {
				args = c.Args()
				return nil
			},
			UseShortOptionHandling: true,
			Flags: []Flag{
				&BoolFlag{Name: "abc", Aliases: []string{"a"}},
				&BoolFlag{Name: "cde", Aliases: []string{"c"}},
				&BoolFlag{Name: "fgh", Aliases: []string{"f"}},
				&StringFlag{Name: "ijk", Aliases: []string{"i"}},
			},
		}

		app := NewApp()
		app.Commands = []*Command{cmd}

		err := app.Run(c.testArgs)

		expect(t, err, c.expectedErr)
		expect(t, args, c.expectedArgs)
	}
}

func TestCommand_Run_DoesNotOverwriteErrorFromBefore(t *testing.T) {
	app := &App{
		Commands: []*Command{
			{
				Name: "bar",
				Before: func(c *Context) error {
					return fmt.Errorf("before error")
				},
				After: func(c *Context) error {
					return fmt.Errorf("after error")
				},
			},
		},
	}

	err := app.Run([]string{"foo", "bar"})
	if err == nil {
		t.Fatalf("expected to receive error from Run, got none")
	}

	if !strings.Contains(err.Error(), "before error") {
		t.Errorf("expected text of error from Before method, but got none in \"%v\"", err)
	}
	if !strings.Contains(err.Error(), "after error") {
		t.Errorf("expected text of error from After method, but got none in \"%v\"", err)
	}
}

func TestCommand_Run_BeforeSavesMetadata(t *testing.T) {
	var receivedMsgFromAction string
	var receivedMsgFromAfter string

	app := &App{
		Commands: []*Command{
			{
				Name: "bar",
				Before: func(c *Context) error {
					c.App.Metadata["msg"] = "hello world"
					return nil
				},
				Action: func(c *Context) error {
					msg, ok := c.App.Metadata["msg"]
					if !ok {
						return errors.New("msg not found")
					}
					receivedMsgFromAction = msg.(string)
					return nil
				},
				After: func(c *Context) error {
					msg, ok := c.App.Metadata["msg"]
					if !ok {
						return errors.New("msg not found")
					}
					receivedMsgFromAfter = msg.(string)
					return nil
				},
			},
		},
	}

	err := app.Run([]string{"foo", "bar"})
	if err != nil {
		t.Fatalf("expected no error from Run, got %s", err)
	}

	expectedMsg := "hello world"

	if receivedMsgFromAction != expectedMsg {
		t.Fatalf("expected msg from Action to match. Given: %q\nExpected: %q",
			receivedMsgFromAction, expectedMsg)
	}
	if receivedMsgFromAfter != expectedMsg {
		t.Fatalf("expected msg from After to match. Given: %q\nExpected: %q",
			receivedMsgFromAction, expectedMsg)
	}
}

func TestCommand_OnUsageError_hasCommandContext(t *testing.T) {
	app := &App{
		Commands: []*Command{
			{
				Name: "bar",
				Flags: []Flag{
					&IntFlag{Name: "flag"},
				},
				OnUsageError: func(c *Context, err error, _ bool) error {
					return fmt.Errorf("intercepted in %s: %s", c.Command.Name, err.Error())
				},
			},
		},
	}

	err := app.Run([]string{"foo", "bar", "--flag=wrong"})
	if err == nil {
		t.Fatalf("expected to receive error from Run, got none")
	}

	if !strings.HasPrefix(err.Error(), "intercepted in bar") {
		t.Errorf("Expect an intercepted error, but got \"%v\"", err)
	}
}

func TestCommand_OnUsageError_WithWrongFlagValue(t *testing.T) {
	app := &App{
		Commands: []*Command{
			{
				Name: "bar",
				Flags: []Flag{
					&IntFlag{Name: "flag"},
				},
				OnUsageError: func(c *Context, err error, _ bool) error {
					if !strings.HasPrefix(err.Error(), "invalid value \"wrong\"") {
						t.Errorf("Expect an invalid value error, but got \"%v\"", err)
					}
					return errors.New("intercepted: " + err.Error())
				},
			},
		},
	}

	err := app.Run([]string{"foo", "bar", "--flag=wrong"})
	if err == nil {
		t.Fatalf("expected to receive error from Run, got none")
	}

	if !strings.HasPrefix(err.Error(), "intercepted: invalid value") {
		t.Errorf("Expect an intercepted error, but got \"%v\"", err)
	}
}

func TestCommand_OnUsageError_WithSubcommand(t *testing.T) {
	app := &App{
		Commands: []*Command{
			{
				Name: "bar",
				Subcommands: []*Command{
					{
						Name: "baz",
					},
				},
				Flags: []Flag{
					&IntFlag{Name: "flag"},
				},
				OnUsageError: func(c *Context, err error, _ bool) error {
					if !strings.HasPrefix(err.Error(), "invalid value \"wrong\"") {
						t.Errorf("Expect an invalid value error, but got \"%v\"", err)
					}
					return errors.New("intercepted: " + err.Error())
				},
			},
		},
	}

	err := app.Run([]string{"foo", "bar", "--flag=wrong"})
	if err == nil {
		t.Fatalf("expected to receive error from Run, got none")
	}

	if !strings.HasPrefix(err.Error(), "intercepted: invalid value") {
		t.Errorf("Expect an intercepted error, but got \"%v\"", err)
	}
}

func TestCommand_Run_SubcommandsCanUseErrWriter(t *testing.T) {
	app := &App{
		ErrWriter: ioutil.Discard,
		Commands: []*Command{
			{
				Name:  "bar",
				Usage: "this is for testing",
				Subcommands: []*Command{
					{
						Name:  "baz",
						Usage: "this is for testing",
						Action: func(c *Context) error {
							if c.App.ErrWriter != ioutil.Discard {
								return fmt.Errorf("ErrWriter not passed")
							}

							return nil
						},
					},
				},
			},
		},
	}

	err := app.Run([]string{"foo", "bar", "baz"})
	if err != nil {
		t.Fatal(err)
	}
}

func TestCommandSkipFlagParsing(t *testing.T) {
	cases := []struct {
		testArgs     args
		expectedArgs *args
		expectedErr  error
	}{
		{testArgs: args{"some-exec", "some-command", "some-arg", "--flag", "foo"}, expectedArgs: &args{"some-arg", "--flag", "foo"}, expectedErr: nil},
		{testArgs: args{"some-exec", "some-command", "some-arg", "--flag=foo"}, expectedArgs: &args{"some-arg", "--flag=foo"}, expectedErr: nil},
	}

	for _, c := range cases {
		var args Args
		app := &App{
			Commands: []*Command{
				{
					SkipFlagParsing: true,
					Name:            "some-command",
					Flags: []Flag{
						&StringFlag{Name: "flag"},
					},
					Action: func(c *Context) error {
						fmt.Printf("%+v\n", c.String("flag"))
						args = c.Args()
						return nil
					},
				},
			},
		}

		err := app.Run(c.testArgs)
		expect(t, err, c.expectedErr)
		expect(t, args, c.expectedArgs)
	}
}
<commit_msg>remove old tests<commit_after>package cli

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"strings"
	"testing"
)

func TestCommandFlagParsing(t *testing.T) {
	cases := []struct {
		testArgs               []string
		skipFlagParsing        bool
		useShortOptionHandling bool
		expectedErr            error
	}{
		// Test normal "not ignoring flags" flow
		{testArgs: []string{"test-cmd", "-break", "blah", "blah"}, skipFlagParsing: false, useShortOptionHandling: false, expectedErr: errors.New("flag provided but not defined: -break")},
		{testArgs: []string{"test-cmd", "blah", "blah"}, skipFlagParsing: true, useShortOptionHandling: false, expectedErr: nil},   // Test SkipFlagParsing without any args that look like flags
		{testArgs: []string{"test-cmd", "blah", "-break"}, skipFlagParsing: true, useShortOptionHandling: false, expectedErr: nil}, // Test SkipFlagParsing with random flag arg
		{testArgs: []string{"test-cmd", "blah", "-help"}, skipFlagParsing: true, useShortOptionHandling: false, expectedErr: nil},  // Test SkipFlagParsing with "special" help flag arg
		{testArgs: []string{"test-cmd", "blah", "-h"}, skipFlagParsing: false, useShortOptionHandling: true, expectedErr: nil},     // Test UseShortOptionHandling
	}

	for _, c := range cases {
		app := &App{Writer: ioutil.Discard}
		set := flag.NewFlagSet("test", 0)
		_ = set.Parse(c.testArgs)

		context := NewContext(app, set, nil)

		command := Command{
			Name:            "test-cmd",
			Aliases:         []string{"tc"},
			Usage:           "this is for testing",
			Description:     "testing",
			Action:          func(_ *Context) error { return nil },
			SkipFlagParsing: c.skipFlagParsing,
		}

		err := command.Run(context)

		expect(t, err, c.expectedErr)
		expect(t, context.Args().Slice(), c.testArgs)
	}
}

func TestParseAndRunShortOpts(t *testing.T) {
	cases := []struct {
		testArgs     args
		expectedErr  error
		expectedArgs Args
	}{
		{testArgs: args{"foo", "test", "-a"}, expectedErr: nil, expectedArgs: &args{}},
		{testArgs: args{"foo", "test", "-c", "arg1", "arg2"}, expectedErr: nil, expectedArgs: &args{"arg1", "arg2"}},
		{testArgs: args{"foo", "test", "-f"}, expectedErr: nil, expectedArgs: &args{}},
		{testArgs: args{"foo", "test", "-ac", "--fgh"}, expectedErr: nil, expectedArgs: &args{}},
		{testArgs: args{"foo", "test", "-af"}, expectedErr: nil, expectedArgs: &args{}},
		{testArgs: args{"foo", "test", "-cf"}, expectedErr: nil, expectedArgs: &args{}},
		{testArgs: args{"foo", "test", "-acf"}, expectedErr: nil, expectedArgs: &args{}},
		{testArgs: args{"foo", "test", "--acf"}, expectedErr: errors.New("flag provided but not defined: -acf"), expectedArgs: nil},
		{testArgs: args{"foo", "test", "-invalid"}, expectedErr: errors.New("flag provided but not defined: -invalid"), expectedArgs: nil},
		{testArgs: args{"foo", "test", "-acf", "-invalid"}, expectedErr: errors.New("flag provided but not defined: -invalid"), expectedArgs: nil},
		{testArgs: args{"foo", "test", "--invalid"}, expectedErr: errors.New("flag provided but not defined: -invalid"), expectedArgs: nil},
		{testArgs: args{"foo", "test", "-acf", "--invalid"}, expectedErr: errors.New("flag provided but not defined: -invalid"), expectedArgs: nil},
		{testArgs: args{"foo", "test", "-acf", "arg1", "-invalid"}, expectedErr: nil, expectedArgs: &args{"arg1", "-invalid"}},
		{testArgs: args{"foo", "test", "-acf", "arg1", "--invalid"}, expectedErr: nil, expectedArgs: &args{"arg1", "--invalid"}},
		{testArgs: args{"foo", "test", "-acfi", "not-arg", "arg1", "-invalid"}, expectedErr: nil, expectedArgs: &args{"arg1", "-invalid"}},
		{testArgs: args{"foo", "test", "-i", "ivalue"}, expectedErr: nil, expectedArgs: &args{}},
		{testArgs: args{"foo", "test", "-i", "ivalue", "arg1"}, expectedErr: nil, expectedArgs: &args{"arg1"}},
		{testArgs: args{"foo", "test", "-i"}, expectedErr: errors.New("flag needs an argument: -i"), expectedArgs: nil},
	}

	for _, c := range cases {
		var args Args
		cmd := &Command{
			Name:        "test",
			Usage:       "this is for testing",
			Description: "testing",
			Action: func(c *Context) error {
				args = c.Args()
				return nil
			},
			UseShortOptionHandling: true,
			Flags: []Flag{
				&BoolFlag{Name: "abc", Aliases: []string{"a"}},
				&BoolFlag{Name: "cde", Aliases: []string{"c"}},
				&BoolFlag{Name: "fgh", Aliases: []string{"f"}},
				&StringFlag{Name: "ijk", Aliases: []string{"i"}},
			},
		}

		app := NewApp()
		app.Commands = []*Command{cmd}

		err := app.Run(c.testArgs)

		expect(t, err, c.expectedErr)
		expect(t, args, c.expectedArgs)
	}
}

func TestCommand_Run_DoesNotOverwriteErrorFromBefore(t *testing.T) {
	app := &App{
		Commands: []*Command{
			{
				Name: "bar",
				Before: func(c *Context) error {
					return fmt.Errorf("before error")
				},
				After: func(c *Context) error {
					return fmt.Errorf("after error")
				},
			},
		},
	}

	err := app.Run([]string{"foo", "bar"})
	if err == nil {
		t.Fatalf("expected to receive error from Run, got none")
	}

	if !strings.Contains(err.Error(), "before error") {
		t.Errorf("expected text of error from Before method, but got none in \"%v\"", err)
	}
	if !strings.Contains(err.Error(), "after error") {
		t.Errorf("expected text of error from After method, but got none in \"%v\"", err)
	}
}

func TestCommand_Run_BeforeSavesMetadata(t *testing.T) {
	var receivedMsgFromAction string
	var receivedMsgFromAfter string

	app := &App{
		Commands: []*Command{
			{
				Name: "bar",
				Before: func(c *Context) error {
					c.App.Metadata["msg"] = "hello world"
					return nil
				},
				Action: func(c *Context) error {
					msg, ok := c.App.Metadata["msg"]
					if !ok {
						return errors.New("msg not found")
					}
					receivedMsgFromAction = msg.(string)
					return nil
				},
				After: func(c *Context) error {
					msg, ok := c.App.Metadata["msg"]
					if !ok {
						return errors.New("msg not found")
					}
					receivedMsgFromAfter = msg.(string)
					return nil
				},
			},
		},
	}

	err := app.Run([]string{"foo", "bar"})
	if err != nil {
		t.Fatalf("expected no error from Run, got %s", err)
	}

	expectedMsg := "hello world"

	if receivedMsgFromAction != expectedMsg {
		t.Fatalf("expected msg from Action to match. Given: %q\nExpected: %q",
			receivedMsgFromAction, expectedMsg)
	}
	if receivedMsgFromAfter != expectedMsg {
		t.Fatalf("expected msg from After to match. Given: %q\nExpected: %q",
			receivedMsgFromAction, expectedMsg)
	}
}

func TestCommand_OnUsageError_hasCommandContext(t *testing.T) {
	app := &App{
		Commands: []*Command{
			{
				Name: "bar",
				Flags: []Flag{
					&IntFlag{Name: "flag"},
				},
				OnUsageError: func(c *Context, err error, _ bool) error {
					return fmt.Errorf("intercepted in %s: %s", c.Command.Name, err.Error())
				},
			},
		},
	}

	err := app.Run([]string{"foo", "bar", "--flag=wrong"})
	if err == nil {
		t.Fatalf("expected to receive error from Run, got none")
	}

	if !strings.HasPrefix(err.Error(), "intercepted in bar") {
		t.Errorf("Expect an intercepted error, but got \"%v\"", err)
	}
}

func TestCommand_OnUsageError_WithWrongFlagValue(t *testing.T) {
	app := &App{
		Commands: []*Command{
			{
				Name: "bar",
				Flags: []Flag{
					&IntFlag{Name: "flag"},
				},
				OnUsageError: func(c *Context, err error, _ bool) error {
					if !strings.HasPrefix(err.Error(), "invalid value \"wrong\"") {
						t.Errorf("Expect an invalid value error, but got \"%v\"", err)
					}
					return errors.New("intercepted: " + err.Error())
				},
			},
		},
	}

	err := app.Run([]string{"foo", "bar", "--flag=wrong"})
	if err == nil {
		t.Fatalf("expected to receive error from Run, got none")
	}

	if !strings.HasPrefix(err.Error(), "intercepted: invalid value") {
		t.Errorf("Expect an intercepted error, but got \"%v\"", err)
	}
}

func TestCommand_OnUsageError_WithSubcommand(t *testing.T) {
	app := &App{
		Commands: []*Command{
			{
				Name: "bar",
				Subcommands: []*Command{
					{
						Name: "baz",
					},
				},
				Flags: []Flag{
					&IntFlag{Name: "flag"},
				},
				OnUsageError: func(c *Context, err error, _ bool) error {
					if !strings.HasPrefix(err.Error(), "invalid value \"wrong\"") {
						t.Errorf("Expect an invalid value error, but got \"%v\"", err)
					}
					return errors.New("intercepted: " + err.Error())
				},
			},
		},
	}

	err := app.Run([]string{"foo", "bar", "--flag=wrong"})
	if err == nil {
		t.Fatalf("expected to receive error from Run, got none")
	}

	if !strings.HasPrefix(err.Error(), "intercepted: invalid value") {
		t.Errorf("Expect an intercepted error, but got \"%v\"", err)
	}
}

func TestCommand_Run_SubcommandsCanUseErrWriter(t *testing.T) {
	app := &App{
		ErrWriter: ioutil.Discard,
		Commands: []*Command{
			{
				Name:  "bar",
				Usage: "this is for testing",
				Subcommands: []*Command{
					{
						Name:  "baz",
						Usage: "this is for testing",
						Action: func(c *Context) error {
							if c.App.ErrWriter != ioutil.Discard {
								return fmt.Errorf("ErrWriter not passed")
							}

							return nil
						},
					},
				},
			},
		},
	}

	err := app.Run([]string{"foo", "bar", "baz"})
	if err != nil {
		t.Fatal(err)
	}
}

func TestCommandSkipFlagParsing(t *testing.T) {
	cases := []struct {
		testArgs     args
		expectedArgs *args
		expectedErr  error
	}{
		{testArgs: args{"some-exec", "some-command", "some-arg", "--flag", "foo"}, expectedArgs: &args{"some-arg", "--flag", "foo"}, expectedErr: nil},
		{testArgs: args{"some-exec", "some-command", "some-arg", "--flag=foo"}, expectedArgs: &args{"some-arg", "--flag=foo"}, expectedErr: nil},
	}

	for _, c := range cases {
		var args Args
		app := &App{
			Commands: []*Command{
				{
					SkipFlagParsing: true,
					Name:            "some-command",
					Flags: []Flag{
						&StringFlag{Name: "flag"},
					},
					Action: func(c *Context) error {
						fmt.Printf("%+v\n", c.String("flag"))
						args = c.Args()
						return nil
					},
				},
			},
		}

		err := app.Run(c.testArgs)
		expect(t, err, c.expectedErr)
		expect(t, args, c.expectedArgs)
	}
}
<|endoftext|>


<commit_before>package pipelineserver

import (
	"fmt"
	"sync"

	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"

	"golang.org/x/net/context"

	"github.com/pachyderm/pachyderm/src/pfs"
	"github.com/pachyderm/pachyderm/src/pps"
	"go.pedge.io/protolog"
)

type pipelineController struct {
	pfsAPIClient      pfs.APIClient
	jobAPIClient      pps.JobAPIClient
	pipelineAPIClient pps.PipelineAPIClient
	pipelineInfo      *pps.PipelineInfo

	ctx       context.Context
	cancel    context.CancelFunc
	waitGroup *sync.WaitGroup
}

func newPipelineController(
	pfsAPIClient pfs.APIClient,
	jobAPIClient pps.JobAPIClient,
	pipelineAPIClient pps.PipelineAPIClient,
	pipelineInfo *pps.PipelineInfo,
) *pipelineController {
	ctx, cancel := context.WithCancel(context.Background())
	return &pipelineController{
		pfsAPIClient,
		jobAPIClient,
		pipelineAPIClient,
		pipelineInfo,
		ctx,
		cancel,
		&sync.WaitGroup{},
	}
}

func (p *pipelineController) Start() error {
	// TODO: do not get all jobs each time, need a limit call on persist, more
	// generally, need all persist calls to have a limit
	jobInfos, err := p.jobAPIClient.ListJob(context.Background(), &pps.ListJobRequest{Pipeline: p.pipelineInfo.Pipeline})
	if err != nil {
		return err
	}
	lastCommit := &pfs.Commit{
		Repo: p.pipelineInfo.Input,
		// TODO: use initial commit id when moved to pfs package
		Id: "scratch",
	}
	if len(jobInfos.JobInfo) > 0 {
		lastCommit = jobInfos.JobInfo[0].Input
	}
	p.waitGroup.Add(1)
	go func() {
		defer p.waitGroup.Done()
		p.run(lastCommit)
	}()
	protolog.Infof("pachyderm.pps.pipelinserver: started pipeline controller for pipeline %s\n", p.pipelineInfo.Pipeline.Name)
	return nil
}

func (p *pipelineController) Cancel() error {
	p.cancel()
	// does not block until run is complete, but run will be in the process of cancelling
	<-p.ctx.Done()
	// wait until run completes
	p.waitGroup.Wait()
	return ignoreCanceledError(p.ctx.Err())
}

func (p *pipelineController) run(lastCommit *pfs.Commit) {
	for {
		// http://blog.golang.org/context
		commitErrorPairC := make(chan commitErrorPair, 1)
		go func() { commitErrorPairC <- p.runInner(p.ctx, lastCommit) }()
		select {
		case <-p.ctx.Done():
			_ = <-commitErrorPairC
			logIfError(p.ctx.Err())
			return
		case commitErrorPair := <-commitErrorPairC:
			// TODO: this will call with the same arguments over and over, there needs to be a better design here
			if !logIfError(commitErrorPair.Err) {
				lastCommit = commitErrorPair.Commit
			}
		}
	}
}

type commitErrorPair struct {
	Commit *pfs.Commit
	Err    error
}

func (p *pipelineController) runInner(ctx context.Context, lastCommit *pfs.Commit) commitErrorPair {
	listCommitRequest := &pfs.ListCommitRequest{
		Repo:       lastCommit.Repo,
		CommitType: pfs.CommitType_COMMIT_TYPE_READ,
		From:       lastCommit,
		Block:      true,
	}
	protolog.Infof("pachyderm.pps.pipelineserver: calling pfs.ListCommit with %v\n", listCommitRequest)
	commitInfos, err := p.pfsAPIClient.ListCommit(ctx, listCommitRequest)
	protolog.Infof("pachyderm.pps.pipelineserver: pfs.ListCommit call returned with %v %v\n", commitInfos, err)
	if err != nil {
		return commitErrorPair{Err: err}
	}
	if len(commitInfos.CommitInfo) == 0 {
		return commitErrorPair{Err: fmt.Errorf("pachyderm.pps.pipelineserver: we expected at least one *pfs.CommitInfo returned from blocking call, but no *pfs.CommitInfo structs were returned for %v", lastCommit)}
	}
	// going in reverse order, oldest to newest
	for _, commitInfo := range commitInfos.CommitInfo {
		protolog.Infof("pachyderm.pps.pipelineserver: got new commit for pipeline %s: %s\n", p.pipelineInfo.Pipeline.Name, commitInfo.Commit)
		if err := p.createJobForCommitInfo(ctx, commitInfo); err != nil {
			return commitErrorPair{Err: err}
		}
	}
	return commitErrorPair{Commit: commitInfos.CommitInfo[len(commitInfos.CommitInfo)-1].Commit}
}

func (p *pipelineController) createJobForCommitInfo(ctx context.Context, commitInfo *pfs.CommitInfo) error {
	parentOutputCommit, err := p.getParentOutputCommit(ctx, commitInfo)
	if err != nil {
		return err
	}
	_, err = p.jobAPIClient.CreateJob(
		ctx,
		&pps.CreateJobRequest{
			Spec: &pps.CreateJobRequest_Pipeline{
				Pipeline: p.pipelineInfo.Pipeline,
			},
			Input:        commitInfo.Commit,
			OutputParent: parentOutputCommit,
		},
	)
	return err
}

func (p *pipelineController) getParentOutputCommit(ctx context.Context, commitInfo *pfs.CommitInfo) (*pfs.Commit, error) {
	for commitInfo.ParentCommit != nil && commitInfo.ParentCommit.Id != pfs.InitialCommitID {
		outputCommit, err := p.getOutputCommit(ctx, commitInfo.ParentCommit)
		if err != nil {
			return nil, err
		}
		if outputCommit != nil {
			return outputCommit, nil
		}
	}
	return &pfs.Commit{
		Repo: commitInfo.Commit.Repo,
		Id:   pfs.InitialCommitID,
	}, nil
}

func (p *pipelineController) getOutputCommit(ctx context.Context, inputCommit *pfs.Commit) (*pfs.Commit, error) {
	jobInfos, err := p.jobAPIClient.ListJob(
		ctx,
		&pps.ListJobRequest{
			Pipeline: p.pipelineInfo.Pipeline,
			Input:    inputCommit,
		},
	)
	if err != nil {
		return nil, err
	}
	// newest to oldest assumed
	for _, jobInfo := range jobInfos.JobInfo {
		if jobInfo.Output != nil && containsSuccessJobStatus(jobInfo.JobStatus) {
			return jobInfo.Output, nil
		}
	}
	return nil, nil
}

// TODO: not assuming that last status is success
func containsSuccessJobStatus(jobStatuses []*pps.JobStatus) bool {
	for _, jobStatus := range jobStatuses {
		if jobStatus.Type == pps.JobStatusType_JOB_STATUS_TYPE_SUCCESS {
			return true
		}
	}
	return false
}

func logIfError(err error) bool {
	if err = ignoreCanceledError(err); err != nil {
		protolog.Errorf("pachyderm.pps.pipelineserver: error in pipeline controller: %s\n", err.Error())
		return true
	}
	return false
}

func ignoreCanceledError(err error) error {
	if err != context.Canceled && grpc.Code(err) != codes.Canceled {
		return err
	}
	return nil
}
<commit_msg>remove pps pipeline server pipeline controller debug log statements<commit_after>package pipelineserver

import (
	"fmt"
	"sync"

	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"

	"golang.org/x/net/context"

	"github.com/pachyderm/pachyderm/src/pfs"
	"github.com/pachyderm/pachyderm/src/pps"
	"go.pedge.io/protolog"
)

type pipelineController struct {
	pfsAPIClient      pfs.APIClient
	jobAPIClient      pps.JobAPIClient
	pipelineAPIClient pps.PipelineAPIClient
	pipelineInfo      *pps.PipelineInfo

	ctx       context.Context
	cancel    context.CancelFunc
	waitGroup *sync.WaitGroup
}

func newPipelineController(
	pfsAPIClient pfs.APIClient,
	jobAPIClient pps.JobAPIClient,
	pipelineAPIClient pps.PipelineAPIClient,
	pipelineInfo *pps.PipelineInfo,
) *pipelineController {
	ctx, cancel := context.WithCancel(context.Background())
	return &pipelineController{
		pfsAPIClient,
		jobAPIClient,
		pipelineAPIClient,
		pipelineInfo,
		ctx,
		cancel,
		&sync.WaitGroup{},
	}
}

func (p *pipelineController) Start() error {
	// TODO: do not get all jobs each time, need a limit call on persist, more
	// generally, need all persist calls to have a limit
	jobInfos, err := p.jobAPIClient.ListJob(context.Background(), &pps.ListJobRequest{Pipeline: p.pipelineInfo.Pipeline})
	if err != nil {
		return err
	}
	lastCommit := &pfs.Commit{
		Repo: p.pipelineInfo.Input,
		// TODO: use initial commit id when moved to pfs package
		Id: "scratch",
	}
	if len(jobInfos.JobInfo) > 0 {
		lastCommit = jobInfos.JobInfo[0].Input
	}
	p.waitGroup.Add(1)
	go func() {
		defer p.waitGroup.Done()
		p.run(lastCommit)
	}()
	protolog.Infof("pachyderm.pps.pipelinserver: started pipeline controller for pipeline %s\n", p.pipelineInfo.Pipeline.Name)
	return nil
}

func (p *pipelineController) Cancel() error {
	p.cancel()
	// does not block until run is complete, but run will be in the process of cancelling
	<-p.ctx.Done()
	// wait until run completes
	p.waitGroup.Wait()
	return ignoreCanceledError(p.ctx.Err())
}

func (p *pipelineController) run(lastCommit *pfs.Commit) {
	for {
		// http://blog.golang.org/context
		commitErrorPairC := make(chan commitErrorPair, 1)
		go func() { commitErrorPairC <- p.runInner(p.ctx, lastCommit) }()
		select {
		case <-p.ctx.Done():
			_ = <-commitErrorPairC
			logIfError(p.ctx.Err())
			return
		case commitErrorPair := <-commitErrorPairC:
			// TODO: this will call with the same arguments over and over, there needs to be a better design here
			if !logIfError(commitErrorPair.Err) {
				lastCommit = commitErrorPair.Commit
			}
		}
	}
}

type commitErrorPair struct {
	Commit *pfs.Commit
	Err    error
}

func (p *pipelineController) runInner(ctx context.Context, lastCommit *pfs.Commit) commitErrorPair {
	listCommitRequest := &pfs.ListCommitRequest{
		Repo:       lastCommit.Repo,
		CommitType: pfs.CommitType_COMMIT_TYPE_READ,
		From:       lastCommit,
		Block:      true,
	}
	commitInfos, err := p.pfsAPIClient.ListCommit(ctx, listCommitRequest)
	if err != nil {
		return commitErrorPair{Err: err}
	}
	if len(commitInfos.CommitInfo) == 0 {
		return commitErrorPair{Err: fmt.Errorf("pachyderm.pps.pipelineserver: we expected at least one *pfs.CommitInfo returned from blocking call, but no *pfs.CommitInfo structs were returned for %v", lastCommit)}
	}
	// going in reverse order, oldest to newest
	for _, commitInfo := range commitInfos.CommitInfo {
		protolog.Infof("pachyderm.pps.pipelineserver: got new commit for pipeline %s: %s\n", p.pipelineInfo.Pipeline.Name, commitInfo.Commit)
		if err := p.createJobForCommitInfo(ctx, commitInfo); err != nil {
			return commitErrorPair{Err: err}
		}
	}
	return commitErrorPair{Commit: commitInfos.CommitInfo[len(commitInfos.CommitInfo)-1].Commit}
}

func (p *pipelineController) createJobForCommitInfo(ctx context.Context, commitInfo *pfs.CommitInfo) error {
	parentOutputCommit, err := p.getParentOutputCommit(ctx, commitInfo)
	if err != nil {
		return err
	}
	_, err = p.jobAPIClient.CreateJob(
		ctx,
		&pps.CreateJobRequest{
			Spec: &pps.CreateJobRequest_Pipeline{
				Pipeline: p.pipelineInfo.Pipeline,
			},
			Input:        commitInfo.Commit,
			OutputParent: parentOutputCommit,
		},
	)
	return err
}

func (p *pipelineController) getParentOutputCommit(ctx context.Context, commitInfo *pfs.CommitInfo) (*pfs.Commit, error) {
	for commitInfo.ParentCommit != nil && commitInfo.ParentCommit.Id != pfs.InitialCommitID {
		outputCommit, err := p.getOutputCommit(ctx, commitInfo.ParentCommit)
		if err != nil {
			return nil, err
		}
		if outputCommit != nil {
			return outputCommit, nil
		}
	}
	return &pfs.Commit{
		Repo: commitInfo.Commit.Repo,
		Id:   pfs.InitialCommitID,
	}, nil
}

func (p *pipelineController) getOutputCommit(ctx context.Context, inputCommit *pfs.Commit) (*pfs.Commit, error) {
	jobInfos, err := p.jobAPIClient.ListJob(
		ctx,
		&pps.ListJobRequest{
			Pipeline: p.pipelineInfo.Pipeline,
			Input:    inputCommit,
		},
	)
	if err != nil {
		return nil, err
	}
	// newest to oldest assumed
	for _, jobInfo := range jobInfos.JobInfo {
		if jobInfo.Output != nil && containsSuccessJobStatus(jobInfo.JobStatus) {
			return jobInfo.Output, nil
		}
	}
	return nil, nil
}

// TODO: not assuming that last status is success
func containsSuccessJobStatus(jobStatuses []*pps.JobStatus) bool {
	for _, jobStatus := range jobStatuses {
		if jobStatus.Type == pps.JobStatusType_JOB_STATUS_TYPE_SUCCESS {
			return true
		}
	}
	return false
}

func logIfError(err error) bool {
	if err = ignoreCanceledError(err); err != nil {
		protolog.Errorf("pachyderm.pps.pipelineserver: error in pipeline controller: %s\n", err.Error())
		return true
	}
	return false
}

func ignoreCanceledError(err error) error {
	if err != context.Canceled && grpc.Code(err) != codes.Canceled {
		return err
	}
	return nil
}
<|endoftext|>