<commit_before>// Copyright (c) 2014-2015 by Michael Dvorkin. All Rights Reserved.
// Use of this source code is governed by a MIT-style license that can
// be found in the LICENSE file.

package donna

func (e *Evaluation) analyzeSafety() {
	var score Score
	var cover, safety Total
	oppositeBishops := e.oppositeBishops()

	if engine.trace {
		defer func() {
			var his, her Score
			e.checkpoint(`+King`, Total{*his.add(cover.white).add(safety.white), *her.add(cover.black).add(safety.black)})
			e.checkpoint(`-Cover`, cover)
			e.checkpoint(`-Safety`, safety)
		}()
	}

	// If any of the pawns or a king have moved then recalculate cover score.
	if e.position.king[White] != e.pawns.king[White] {
		e.pawns.cover[White] = e.kingCover(White)
		e.pawns.king[White] = e.position.king[White]
	}
	if e.position.king[Black] != e.pawns.king[Black] {
		e.pawns.cover[Black] = e.kingCover(Black)
		e.pawns.king[Black] = e.position.king[Black]
	}

	// Fetch king cover score from the pawn cache.
	cover.white.add(e.pawns.cover[White])
	cover.black.add(e.pawns.cover[Black])

	// Compute king's safety for both sides.
	if e.safety[White].threats > 0 {
		safety.white = e.kingSafety(White)
	}
	if e.safety[Black].threats > 0 {
		safety.black = e.kingSafety(Black)
	}

	// Less safe with opposite bishops.
	if oppositeBishops && e.isKingUnsafe(White) && safety.white.midgame < -onePawn / 10 * 8 {
		safety.white.midgame -= bishopDanger.midgame
	}
	if oppositeBishops && e.isKingUnsafe(Black) && safety.black.midgame < -onePawn / 10 * 8 {
		safety.black.midgame -= bishopDanger.midgame
	}

	// Calculate total king safety and pawn cover score.
	score.add(safety.white).sub(safety.black).apply(weightSafety)
	score.add(cover.white).sub(cover.black)
	e.score.add(score)
}

func (e *Evaluation) kingSafety(color uint8) (score Score) {
	p, rival := e.position, color^1
	safetyIndex, square := 0, int(p.king[color])

	// Find squares around the king that are being attacked by the
	// enemy and defended by our king only.
	defended := e.attacks[pawn(color)] | e.attacks[knight(color)] |
	            e.attacks[bishop(color)] | e.attacks[rook(color)] |
	            e.attacks[queen(color)]
	weak := e.attacks[king(color)] & e.attacks[rival] & ^defended

	// Find possible queen checks on weak squares around the king.
	// We only consider squares where the queen is protected and
	// can't be captured by the king.
	protected := e.attacks[pawn(rival)] | e.attacks[knight(rival)] |
		     e.attacks[bishop(rival)] | e.attacks[rook(rival)] |
		     e.attacks[king(rival)]
	checks := weak & e.attacks[queen(rival)] & protected & ^p.outposts[rival]
	if checks.any() {
		safetyIndex += queenCheck * checks.count()
	}

	// Out of all squares available for enemy pieces select the ones
	// that are not under our attack.
	safe := ^(e.attacks[color] | p.outposts[rival])

	// Are there any safe squares from where enemy Knight could give
	// us a check?
	if checks := knightMoves[square] & safe & e.attacks[knight(rival)]; checks.any() {
		safetyIndex += checks.count()
	}

	// Are there any safe squares from where enemy Bishop could give us a check?
	safeBishopMoves := p.bishopMoves(square) & safe
	if checks := safeBishopMoves & e.attacks[bishop(rival)]; checks.any() {
		safetyIndex += checks.count()
	}

	// Are there any safe squares from where enemy Rook could give us a check?
	safeRookMoves := p.rookMoves(square) & safe
	if checks := safeRookMoves & e.attacks[rook(rival)]; checks.any() {
		safetyIndex += checks.count()
	}

	// Are there any safe squares from where enemy Queen could give us a check?
	if checks := (safeBishopMoves | safeRookMoves) & e.attacks[queen(rival)]; checks.any() {
		safetyIndex += queenCheck / 2 * checks.count()
	}

	threatIndex := min(16, e.safety[color].attackers * e.safety[color].threats / 2) +
			(e.safety[color].attacks + weak.count()) * 3 +
			rank(color, square) - e.pawns.cover[color].midgame / 16
	safetyIndex = min(63, max(0, safetyIndex + threatIndex))

	score.midgame -= kingSafety[safetyIndex]

	return
}

func (e *Evaluation) kingCover(color uint8) (bonus Score) {
	p, square := e.position, int(e.position.king[color])

	// Don't bother with the cover if the king is too far out.
	if rank(color, square) <= A3H3 {
		// If we still have castle rights encourage castle pawns to stay intact
		// by scoring least safe castle.
		bonus.midgame = e.kingCoverBonus(color, square)
		if p.castles & castleKingside[color] != 0 {
			bonus.midgame = max(bonus.midgame, e.kingCoverBonus(color, homeKing[color] + 2))
		}
		if p.castles & castleQueenside[color] != 0 {
			bonus.midgame = max(bonus.midgame, e.kingCoverBonus(color, homeKing[color] - 2))
		}
	}

	bonus.endgame = e.kingPawnProximity(color, square)

	return
}

func (e *Evaluation) kingCoverBonus(color uint8, square int) (bonus int) {
	bonus = onePawn + onePawn / 3

	// Get pawns adjacent to and in front of the king.
	row, col := coordinate(square)
	area := maskRank[row] | maskPassed[color][square]
	cover := e.position.outposts[pawn(color)] & area
	storm := e.position.outposts[pawn(color^1)] & area

	// For each of the cover files find the closest friendly pawn. The penalty
	// is carried if the pawn is missing or is too far from the king.
	from, to := max(B1, col) - 1, min(G1, col) + 1
	for c := from; c <= to; c++ {

		// Friendly pawns protecting the kings.
		closest := 0
		if pawns := (cover & maskFile[c]); pawns.any() {
			closest = rank(color, pawns.closest(color))
		}
		bonus -= penaltyCover[closest]

		// Enemy pawns facing the king.
		if pawns := (storm & maskFile[c]); pawns.any() {
			farthest := rank(color, pawns.farthest(color^1))
			if closest == 0 { // No opposing friendly pawn.
				bonus -= penaltyStorm[farthest]
			} else if farthest == closest + 1 {
				bonus -= penaltyStormBlocked[farthest]
			} else {
				bonus -= penaltyStormUnblocked[farthest]
			}
		}
	}

	return
}

// Calculates endgame penalty to encourage a king stay closer to friendly pawns.
func (e *Evaluation) kingPawnProximity(color uint8, square int) (penalty int) {
	if pawns := e.position.outposts[pawn(color)]; pawns.any() && (pawns & e.attacks[king(color)]).empty() {
		proximity := 8

		for pawns.any() {
			proximity = min(proximity, distance[square][pawns.pop()])
		}

		penalty = -kingByPawn.endgame * (proximity - 1)
	}

	return
}

<commit_msg>Tempo bonus for checks<commit_after>// Copyright (c) 2014-2015 by Michael Dvorkin. All Rights Reserved.
// Use of this source code is governed by a MIT-style license that can
// be found in the LICENSE file.

package donna

func (e *Evaluation) analyzeSafety() {
	var score Score
	var cover, safety Total

	if engine.trace {
		defer func() {
			var our, their Score
			e.checkpoint(`+King`, Total{*our.add(cover.white).add(safety.white), *their.add(cover.black).add(safety.black)})
			e.checkpoint(`-Cover`, cover)
			e.checkpoint(`-Safety`, safety)
		}()
	}

	// If any of the pawns or a king have moved then recalculate cover score.
	if e.position.king[White] != e.pawns.king[White] {
		e.pawns.cover[White] = e.kingCover(White)
		e.pawns.king[White] = e.position.king[White]
	}
	if e.position.king[Black] != e.pawns.king[Black] {
		e.pawns.cover[Black] = e.kingCover(Black)
		e.pawns.king[Black] = e.position.king[Black]
	}

	// Fetch king cover score from the pawn cache.
	cover.white.add(e.pawns.cover[White])
	cover.black.add(e.pawns.cover[Black])

	// Calculate king's safety for both sides.
	if e.safety[White].threats > 0 {
		safety.white = e.kingSafety(White)
	}
	if e.safety[Black].threats > 0 {
		safety.black = e.kingSafety(Black)
	}

	// Calculate total king safety and pawn cover score.
	score.add(safety.white).sub(safety.black).apply(weightSafety)
	score.add(cover.white).sub(cover.black)
	e.score.add(score)
}

func (e *Evaluation) kingSafety(color uint8) (score Score) {
	p, rival := e.position, color^1
	safetyIndex, checkers, square := 0, 0, int(p.king[color])

	// Find squares around the king that are being attacked by the
	// enemy and defended by our king only.
	defended := e.attacks[pawn(color)] | e.attacks[knight(color)] |
	            e.attacks[bishop(color)] | e.attacks[rook(color)] |
	            e.attacks[queen(color)]
	weak := e.attacks[king(color)] & e.attacks[rival] & ^defended

	// Find possible queen checks on weak squares around the king.
	// We only consider squares where the queen is protected and
	// can't be captured by the king.
	protected := e.attacks[pawn(rival)] | e.attacks[knight(rival)] |
		     e.attacks[bishop(rival)] | e.attacks[rook(rival)] |
		     e.attacks[king(rival)]
	checks := weak & e.attacks[queen(rival)] & protected & ^p.outposts[rival]
	if checks.any() {
		checkers++
		safetyIndex += queenCheck * checks.count()
	}

	// Out of all squares available for enemy pieces select the ones
	// that are not under our attack.
	safe := ^(e.attacks[color] | p.outposts[rival])

	// Are there any safe squares from where enemy Knight could give
	// us a check?
	if checks := knightMoves[square] & safe & e.attacks[knight(rival)]; checks.any() {
		checkers++
		safetyIndex += checks.count()
	}

	// Are there any safe squares from where enemy Bishop could give us a check?
	safeBishopMoves := p.bishopMoves(square) & safe
	if checks := safeBishopMoves & e.attacks[bishop(rival)]; checks.any() {
		checkers++
		safetyIndex += checks.count()
	}

	// Are there any safe squares from where enemy Rook could give us a check?
	safeRookMoves := p.rookMoves(square) & safe
	if checks := safeRookMoves & e.attacks[rook(rival)]; checks.any() {
		checkers++
		safetyIndex += checks.count()
	}

	// Are there any safe squares from where enemy Queen could give us a check?
	if checks := (safeBishopMoves | safeRookMoves) & e.attacks[queen(rival)]; checks.any() {
		checkers++
		safetyIndex += queenCheck / 2 * checks.count()
	}

	threatIndex := min(16, e.safety[color].attackers * e.safety[color].threats / 2) +
			(e.safety[color].attacks + weak.count()) * 3 +
			rank(color, square) - e.pawns.cover[color].midgame / 16
	safetyIndex = min(63, max(0, safetyIndex + threatIndex))

	score.midgame -= kingSafety[safetyIndex]

	if checkers > 0 {
		score.add(rightToMove)
		if count > 1 {
			score.add(rightToMove)
		}
	}

	return score
}

func (e *Evaluation) kingCover(color uint8) (bonus Score) {
	p, square := e.position, int(e.position.king[color])

	// Don't bother with the cover if the king is too far out.
	if rank(color, square) <= A3H3 {
		// If we still have castle rights encourage castle pawns to stay intact
		// by scoring least safe castle.
		bonus.midgame = e.kingCoverBonus(color, square)
		if p.castles & castleKingside[color] != 0 {
			bonus.midgame = max(bonus.midgame, e.kingCoverBonus(color, homeKing[color] + 2))
		}
		if p.castles & castleQueenside[color] != 0 {
			bonus.midgame = max(bonus.midgame, e.kingCoverBonus(color, homeKing[color] - 2))
		}
	}

	bonus.endgame = e.kingPawnProximity(color, square)

	return
}

func (e *Evaluation) kingCoverBonus(color uint8, square int) (bonus int) {
	bonus = onePawn + onePawn / 3

	// Get pawns adjacent to and in front of the king.
	row, col := coordinate(square)
	area := maskRank[row] | maskPassed[color][square]
	cover := e.position.outposts[pawn(color)] & area
	storm := e.position.outposts[pawn(color^1)] & area

	// For each of the cover files find the closest friendly pawn. The penalty
	// is carried if the pawn is missing or is too far from the king.
	from, to := max(B1, col) - 1, min(G1, col) + 1
	for c := from; c <= to; c++ {

		// Friendly pawns protecting the kings.
		closest := 0
		if pawns := (cover & maskFile[c]); pawns.any() {
			closest = rank(color, pawns.closest(color))
		}
		bonus -= penaltyCover[closest]

		// Enemy pawns facing the king.
		if pawns := (storm & maskFile[c]); pawns.any() {
			farthest := rank(color, pawns.farthest(color^1))
			if closest == 0 { // No opposing friendly pawn.
				bonus -= penaltyStorm[farthest]
			} else if farthest == closest + 1 {
				bonus -= penaltyStormBlocked[farthest]
			} else {
				bonus -= penaltyStormUnblocked[farthest]
			}
		}
	}

	return
}

// Calculates endgame penalty to encourage a king stay closer to friendly pawns.
func (e *Evaluation) kingPawnProximity(color uint8, square int) (penalty int) {
	if pawns := e.position.outposts[pawn(color)]; pawns.any() && (pawns & e.attacks[king(color)]).empty() {
		proximity := 8

		for pawns.any() {
			proximity = min(proximity, distance[square][pawns.pop()])
		}

		penalty = -kingByPawn.endgame * (proximity - 1)
	}

	return
}

<|endoftext|>


<commit_before>// A set of utility functions for working with maps.
// Generally, maps and slices of any kind will work, but performance
// is optimized for maps returned by json.Unmarshal(b, &interface{}).  If
// all the maps are map[string]interface{}, and all the slices are
// []interface{}, and all the rest of the values are primitives, then
// reflection is avoided.
package maps

import (
	"encoding/json"
	"github.com/ansel1/merry"
	"github.com/elgs/gosplitargs"
	"reflect"
	"strconv"
	"strings"
)

// return a slice of the keys in the map
func Keys(m map[string]interface{}) (keys []string) {
	for key := range m {
		keys = append(keys, key)
	}
	return keys
}

// return a new map, which is the deep merge of m1 and m2.
// values in m2 override values in m1.
// This recurses into nested slices and maps
// Slices are merged simply by adding any m2 values which aren't
// already in m1's slice.  This won't do anything fancy with
// slices that have duplicate values.  Order is ignored.  E.g.:
//
//    [5, 6, 7] + [5, 5, 5, 4] = [5, 6, 7, 4]
//
// All values in the result will be normalized according to the following
// rules:
//
// 1. All maps with string keys will be converted into map[string]interface{}
// 2. All slices will be converted to []interface{}
// 3. All primitive numeric types will be converted into float64
// 4. All other types will not be converted, and will not be merged.  v2's value will just overwrite v1's
//
// New copies of all maps and slices are made, so v1 and v2 will not be modified.
func Merge(v1, v2 interface{}) interface{} {
	v1, _ = normalize(v1, true, false, true)
	v2, _ = normalize(v2, true, false, true)
	switch t1 := v1.(type) {
	case map[string]interface{}:
		if t2, isMap := v2.(map[string]interface{}); isMap {
			for key, value := range t2 {
				t1[key] = Merge(t1[key], value)
			}
			return t1
		}
	case []interface{}:
		if t2, isSlice := v2.([]interface{}); isSlice {
			orig := t1[:]
			for _, value := range t2 {
				if !sliceContains(orig, value) {
					t1 = append(t1, value)
				}
			}
			return t1
		}
	}
	return v2
}

func sliceContains(s []interface{}, v interface{}) bool {
	switch v.(type) {
	case string, float64, bool, nil:
		for _, value := range s {
			if value == v {
				return true
			}
		}
		return false
	}
	for _, value := range s {
		if reflect.DeepEqual(v, value) {
			return true
		}
	}
	return false
}

// Returns true if m1 contains all the key paths as m2, and
// the values at those paths are the equal.  I.E. returns
// true if m2 is a subset of m1.
// This will recurse into nested maps.
// When comparing to slice values, it will return true if
// slice 1 has at least one value which contains each of the
// values in slice 2.  It's kind of dumb though.  If slice 1
// contains a single value, say a big map, which contains *all*
// the values in slice 2, then this will return true.  In other words.
// when a match in slice 1 is found, that item is *not* removed from
// the search when matching the next value in slice 2.
// Examples:
//
//     {"color":"red"} contains {}
//     {"color":"red"} contains {"color":"red"}
//     {"color":"red","flavor":"beef"} contains {"color":"red"}
//     {"labels":{"color":"red","flavor":"beef"}} contains {"labels":{"flavor":"beef"}}
//     {"tags":["red","green","blue"]} contains {"tags":["red","green"]}

// This is what I mean about slice containment being a little simplistic:
//
//     {"resources":[{"type":"car","color":"red","wheels":4}]} contains {"resources":[{"type":"car"},{"color","red"},{"wheels":4}]}
//
// That will return true, despite there being 3 items in contained slice and only one item in the containing slice.  The
// one item in the containing slice matches each of the items in the contained slice.
func Contains(v1, v2 interface{}) bool {
	v1, _ = normalize(v1, false, false, false)
	v2, _ = normalize(v2, false, false, false)

	switch t1 := v1.(type) {
	case bool, nil, string, float64:
		return v1 == v2
	case map[string]interface{}:
		t2, isMap := v2.(map[string]interface{})
		if !isMap {
			// v1 is a map, but v2 isn't; v1 can't contain v2
			return false
		}
		if len(t2) > len(t1) {
			// if t2 is bigger than t1, then t1 can't contain t2
			return false
		}
		for key, val2 := range t2 {
			val1, present := t1[key]
			if !present || !Contains(val1, val2) {
				return false
			}
		}
		return true
	case []interface{}:
		t2, isSlice := v2.([]interface{})
		if !isSlice {
			// v1 is a slice, but v2 isn't; v1 can't contain v2
			return false
		}
		// first, normalize the values in v1, so we
		// don't re-normalize them in each loop
		t1copy := make([]interface{}, len(t1))
		for i, value := range t1 {
			t1copy[i], _ = normalize(value, false, false, false)
		}
		for _, val2 := range t2 {
			found := false
		Search:
			for _, value := range t1copy {
				if Contains(value, val2) {
					found = true
					break Search
				}
			}
			if !found {
				// one of the values in v2 was not found in v1
				return false
			}
		}
		return true
	default:
		return reflect.DeepEqual(v1, v2)
	}
}

// returns true if trees share common key paths, but the values
// at those paths are not equal.
// i.e. if the two maps were merged, no values would be overwritten
// conflicts == !contains(v1, v2) && !excludes(v1, v2)
// conflicts == !contains(merge(v1, v2), v1)
func Conflicts(m1, m2 map[string]interface{}) bool {
	return !Contains(Merge(m1, m2), m1)
}

func normalize(v interface{}, makeCopies, doMarshaling, recurse bool) (v2 interface{}, err error) {
	v2 = v
	copied := false
	switch t := v.(type) {
	case bool, string, nil, float64:
		return
	case int:
		return float64(t), nil
	case int8:
		return float64(t), nil
	case int16:
		return float64(t), nil
	case int32:
		return float64(t), nil
	case int64:
		return float64(t), nil
	case float32:
		return float64(t), nil
	case uint:
		return float64(t), nil
	case uint8:
		return float64(t), nil
	case uint16:
		return float64(t), nil
	case uint32:
		return float64(t), nil
	case uint64:
		return float64(t), nil
	case map[string]interface{}, []interface{}:
		if !makeCopies && !recurse {
			return
		}
	default:
		rv := reflect.ValueOf(v)
		if rv.Kind() == reflect.Map && rv.Type().Key().Kind() == reflect.String {
			copied = true
			m := make(map[string]interface{}, rv.Len())
			for _, v := range rv.MapKeys() {
				m[v.String()] = rv.MapIndex(v).Interface()
			}
			v2 = m
		} else if rv.Kind() == reflect.Slice {
			copied = true
			l := rv.Len()
			s := make([]interface{}, l)
			for i := 0; i < l; i++ {
				s[i] = rv.Index(i).Interface()
			}
			v2 = s
		} else if doMarshaling {
			// marshal/unmarshal
			var b []byte
			b, err = json.Marshal(v)
			if err != nil {
				return
			}
			v2 = nil
			err = json.Unmarshal(b, &v2)
			return
		}
	}
	if recurse || (makeCopies && !copied) {
		switch t := v2.(type) {
		case map[string]interface{}:
			var m map[string]interface{}
			if makeCopies && !copied {
				m = make(map[string]interface{}, len(t))
			} else {
				// modify in place
				m = t
			}
			v2 = m
			for key, value := range t {
				if recurse {
					if value, err = normalize(value, makeCopies, doMarshaling, recurse); err != nil {
						return
					}
				}
				m[key] = value
			}
		case []interface{}:
			var s []interface{}
			if makeCopies && !copied {
				s = make([]interface{}, len(t))
			} else {
				// modify in place
				s = t
			}
			v2 = s
			for i := 0; i < len(t); i++ {
				if recurse {
					if s[i], err = normalize(t[i], makeCopies, doMarshaling, recurse); err != nil {
						return
					}
				} else {
					s[i] = t[i]
				}
			}
		default:
			panic("Should be either a map or slice by now")
		}
	}

	return
}

// Recursively converts v1 into a tree of maps, slices, and primitives.
// The types in the result will be the types the json package uses for unmarshalling
// into interface{}.  The rules are:
//
// 1. All maps with string keys will be converted into map[string]interface{}
// 2. All slices will be converted to []interface{}
// 3. All primitive numeric types will be converted into float64
// 4. string, bool, and nil are unmodified
// 5. All other values will be converted into the above types by doing a json.Marshal and Unmarshal
//
// Values in v1 will be modified in place if possible
func Normalize(v1 interface{}) (interface{}, error) {
	return normalize(v1, false, true, true)
}

var PathNotFoundError = merry.New("Path not found")
var PathNotMapError = merry.New("Path not map")
var PathNotSliceError = merry.New("Path not slice")
var IndexOutOfBoundsError = merry.New("Index out of bounds")

// Extracts the value at path from v.
// Path is in the form:
//
//     response.things[2].color.red
//
// You can use `merry` to test the types of return errors:
//
//     _, err := maps.Get("","")
//     if merry.Is(err, maps.PathNotFoundError) {
//       ...
//
// `v` can be any primitive, map (must be keyed by string, but any value type), or slice, nested arbitrarily deep
func Get(v interface{}, path string) (interface{}, error) {
	parts, err := gosplitargs.SplitArgs(path, "\\.", false)
	if err != nil {
		return nil, merry.Prepend(err, "Couldn't parse the path")
	}
	out := v
	for i := 0; i < len(parts); i++ {
		part := strings.TrimSpace(parts[i])
		if len(part) == 0 {
			continue
		}
		sliceIdx := -1
		// first check of the path part ends in an array index, like
		//
		//     tags[2]
		//
		// Extract the "2", and truncate the part to "tags"
		if bracketIdx := strings.Index(part, "["); bracketIdx > -1 && strings.HasSuffix(part, "]") {
			if idx, err := strconv.Atoi(part[bracketIdx+1 : len(part)-1]); err == nil {
				sliceIdx = idx
				part = part[0:bracketIdx]
			}
		}

		if part = strings.TrimSpace(part); len(part) > 0 {
			// map key
			out, _ = normalize(out, false, false, false)
			if m, ok := out.(map[string]interface{}); ok {
				var present bool
				if out, present = m[part]; !present {
					return nil, PathNotFoundError.WithMessagef("%s not found", strings.Join(parts[0:i+1], "."))
				}
			} else {
				errPath := strings.Join(parts[0:i], ".")
				if len(errPath) == 0 {
					errPath = "v"
				}
				return nil, PathNotMapError.WithMessagef("%s is not a map", errPath)
			}
		}
		if sliceIdx > -1 {
			// slice index
			out, _ = normalize(out, false, false, false)
			if s, ok := out.([]interface{}); ok {
				if l := len(s); l <= sliceIdx {
					return nil, IndexOutOfBoundsError.WithMessagef("Index out of bounds at %s (len = %v)", strings.Join(parts[0:i+1], "."), l)
				} else {
					out = s[sliceIdx]
				}
			} else {
				errPath := strings.Join(append(parts[0:i], part), ".")
				if len(errPath) == 0 {
					errPath = "v"
				}
				return nil, PathNotSliceError.WithMessagef("%s is not a slice", errPath)
			}
		}
	}
	return out, nil
}

// returns true if v is:
//
// 1. nil
// 2. an empty string
// 3. an empty slice
// 4. an empty map
// 5. an empty array
// 6. an empty channel
//
// returns false otherwise
func Empty(v interface{}) bool {
	// no op.  just means the value wasn't a type that supports Len()
	defer func() { recover() }()
	switch t := v.(type) {
	case bool, int, int8, int16, int32, int64, float32, float64, uint, uint8, uint16, uint32, uint64:
		return false
	case nil:
		return true
	case string:
		return len(strings.TrimSpace(t)) == 0
	case map[string]interface{}:
		return len(t) == 0
	case []interface{}:
		return len(t) == 0
	default:
		rv := reflect.ValueOf(v)
		if rv.IsNil() {
			// handle case of (*Widget)(nil)
			return true
		}
		if rv.Kind() == reflect.Ptr {
			return Empty(rv.Elem())
		}
		return reflect.ValueOf(v).Len() == 0
	}
}
<commit_msg>capture new stack for get errors<commit_after>// A set of utility functions for working with maps.
// Generally, maps and slices of any kind will work, but performance
// is optimized for maps returned by json.Unmarshal(b, &interface{}).  If
// all the maps are map[string]interface{}, and all the slices are
// []interface{}, and all the rest of the values are primitives, then
// reflection is avoided.
package maps

import (
	"encoding/json"
	"github.com/ansel1/merry"
	"github.com/elgs/gosplitargs"
	"reflect"
	"strconv"
	"strings"
)

// return a slice of the keys in the map
func Keys(m map[string]interface{}) (keys []string) {
	for key := range m {
		keys = append(keys, key)
	}
	return keys
}

// return a new map, which is the deep merge of m1 and m2.
// values in m2 override values in m1.
// This recurses into nested slices and maps
// Slices are merged simply by adding any m2 values which aren't
// already in m1's slice.  This won't do anything fancy with
// slices that have duplicate values.  Order is ignored.  E.g.:
//
//    [5, 6, 7] + [5, 5, 5, 4] = [5, 6, 7, 4]
//
// All values in the result will be normalized according to the following
// rules:
//
// 1. All maps with string keys will be converted into map[string]interface{}
// 2. All slices will be converted to []interface{}
// 3. All primitive numeric types will be converted into float64
// 4. All other types will not be converted, and will not be merged.  v2's value will just overwrite v1's
//
// New copies of all maps and slices are made, so v1 and v2 will not be modified.
func Merge(v1, v2 interface{}) interface{} {
	v1, _ = normalize(v1, true, false, true)
	v2, _ = normalize(v2, true, false, true)
	switch t1 := v1.(type) {
	case map[string]interface{}:
		if t2, isMap := v2.(map[string]interface{}); isMap {
			for key, value := range t2 {
				t1[key] = Merge(t1[key], value)
			}
			return t1
		}
	case []interface{}:
		if t2, isSlice := v2.([]interface{}); isSlice {
			orig := t1[:]
			for _, value := range t2 {
				if !sliceContains(orig, value) {
					t1 = append(t1, value)
				}
			}
			return t1
		}
	}
	return v2
}

func sliceContains(s []interface{}, v interface{}) bool {
	switch v.(type) {
	case string, float64, bool, nil:
		for _, value := range s {
			if value == v {
				return true
			}
		}
		return false
	}
	for _, value := range s {
		if reflect.DeepEqual(v, value) {
			return true
		}
	}
	return false
}

// Returns true if m1 contains all the key paths as m2, and
// the values at those paths are the equal.  I.E. returns
// true if m2 is a subset of m1.
// This will recurse into nested maps.
// When comparing to slice values, it will return true if
// slice 1 has at least one value which contains each of the
// values in slice 2.  It's kind of dumb though.  If slice 1
// contains a single value, say a big map, which contains *all*
// the values in slice 2, then this will return true.  In other words.
// when a match in slice 1 is found, that item is *not* removed from
// the search when matching the next value in slice 2.
// Examples:
//
//     {"color":"red"} contains {}
//     {"color":"red"} contains {"color":"red"}
//     {"color":"red","flavor":"beef"} contains {"color":"red"}
//     {"labels":{"color":"red","flavor":"beef"}} contains {"labels":{"flavor":"beef"}}
//     {"tags":["red","green","blue"]} contains {"tags":["red","green"]}

// This is what I mean about slice containment being a little simplistic:
//
//     {"resources":[{"type":"car","color":"red","wheels":4}]} contains {"resources":[{"type":"car"},{"color","red"},{"wheels":4}]}
//
// That will return true, despite there being 3 items in contained slice and only one item in the containing slice.  The
// one item in the containing slice matches each of the items in the contained slice.
func Contains(v1, v2 interface{}) bool {
	v1, _ = normalize(v1, false, false, false)
	v2, _ = normalize(v2, false, false, false)

	switch t1 := v1.(type) {
	case bool, nil, string, float64:
		return v1 == v2
	case map[string]interface{}:
		t2, isMap := v2.(map[string]interface{})
		if !isMap {
			// v1 is a map, but v2 isn't; v1 can't contain v2
			return false
		}
		if len(t2) > len(t1) {
			// if t2 is bigger than t1, then t1 can't contain t2
			return false
		}
		for key, val2 := range t2 {
			val1, present := t1[key]
			if !present || !Contains(val1, val2) {
				return false
			}
		}
		return true
	case []interface{}:
		t2, isSlice := v2.([]interface{})
		if !isSlice {
			// v1 is a slice, but v2 isn't; v1 can't contain v2
			return false
		}
		// first, normalize the values in v1, so we
		// don't re-normalize them in each loop
		t1copy := make([]interface{}, len(t1))
		for i, value := range t1 {
			t1copy[i], _ = normalize(value, false, false, false)
		}
		for _, val2 := range t2 {
			found := false
		Search:
			for _, value := range t1copy {
				if Contains(value, val2) {
					found = true
					break Search
				}
			}
			if !found {
				// one of the values in v2 was not found in v1
				return false
			}
		}
		return true
	default:
		return reflect.DeepEqual(v1, v2)
	}
}

// returns true if trees share common key paths, but the values
// at those paths are not equal.
// i.e. if the two maps were merged, no values would be overwritten
// conflicts == !contains(v1, v2) && !excludes(v1, v2)
// conflicts == !contains(merge(v1, v2), v1)
func Conflicts(m1, m2 map[string]interface{}) bool {
	return !Contains(Merge(m1, m2), m1)
}

func normalize(v interface{}, makeCopies, doMarshaling, recurse bool) (v2 interface{}, err error) {
	v2 = v
	copied := false
	switch t := v.(type) {
	case bool, string, nil, float64:
		return
	case int:
		return float64(t), nil
	case int8:
		return float64(t), nil
	case int16:
		return float64(t), nil
	case int32:
		return float64(t), nil
	case int64:
		return float64(t), nil
	case float32:
		return float64(t), nil
	case uint:
		return float64(t), nil
	case uint8:
		return float64(t), nil
	case uint16:
		return float64(t), nil
	case uint32:
		return float64(t), nil
	case uint64:
		return float64(t), nil
	case map[string]interface{}, []interface{}:
		if !makeCopies && !recurse {
			return
		}
	default:
		rv := reflect.ValueOf(v)
		if rv.Kind() == reflect.Map && rv.Type().Key().Kind() == reflect.String {
			copied = true
			m := make(map[string]interface{}, rv.Len())
			for _, v := range rv.MapKeys() {
				m[v.String()] = rv.MapIndex(v).Interface()
			}
			v2 = m
		} else if rv.Kind() == reflect.Slice {
			copied = true
			l := rv.Len()
			s := make([]interface{}, l)
			for i := 0; i < l; i++ {
				s[i] = rv.Index(i).Interface()
			}
			v2 = s
		} else if doMarshaling {
			// marshal/unmarshal
			var b []byte
			b, err = json.Marshal(v)
			if err != nil {
				return
			}
			v2 = nil
			err = json.Unmarshal(b, &v2)
			return
		}
	}
	if recurse || (makeCopies && !copied) {
		switch t := v2.(type) {
		case map[string]interface{}:
			var m map[string]interface{}
			if makeCopies && !copied {
				m = make(map[string]interface{}, len(t))
			} else {
				// modify in place
				m = t
			}
			v2 = m
			for key, value := range t {
				if recurse {
					if value, err = normalize(value, makeCopies, doMarshaling, recurse); err != nil {
						return
					}
				}
				m[key] = value
			}
		case []interface{}:
			var s []interface{}
			if makeCopies && !copied {
				s = make([]interface{}, len(t))
			} else {
				// modify in place
				s = t
			}
			v2 = s
			for i := 0; i < len(t); i++ {
				if recurse {
					if s[i], err = normalize(t[i], makeCopies, doMarshaling, recurse); err != nil {
						return
					}
				} else {
					s[i] = t[i]
				}
			}
		default:
			panic("Should be either a map or slice by now")
		}
	}

	return
}

// Recursively converts v1 into a tree of maps, slices, and primitives.
// The types in the result will be the types the json package uses for unmarshalling
// into interface{}.  The rules are:
//
// 1. All maps with string keys will be converted into map[string]interface{}
// 2. All slices will be converted to []interface{}
// 3. All primitive numeric types will be converted into float64
// 4. string, bool, and nil are unmodified
// 5. All other values will be converted into the above types by doing a json.Marshal and Unmarshal
//
// Values in v1 will be modified in place if possible
func Normalize(v1 interface{}) (interface{}, error) {
	return normalize(v1, false, true, true)
}

var PathNotFoundError = merry.New("Path not found")
var PathNotMapError = merry.New("Path not map")
var PathNotSliceError = merry.New("Path not slice")
var IndexOutOfBoundsError = merry.New("Index out of bounds")

// Extracts the value at path from v.
// Path is in the form:
//
//     response.things[2].color.red
//
// You can use `merry` to test the types of return errors:
//
//     _, err := maps.Get("","")
//     if merry.Is(err, maps.PathNotFoundError) {
//       ...
//
// `v` can be any primitive, map (must be keyed by string, but any value type), or slice, nested arbitrarily deep
func Get(v interface{}, path string) (interface{}, error) {
	parts, err := gosplitargs.SplitArgs(path, "\\.", false)
	if err != nil {
		return nil, merry.Prepend(err, "Couldn't parse the path")
	}
	out := v
	for i := 0; i < len(parts); i++ {
		part := strings.TrimSpace(parts[i])
		if len(part) == 0 {
			continue
		}
		sliceIdx := -1
		// first check of the path part ends in an array index, like
		//
		//     tags[2]
		//
		// Extract the "2", and truncate the part to "tags"
		if bracketIdx := strings.Index(part, "["); bracketIdx > -1 && strings.HasSuffix(part, "]") {
			if idx, err := strconv.Atoi(part[bracketIdx+1 : len(part)-1]); err == nil {
				sliceIdx = idx
				part = part[0:bracketIdx]
			}
		}

		if part = strings.TrimSpace(part); len(part) > 0 {
			// map key
			out, _ = normalize(out, false, false, false)
			if m, ok := out.(map[string]interface{}); ok {
				var present bool
				if out, present = m[part]; !present {
					return nil, PathNotFoundError.Here().WithMessagef("%s not found", strings.Join(parts[0:i+1], "."))
				}
			} else {
				errPath := strings.Join(parts[0:i], ".")
				if len(errPath) == 0 {
					errPath = "v"
				}
				return nil, PathNotMapError.Here().WithMessagef("%s is not a map", errPath)
			}
		}
		if sliceIdx > -1 {
			// slice index
			out, _ = normalize(out, false, false, false)
			if s, ok := out.([]interface{}); ok {
				if l := len(s); l <= sliceIdx {
					return nil, IndexOutOfBoundsError.Here().WithMessagef("Index out of bounds at %s (len = %v)", strings.Join(parts[0:i+1], "."), l)
				} else {
					out = s[sliceIdx]
				}
			} else {
				errPath := strings.Join(append(parts[0:i], part), ".")
				if len(errPath) == 0 {
					errPath = "v"
				}
				return nil, PathNotSliceError.Here().WithMessagef("%s is not a slice", errPath)
			}
		}
	}
	return out, nil
}

// returns true if v is:
//
// 1. nil
// 2. an empty string
// 3. an empty slice
// 4. an empty map
// 5. an empty array
// 6. an empty channel
//
// returns false otherwise
func Empty(v interface{}) bool {
	// no op.  just means the value wasn't a type that supports Len()
	defer func() { recover() }()
	switch t := v.(type) {
	case bool, int, int8, int16, int32, int64, float32, float64, uint, uint8, uint16, uint32, uint64:
		return false
	case nil:
		return true
	case string:
		return len(strings.TrimSpace(t)) == 0
	case map[string]interface{}:
		return len(t) == 0
	case []interface{}:
		return len(t) == 0
	default:
		rv := reflect.ValueOf(v)
		if rv.IsNil() {
			// handle case of (*Widget)(nil)
			return true
		}
		if rv.Kind() == reflect.Ptr {
			return Empty(rv.Elem())
		}
		return reflect.ValueOf(v).Len() == 0
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"

	"github.com/schollz/goagrep/goagrep"
)

func init() {
	// Build database
	// only needs to be done once!
	databaseFile := "words.db"
	wordlist := "testlist"
	tupleLength := 3
	if _, err := os.Stat(databaseFile); os.IsNotExist(err) {
		goagrep.GenerateDB(wordlist, databaseFile, tupleLength)
	}

}
func main() {
	// Find word
	databaseFile := "words.db"
	searchWord := "heroint"
	word, score := goagrep.GetMatch(searchWord, databaseFile)
	fmt.Println(word, score)
}
<commit_msg>Fixed to use 1.6 version<commit_after>package main

import (
	"fmt"
	"os"

	"github.com/schollz/goagrep/goagrep"
)

var databaseFile string
var wordlist string
var tupleLength int

func init() {
	databaseFile = "words.db"
	wordlist = "testlist"
	tupleLength = 5

	// Build database
	if _, err := os.Stat(databaseFile); os.IsNotExist(err) {
		goagrep.GenerateDB(wordlist, databaseFile, tupleLength, true)
	}
}

func main() {
	// Find word
	searchWord := "heroint"
	word, score, err := goagrep.GetMatch(searchWord, databaseFile)
	fmt.Println(word, score, err)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"log"
	"os"
	"reflect"

	// sm "github.com/ebittleman/go-servicemanager"
	sm "gopkg.in/ebittleman/go-servicemanager.v0.0.1"
)

const LOG_SERVICE = "Log"
const CONFIG_SERVICE = "Config"

type Adder interface {
	Add(int, int) int
}

type simpleAdder struct{}

func (s *simpleAdder) Add(a, b int) int {
	return a + b
}

type halfAdder struct{}

func (s *halfAdder) Add(a, b int) int {
	return int((a + b) / 2)
}

type notAdder struct{}

type Calculator struct {
	Adder Adder
	Log   *log.Logger `inject:"Log"`
}

func (c *Calculator) Add(a, b int) int {
	ans := c.Adder.Add(a, b)
	c.Log.Printf("%d + %d = %d\n", a, b, ans)
	return ans
}

var factories map[string]sm.ServiceFactoryCallback

func init() {
	factories = map[string]sm.ServiceFactoryCallback{
		"SimpleAdder": func(sl sm.ServiceLocator) (interface{}, error) {
			return &simpleAdder{}, nil
		},

		"HalfAdder": func(sl sm.ServiceLocator) (interface{}, error) {
			return &halfAdder{}, nil
		},

		"NotAdder": func(sl sm.ServiceLocator) (interface{}, error) {
			return &notAdder{}, nil
		},

		"Calculator": func(sl sm.ServiceLocator) (interface{}, error) {
			adder, _ := sl.Get("SimpleAdder")
			return &Calculator{Adder: adder.(Adder)}, nil
		},

		"HalfCalculator": func(sl sm.ServiceLocator) (interface{}, error) {
			adder, _ := sl.Get("HalfAdder")
			return &Calculator{Adder: adder.(Adder)}, nil
		},

		"NotCalculator": func(sl sm.ServiceLocator) (interface{}, error) {
			adder, _ := sl.Get("NotAdder")
			return &Calculator{Adder: adder.(Adder)}, nil
		},

		CONFIG_SERVICE: func(sl sm.ServiceLocator) (interface{}, error) {
			return map[string]interface{}{
				"log.writer": os.Stderr,
				"log.prefix": "example: ",
				"log.flags":  log.LstdFlags,
			}, nil
		},

		LOG_SERVICE: func(sl sm.ServiceLocator) (interface{}, error) {
			config := GetConfig(sl)
			return log.New(
				config["log.writer"].(io.Writer),
				config["log.prefix"].(string),
				config["log.flags"].(int),
			), nil
		},
	}
}

func main() {
	manager := sm.New()

	for name, factory := range factories {
		manager.Set(name, factory)
	}

	log := GetLog(manager)

	_, err := GetCalculator("NotCalculator", manager)

	if err != nil {
		log.Println(err)
	}

	calc, err := GetCalculator("Calculator", manager)

	if err != nil {
		log.Fatalln(err)
	}

	halfy, err := GetCalculator("HalfCalculator", manager)

	if err != nil {
		log.Fatalln(err)
	}

	one := halfy.Add(1, 1)

	if one != 1 {
		log.Fatalln(fmt.Errorf("Expected: 1, Got: %d", one))
	}

	two := calc.Add(1, 1)

	if two != 2 {
		log.Fatalln(fmt.Errorf("Expected: 2, Got: %d", two))
	}

}

func GetLog(sl sm.ServiceLocator) *log.Logger {
	inst, err := sl.Get(LOG_SERVICE)

	if err != nil {
		panic(err)
	}

	return inst.(*log.Logger)
}

func GetConfig(sl sm.ServiceLocator) map[string]interface{} {
	inst, err := sl.Get(CONFIG_SERVICE)

	if err != nil {
		panic(err)
	}

	return inst.(map[string]interface{})
}

func GetCalculator(name string, sl sm.ServiceLocator) (*Calculator, error) {
	inst, err := sl.Get(name)

	if err != nil {
		return nil, err
	}

	calculator, ok := inst.(*Calculator)

	if !ok {
		return nil, fmt.Errorf("Expected: *Calculator, Got: %v", reflect.TypeOf(inst))
	}

	return calculator, nil
}
<commit_msg>setting import to major package<commit_after>package main

import (
	"fmt"
	"io"
	"log"
	"os"
	"reflect"

	// sm "github.com/ebittleman/go-servicemanager"
	sm "gopkg.in/ebittleman/go-servicemanager.v0"
)

const LOG_SERVICE = "Log"
const CONFIG_SERVICE = "Config"

type Adder interface {
	Add(int, int) int
}

type simpleAdder struct{}

func (s *simpleAdder) Add(a, b int) int {
	return a + b
}

type halfAdder struct{}

func (s *halfAdder) Add(a, b int) int {
	return int((a + b) / 2)
}

type notAdder struct{}

type Calculator struct {
	Adder Adder
	Log   *log.Logger `inject:"Log"`
}

func (c *Calculator) Add(a, b int) int {
	ans := c.Adder.Add(a, b)
	c.Log.Printf("%d + %d = %d\n", a, b, ans)
	return ans
}

var factories map[string]sm.ServiceFactoryCallback

func init() {
	factories = map[string]sm.ServiceFactoryCallback{
		"SimpleAdder": func(sl sm.ServiceLocator) (interface{}, error) {
			return &simpleAdder{}, nil
		},

		"HalfAdder": func(sl sm.ServiceLocator) (interface{}, error) {
			return &halfAdder{}, nil
		},

		"NotAdder": func(sl sm.ServiceLocator) (interface{}, error) {
			return &notAdder{}, nil
		},

		"Calculator": func(sl sm.ServiceLocator) (interface{}, error) {
			adder, _ := sl.Get("SimpleAdder")
			return &Calculator{Adder: adder.(Adder)}, nil
		},

		"HalfCalculator": func(sl sm.ServiceLocator) (interface{}, error) {
			adder, _ := sl.Get("HalfAdder")
			return &Calculator{Adder: adder.(Adder)}, nil
		},

		"NotCalculator": func(sl sm.ServiceLocator) (interface{}, error) {
			adder, _ := sl.Get("NotAdder")
			return &Calculator{Adder: adder.(Adder)}, nil
		},

		CONFIG_SERVICE: func(sl sm.ServiceLocator) (interface{}, error) {
			return map[string]interface{}{
				"log.writer": os.Stderr,
				"log.prefix": "example: ",
				"log.flags":  log.LstdFlags,
			}, nil
		},

		LOG_SERVICE: func(sl sm.ServiceLocator) (interface{}, error) {
			config := GetConfig(sl)
			return log.New(
				config["log.writer"].(io.Writer),
				config["log.prefix"].(string),
				config["log.flags"].(int),
			), nil
		},
	}
}

func main() {
	manager := sm.New()

	for name, factory := range factories {
		manager.Set(name, factory)
	}

	log := GetLog(manager)

	_, err := GetCalculator("NotCalculator", manager)

	if err != nil {
		log.Println(err)
	}

	calc, err := GetCalculator("Calculator", manager)

	if err != nil {
		log.Fatalln(err)
	}

	halfy, err := GetCalculator("HalfCalculator", manager)

	if err != nil {
		log.Fatalln(err)
	}

	one := halfy.Add(1, 1)

	if one != 1 {
		log.Fatalln(fmt.Errorf("Expected: 1, Got: %d", one))
	}

	two := calc.Add(1, 1)

	if two != 2 {
		log.Fatalln(fmt.Errorf("Expected: 2, Got: %d", two))
	}

}

func GetLog(sl sm.ServiceLocator) *log.Logger {
	inst, err := sl.Get(LOG_SERVICE)

	if err != nil {
		panic(err)
	}

	return inst.(*log.Logger)
}

func GetConfig(sl sm.ServiceLocator) map[string]interface{} {
	inst, err := sl.Get(CONFIG_SERVICE)

	if err != nil {
		panic(err)
	}

	return inst.(map[string]interface{})
}

func GetCalculator(name string, sl sm.ServiceLocator) (*Calculator, error) {
	inst, err := sl.Get(name)

	if err != nil {
		return nil, err
	}

	calculator, ok := inst.(*Calculator)

	if !ok {
		return nil, fmt.Errorf("Expected: *Calculator, Got: %v", reflect.TypeOf(inst))
	}

	return calculator, nil
}
<|endoftext|>


<commit_before>package workercmd

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/user"
	"path/filepath"
	"strings"
	"time"

	"code.cloudfoundry.org/lager"
	"github.com/concourse/concourse/atc"
	concourseCmd "github.com/concourse/concourse/cmd"
	"github.com/concourse/flag"
	"github.com/jessevdk/go-flags"
	"github.com/tedsuo/ifrit"
)

type Certs struct {
	Dir string `long:"certs-dir" description:"Directory to use when creating the resource certificates volume."`
}

type RuntimeConfiguration struct {
	Runtime string `long:"runtime" default:"guardian" choice:"guardian" choice:"containerd" choice:"houdini" description:"Runtime to use with the worker. Please note that Houdini is insecure and doesn't run 'tasks' in containers."`
}

type GuardianRuntime struct {
	Bin            string        `long:"bin"        description:"Path to a garden server executable (non-absolute names get resolved from $PATH)."`
	DNS            DNSConfig     `group:"DNS Proxy Configuration" namespace:"dns-proxy"`
	RequestTimeout time.Duration `long:"request-timeout" default:"5m" description:"How long to wait for requests to the Garden server to complete. 0 means no timeout."`

	Config      flag.File `long:"config"     description:"Path to a config file to use for the Garden backend. e.g. 'foo-bar=a,b' for '--foo-bar a --foo-bar b'."`
	BinaryFlags GdnBinaryFlags
}

type ContainerdRuntime struct {
	Config         flag.File     `long:"config"     description:"Path to a config file to use for the Containerd daemon."`
	Bin            string        `long:"bin"        description:"Path to a containerd executable (non-absolute names get resolved from $PATH)."`
	InitBin        string        `long:"init-bin"   default:"/usr/local/concourse/bin/init" description:"Path to an init executable (non-absolute names get resolved from $PATH)."`
	CNIPluginsDir  string        `long:"cni-plugins-dir" default:"/usr/local/concourse/bin" description:"Path to CNI network plugins."`
	RequestTimeout time.Duration `long:"request-timeout" default:"5m" description:"How long to wait for requests to Containerd to complete. 0 means no timeout."`

	Network struct {
		ExternalIP flag.IP `long:"external-ip" description:"IP address to use to reach container's mapped ports. Autodetected if not specified."`
		//TODO can DNSConfig be simplifed to just a bool rather than struct with a bool?
		DNS                DNSConfig `group:"DNS Proxy Configuration" namespace:"dns-proxy"`
		DNSServers         []string  `long:"dns-server" description:"DNS server IP address to use instead of automatically determined servers. Can be specified multiple times."`
		RestrictedNetworks []string  `long:"restricted-network" description:"Network ranges to which traffic from containers will be restricted. Can be specified multiple times."`
		Pool               string    `long:"network-pool" default:"10.80.0.0/16" description:"Network range to use for dynamically allocated container subnets."`
		MTU                int       `long:"mtu" description:"MTU size for container network interfaces. Defaults to the MTU of the interface used for outbound access by the host. Max allowed value is 1500."`
	} `group:"Container Networking"`

	MaxContainers int `long:"max-containers" default:"250" description:"Max container capacity. 0 means no limit."`
}

const containerdRuntime = "containerd"
const guardianRuntime = "guardian"
const houdiniRuntime = "houdini"

func (cmd WorkerCommand) LessenRequirements(prefix string, command *flags.Command) {
	// configured as work-dir/volumes
	command.FindOptionByLongName(prefix + "baggageclaim-volumes").Required = false
}

// Chooses the appropriate runtime based on CONCOURSE_RUNTIME_TYPE.
// The runtime is represented as a Ifrit runner that must include a Garden Server process. The Garden server exposes API
// endpoints that allow the ATC to make container related requests to the worker.
// The runner may also include additional processes such as the runtime's daemon or a DNS proxy server.
func (cmd *WorkerCommand) gardenServerRunner(logger lager.Logger) (atc.Worker, ifrit.Runner, error) {
	err := cmd.checkRoot()
	if err != nil {
		return atc.Worker{}, nil, err
	}

	err = cmd.verifyRuntimeFlags()
	if err != nil {
		return atc.Worker{}, nil, err
	}

	worker := cmd.Worker.Worker()
	worker.Platform = "linux"

	if cmd.Certs.Dir != "" {
		worker.CertsPath = &cmd.Certs.Dir
	}

	worker.ResourceTypes, err = cmd.loadResources(logger.Session("load-resources"))
	if err != nil {
		return atc.Worker{}, nil, err
	}

	worker.Name, err = cmd.workerName()
	if err != nil {
		return atc.Worker{}, nil, err
	}

	trySetConcourseDirInPATH()

	var runner ifrit.Runner

	switch {
	case cmd.Runtime == houdiniRuntime:
		runner, err = cmd.houdiniRunner(logger)
	case cmd.Runtime == containerdRuntime:
		runner, err = cmd.containerdRunner(logger)
	case cmd.Runtime == guardianRuntime:
		runner, err = cmd.guardianRunner(logger)
	default:
		err = fmt.Errorf("unsupported Runtime :%s", cmd.Runtime)
	}

	if err != nil {
		return atc.Worker{}, nil, err
	}

	return worker, runner, nil
}

func trySetConcourseDirInPATH() {
	binDir := concourseCmd.DiscoverAsset("bin")
	if binDir == "" {
		return
	}

	err := os.Setenv("PATH", binDir+":"+os.Getenv("PATH"))
	if err != nil {
		// programming mistake
		panic(fmt.Errorf("failed to set PATH environment variable: %w", err))
	}
}

var ErrNotRoot = errors.New("worker must be run as root")

func (cmd *WorkerCommand) checkRoot() error {
	currentUser, err := user.Current()
	if err != nil {
		return err
	}

	if currentUser.Uid != "0" {
		return ErrNotRoot
	}

	return nil
}

func (cmd *WorkerCommand) dnsProxyRunner(logger lager.Logger) (ifrit.Runner, error) {
	server, err := cmd.Guardian.DNS.Server()
	if err != nil {
		return nil, err
	}

	return ifrit.RunFunc(func(signals <-chan os.Signal, ready chan<- struct{}) error {
		server.NotifyStartedFunc = func() {
			close(ready)
			logger.Info("started")
		}

		serveErr := make(chan error, 1)

		go func() {
			serveErr <- server.ListenAndServe()
		}()

		for {
			select {
			case err := <-serveErr:
				return err
			case <-signals:
				server.Shutdown()
			}
		}
	}), nil
}

func (cmd *WorkerCommand) loadResources(logger lager.Logger) ([]atc.WorkerResourceType, error) {
	var types []atc.WorkerResourceType

	if cmd.ResourceTypes != "" {
		basePath := cmd.ResourceTypes.Path()

		entries, err := ioutil.ReadDir(basePath)
		if err != nil {
			logger.Error("failed-to-read-resources-dir", err)
			return nil, err
		}

		for _, e := range entries {
			meta, err := ioutil.ReadFile(filepath.Join(basePath, e.Name(), "resource_metadata.json"))
			if err != nil {
				logger.Error("failed-to-read-resource-type-metadata", err)
				return nil, err
			}

			var t atc.WorkerResourceType
			err = json.Unmarshal(meta, &t)
			if err != nil {
				logger.Error("failed-to-unmarshal-resource-type-metadata", err)
				return nil, err
			}

			t.Image = filepath.Join(basePath, e.Name(), "rootfs.tgz")

			types = append(types, t)
		}
	}

	return types, nil
}

func (cmd *WorkerCommand) hasFlags(prefix string) bool {
	env := os.Environ()

	for _, envVar := range env {
		if strings.HasPrefix(envVar, prefix) {
			return true
		}
	}

	return false
}

const guardianEnvPrefix = "CONCOURSE_GARDEN_"
const containerdEnvPrefix = "CONCOURSE_CONTAINERD_"

// Checks if runtime specific flags provided match the selected runtime type
func (cmd *WorkerCommand) verifyRuntimeFlags() error {
	switch {
	case cmd.Runtime == houdiniRuntime:
		if cmd.hasFlags(guardianEnvPrefix) || cmd.hasFlags(containerdEnvPrefix) {
			return fmt.Errorf("cannot use %s or %s environment variables with Houdini", guardianEnvPrefix, containerdEnvPrefix)
		}
	case cmd.Runtime == containerdRuntime:
		if cmd.hasFlags(guardianEnvPrefix) {
			return fmt.Errorf("cannot use %s environment variables with Containerd", guardianEnvPrefix)
		}
	case cmd.Runtime == guardianRuntime:
		if cmd.hasFlags(containerdEnvPrefix) {
			return fmt.Errorf("cannot use %s environment variables with Guardian", containerdEnvPrefix)
		}
	default:
		return fmt.Errorf("unsupported Runtime :%s", cmd.Runtime)
	}

	return nil
}
<commit_msg>remove claimed max allowed value<commit_after>package workercmd

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/user"
	"path/filepath"
	"strings"
	"time"

	"code.cloudfoundry.org/lager"
	"github.com/concourse/concourse/atc"
	concourseCmd "github.com/concourse/concourse/cmd"
	"github.com/concourse/flag"
	"github.com/jessevdk/go-flags"
	"github.com/tedsuo/ifrit"
)

type Certs struct {
	Dir string `long:"certs-dir" description:"Directory to use when creating the resource certificates volume."`
}

type RuntimeConfiguration struct {
	Runtime string `long:"runtime" default:"guardian" choice:"guardian" choice:"containerd" choice:"houdini" description:"Runtime to use with the worker. Please note that Houdini is insecure and doesn't run 'tasks' in containers."`
}

type GuardianRuntime struct {
	Bin            string        `long:"bin"        description:"Path to a garden server executable (non-absolute names get resolved from $PATH)."`
	DNS            DNSConfig     `group:"DNS Proxy Configuration" namespace:"dns-proxy"`
	RequestTimeout time.Duration `long:"request-timeout" default:"5m" description:"How long to wait for requests to the Garden server to complete. 0 means no timeout."`

	Config      flag.File `long:"config"     description:"Path to a config file to use for the Garden backend. e.g. 'foo-bar=a,b' for '--foo-bar a --foo-bar b'."`
	BinaryFlags GdnBinaryFlags
}

type ContainerdRuntime struct {
	Config         flag.File     `long:"config"     description:"Path to a config file to use for the Containerd daemon."`
	Bin            string        `long:"bin"        description:"Path to a containerd executable (non-absolute names get resolved from $PATH)."`
	InitBin        string        `long:"init-bin"   default:"/usr/local/concourse/bin/init" description:"Path to an init executable (non-absolute names get resolved from $PATH)."`
	CNIPluginsDir  string        `long:"cni-plugins-dir" default:"/usr/local/concourse/bin" description:"Path to CNI network plugins."`
	RequestTimeout time.Duration `long:"request-timeout" default:"5m" description:"How long to wait for requests to Containerd to complete. 0 means no timeout."`

	Network struct {
		ExternalIP flag.IP `long:"external-ip" description:"IP address to use to reach container's mapped ports. Autodetected if not specified."`
		//TODO can DNSConfig be simplifed to just a bool rather than struct with a bool?
		DNS                DNSConfig `group:"DNS Proxy Configuration" namespace:"dns-proxy"`
		DNSServers         []string  `long:"dns-server" description:"DNS server IP address to use instead of automatically determined servers. Can be specified multiple times."`
		RestrictedNetworks []string  `long:"restricted-network" description:"Network ranges to which traffic from containers will be restricted. Can be specified multiple times."`
		Pool               string    `long:"network-pool" default:"10.80.0.0/16" description:"Network range to use for dynamically allocated container subnets."`
		MTU                int       `long:"mtu" description:"MTU size for container network interfaces. Defaults to the MTU of the interface used for outbound access by the host."`
	} `group:"Container Networking"`

	MaxContainers int `long:"max-containers" default:"250" description:"Max container capacity. 0 means no limit."`
}

const containerdRuntime = "containerd"
const guardianRuntime = "guardian"
const houdiniRuntime = "houdini"

func (cmd WorkerCommand) LessenRequirements(prefix string, command *flags.Command) {
	// configured as work-dir/volumes
	command.FindOptionByLongName(prefix + "baggageclaim-volumes").Required = false
}

// Chooses the appropriate runtime based on CONCOURSE_RUNTIME_TYPE.
// The runtime is represented as a Ifrit runner that must include a Garden Server process. The Garden server exposes API
// endpoints that allow the ATC to make container related requests to the worker.
// The runner may also include additional processes such as the runtime's daemon or a DNS proxy server.
func (cmd *WorkerCommand) gardenServerRunner(logger lager.Logger) (atc.Worker, ifrit.Runner, error) {
	err := cmd.checkRoot()
	if err != nil {
		return atc.Worker{}, nil, err
	}

	err = cmd.verifyRuntimeFlags()
	if err != nil {
		return atc.Worker{}, nil, err
	}

	worker := cmd.Worker.Worker()
	worker.Platform = "linux"

	if cmd.Certs.Dir != "" {
		worker.CertsPath = &cmd.Certs.Dir
	}

	worker.ResourceTypes, err = cmd.loadResources(logger.Session("load-resources"))
	if err != nil {
		return atc.Worker{}, nil, err
	}

	worker.Name, err = cmd.workerName()
	if err != nil {
		return atc.Worker{}, nil, err
	}

	trySetConcourseDirInPATH()

	var runner ifrit.Runner

	switch {
	case cmd.Runtime == houdiniRuntime:
		runner, err = cmd.houdiniRunner(logger)
	case cmd.Runtime == containerdRuntime:
		runner, err = cmd.containerdRunner(logger)
	case cmd.Runtime == guardianRuntime:
		runner, err = cmd.guardianRunner(logger)
	default:
		err = fmt.Errorf("unsupported Runtime :%s", cmd.Runtime)
	}

	if err != nil {
		return atc.Worker{}, nil, err
	}

	return worker, runner, nil
}

func trySetConcourseDirInPATH() {
	binDir := concourseCmd.DiscoverAsset("bin")
	if binDir == "" {
		return
	}

	err := os.Setenv("PATH", binDir+":"+os.Getenv("PATH"))
	if err != nil {
		// programming mistake
		panic(fmt.Errorf("failed to set PATH environment variable: %w", err))
	}
}

var ErrNotRoot = errors.New("worker must be run as root")

func (cmd *WorkerCommand) checkRoot() error {
	currentUser, err := user.Current()
	if err != nil {
		return err
	}

	if currentUser.Uid != "0" {
		return ErrNotRoot
	}

	return nil
}

func (cmd *WorkerCommand) dnsProxyRunner(logger lager.Logger) (ifrit.Runner, error) {
	server, err := cmd.Guardian.DNS.Server()
	if err != nil {
		return nil, err
	}

	return ifrit.RunFunc(func(signals <-chan os.Signal, ready chan<- struct{}) error {
		server.NotifyStartedFunc = func() {
			close(ready)
			logger.Info("started")
		}

		serveErr := make(chan error, 1)

		go func() {
			serveErr <- server.ListenAndServe()
		}()

		for {
			select {
			case err := <-serveErr:
				return err
			case <-signals:
				server.Shutdown()
			}
		}
	}), nil
}

func (cmd *WorkerCommand) loadResources(logger lager.Logger) ([]atc.WorkerResourceType, error) {
	var types []atc.WorkerResourceType

	if cmd.ResourceTypes != "" {
		basePath := cmd.ResourceTypes.Path()

		entries, err := ioutil.ReadDir(basePath)
		if err != nil {
			logger.Error("failed-to-read-resources-dir", err)
			return nil, err
		}

		for _, e := range entries {
			meta, err := ioutil.ReadFile(filepath.Join(basePath, e.Name(), "resource_metadata.json"))
			if err != nil {
				logger.Error("failed-to-read-resource-type-metadata", err)
				return nil, err
			}

			var t atc.WorkerResourceType
			err = json.Unmarshal(meta, &t)
			if err != nil {
				logger.Error("failed-to-unmarshal-resource-type-metadata", err)
				return nil, err
			}

			t.Image = filepath.Join(basePath, e.Name(), "rootfs.tgz")

			types = append(types, t)
		}
	}

	return types, nil
}

func (cmd *WorkerCommand) hasFlags(prefix string) bool {
	env := os.Environ()

	for _, envVar := range env {
		if strings.HasPrefix(envVar, prefix) {
			return true
		}
	}

	return false
}

const guardianEnvPrefix = "CONCOURSE_GARDEN_"
const containerdEnvPrefix = "CONCOURSE_CONTAINERD_"

// Checks if runtime specific flags provided match the selected runtime type
func (cmd *WorkerCommand) verifyRuntimeFlags() error {
	switch {
	case cmd.Runtime == houdiniRuntime:
		if cmd.hasFlags(guardianEnvPrefix) || cmd.hasFlags(containerdEnvPrefix) {
			return fmt.Errorf("cannot use %s or %s environment variables with Houdini", guardianEnvPrefix, containerdEnvPrefix)
		}
	case cmd.Runtime == containerdRuntime:
		if cmd.hasFlags(guardianEnvPrefix) {
			return fmt.Errorf("cannot use %s environment variables with Containerd", guardianEnvPrefix)
		}
	case cmd.Runtime == guardianRuntime:
		if cmd.hasFlags(containerdEnvPrefix) {
			return fmt.Errorf("cannot use %s environment variables with Guardian", containerdEnvPrefix)
		}
	default:
		return fmt.Errorf("unsupported Runtime :%s", cmd.Runtime)
	}

	return nil
}
<|endoftext|>


<commit_before>package capn

import (
	"bytes"
	"encoding/binary"
	"errors"
	"io"
	"math"
)

var (
	errBufferCall     = errors.New("capn: can't call on a memory buffer")
	ErrInvalidSegment = errors.New("capn: invalid segment id")
	ErrTooMuchData    = errors.New("capn: too much data in stream")
)

type buffer struct {
	Segment
	capTable
}

// NewBuffer creates an expanding single segment buffer. Creating new objects
// will expand the buffer. Data can be nil (or length 0 with some capacity) if
// creating a new session. If parsing an existing segment than data should be
// the segment contents and will not be copied.
func NewBuffer(data []byte) *Segment {
	if uint64(len(data)) > uint64(math.MaxUint32) {
		return nil
	}

	b := new(buffer)
	b.Message = b
	b.Data = data
	return &b.Segment
}

func (b *buffer) NewSegment(minsz int) (*Segment, error) {
	if minsz < 4096 {
		minsz = 4096
	}
	if uint64(len(b.Data)) > uint64(math.MaxUint32)-uint64(minsz) {
		return nil, ErrOverlarge
	}
	b.Data = append(b.Data, make([]byte, minsz)...)
	b.Data = b.Data[:len(b.Data)-minsz]
	return &b.Segment, nil
}

func (b *buffer) Lookup(segid uint32) (*Segment, error) {
	if segid == 0 {
		return &b.Segment, nil
	} else {
		return nil, ErrInvalidSegment
	}
}

type multiBuffer struct {
	segments []*Segment
	capTable
}

// NewmultiBuffer creates a new multi segment message. Creating new objects
// will try and reuse the buffers available, but will create new ones if there
// is insufficient capacity. When parsing an existing message data should be
// the list of segments. The data buffers will not be copied.
func NewmultiBuffer(data [][]byte) *Segment {
	m := &multiBuffer{
		segments: make([]*Segment, len(data)),
	}
	for i, d := range data {
		m.segments[i] = &Segment{m, d, uint32(i), false}
	}
	if len(data) > 0 {
		return m.segments[0]
	}
	return &Segment{m, nil, 0xFFFFFFFF, false}
}

var (
	MaxSegmentNumber = 1024
	MaxTotalSize     = 1024 * 1024 * 1024
)

func (m *multiBuffer) NewSegment(minsz int) (*Segment, error) {
	for _, s := range m.segments {
		if len(s.Data)+minsz <= cap(s.Data) {
			return s, nil
		}
	}

	if minsz < 4096 {
		minsz = 4096
	}
	s := &Segment{m, make([]byte, 0, minsz), uint32(len(m.segments)), false}
	m.segments = append(m.segments, s)
	return s, nil
}

func (m *multiBuffer) Lookup(segid uint32) (*Segment, error) {
	if uint(segid) < uint(len(m.segments)) {
		return m.segments[segid], nil
	} else {
		return nil, ErrInvalidSegment
	}
}

// ReadFromStream reads a non-packed serialized stream from r. buf is used to
// buffer the read contents, can be nil, and is provided so that the buffer
// can be reused between messages. The returned segment is the first segment
// read, which contains the root pointer.
func ReadFromStream(r io.Reader, buf *bytes.Buffer) (*Segment, error) {
	if buf == nil {
		buf = new(bytes.Buffer)
	} else {
		buf.Reset()
	}

	if _, err := io.CopyN(buf, r, 4); err != nil {
		return nil, err
	}

	if binary.LittleEndian.Uint32(buf.Bytes()[:]) >= uint32(MaxSegmentNumber) {
		return nil, ErrTooMuchData
	}

	segnum := int(binary.LittleEndian.Uint32(buf.Bytes()[:]) + 1)
	hdrsz := 8*(segnum/2) + 4

	if _, err := io.CopyN(buf, r, int64(hdrsz)); err != nil {
		return nil, err
	}

	total := 0
	for i := 0; i < segnum; i++ {
		sz := binary.LittleEndian.Uint32(buf.Bytes()[4*i+4:])
		if uint64(total)+uint64(sz)*8 > uint64(MaxTotalSize) {
			return nil, ErrTooMuchData
		}
		total += int(sz) * 8
	}

	if _, err := io.CopyN(buf, r, int64(total)); err != nil {
		return nil, err
	}

	hdrv := buf.Bytes()[4 : hdrsz+4]
	datav := buf.Bytes()[hdrsz+4:]

	if segnum == 1 {
		sz := int(binary.LittleEndian.Uint32(hdrv)) * 8
		return NewBuffer(datav[:sz]), nil
	}

	m := &multiBuffer{segments: make([]*Segment, segnum)}
	for i := 0; i < segnum; i++ {
		sz := int(binary.LittleEndian.Uint32(hdrv[4*i:])) * 8
		m.segments[i] = &Segment{m, datav[:sz], uint32(i), false}
		datav = datav[sz:]
	}

	return m.segments[0], nil
}

// ReadFromMemoryZeroCopy: like ReadFromStream, but reads a non-packed
// serialized stream that already resides in memory in the argument data.
// The returned segment is the first segment read, which contains
// the root pointer. The returned bytesRead says how many bytes were
// consumed from data in making seg. The caller should advance the
// data slice by doing data = data[bytesRead:] between successive calls
// to ReadFromMemoryZeroCopy().
func ReadFromMemoryZeroCopy(data []byte) (seg *Segment, bytesRead int64, err error) {

	if len(data) < 4 {
		return nil, 0, io.EOF
	}

	if binary.LittleEndian.Uint32(data[0:4]) >= uint32(MaxSegmentNumber) {
		return nil, 0, ErrTooMuchData
	}

	segnum := int(binary.LittleEndian.Uint32(data[0:4]) + 1)
	hdrsz := 8*(segnum/2) + 4

	b := data[0:(hdrsz + 4)]

	total := 0
	for i := 0; i < segnum; i++ {
		sz := binary.LittleEndian.Uint32(b[4*i+4:])
		if uint64(total)+uint64(sz)*8 > uint64(MaxTotalSize) {
			return nil, 0, ErrTooMuchData
		}
		total += int(sz) * 8
	}
	if total == 0 {
		return nil, 0, io.EOF
	}

	hdrv := data[4:(hdrsz + 4)]
	datav := data[hdrsz+4:]
	m := &multiBuffer{segments: make([]*Segment, segnum)}
	for i := 0; i < segnum; i++ {
		sz := int(binary.LittleEndian.Uint32(hdrv[4*i:])) * 8
		m.segments[i] = &Segment{m, datav[:sz], uint32(i), false}
		datav = datav[sz:]
	}

	return m.segments[0], int64(4 + hdrsz + total), nil
}

// WriteTo writes the message that the segment is part of to the
// provided stream in serialized form.
func (s *Segment) WriteTo(w io.Writer) (int64, error) {
	segnum := uint32(1)
	for {
		if seg, _ := s.Message.Lookup(segnum); seg == nil {
			break
		}
		segnum++
	}

	hdrv := make([]uint8, 8*(segnum/2)+8)
	binary.LittleEndian.PutUint32(hdrv, segnum-1)
	for i := uint32(0); i < segnum; i++ {
		seg, _ := s.Message.Lookup(i)
		binary.LittleEndian.PutUint32(hdrv[4*i+4:], uint32(len(seg.Data)/8))
	}

	if n, err := w.Write(hdrv); err != nil {
		return int64(n), err
	}
	written := int64(len(hdrv))

	for i := uint32(0); i < segnum; i++ {
		seg, _ := s.Message.Lookup(i)
		if n, err := w.Write(seg.Data); err != nil {
			return written + int64(n), err
		} else {
			written += int64(n)
		}
	}

	return written, nil
}

type capTable []Client

func (tab capTable) CapTable() []Client {
	return []Client(tab)
}

func (tab *capTable) AddCap(c Client) uint32 {
	n := uint32(len(*tab))
	*tab = append(*tab, c)
	return n
}
<commit_msg>s/NewmultiBuffer/NewMultiBuffer/<commit_after>package capn

import (
	"bytes"
	"encoding/binary"
	"errors"
	"io"
	"math"
)

var (
	errBufferCall     = errors.New("capn: can't call on a memory buffer")
	ErrInvalidSegment = errors.New("capn: invalid segment id")
	ErrTooMuchData    = errors.New("capn: too much data in stream")
)

type buffer struct {
	Segment
	capTable
}

// NewBuffer creates an expanding single segment buffer. Creating new objects
// will expand the buffer. Data can be nil (or length 0 with some capacity) if
// creating a new session. If parsing an existing segment then data should be
// the segment contents and will not be copied.
func NewBuffer(data []byte) *Segment {
	if uint64(len(data)) > uint64(math.MaxUint32) {
		return nil
	}

	b := new(buffer)
	b.Message = b
	b.Data = data
	return &b.Segment
}

func (b *buffer) NewSegment(minsz int) (*Segment, error) {
	if minsz < 4096 {
		minsz = 4096
	}
	if uint64(len(b.Data)) > uint64(math.MaxUint32)-uint64(minsz) {
		return nil, ErrOverlarge
	}
	b.Data = append(b.Data, make([]byte, minsz)...)
	b.Data = b.Data[:len(b.Data)-minsz]
	return &b.Segment, nil
}

func (b *buffer) Lookup(segid uint32) (*Segment, error) {
	if segid == 0 {
		return &b.Segment, nil
	} else {
		return nil, ErrInvalidSegment
	}
}

type multiBuffer struct {
	segments []*Segment
	capTable
}

// NewMultiBuffer creates a new multi segment message. Creating new objects
// will try and reuse the buffers available, but will create new ones if there
// is insufficient capacity. When parsing an existing message data should be
// the list of segments. The data buffers will not be copied.
func NewMultiBuffer(data [][]byte) *Segment {
	m := &multiBuffer{
		segments: make([]*Segment, len(data)),
	}
	for i, d := range data {
		m.segments[i] = &Segment{m, d, uint32(i), false}
	}
	if len(data) > 0 {
		return m.segments[0]
	}
	return &Segment{m, nil, 0xFFFFFFFF, false}
}

var (
	MaxSegmentNumber = 1024
	MaxTotalSize     = 1024 * 1024 * 1024
)

func (m *multiBuffer) NewSegment(minsz int) (*Segment, error) {
	for _, s := range m.segments {
		if len(s.Data)+minsz <= cap(s.Data) {
			return s, nil
		}
	}

	if minsz < 4096 {
		minsz = 4096
	}
	s := &Segment{m, make([]byte, 0, minsz), uint32(len(m.segments)), false}
	m.segments = append(m.segments, s)
	return s, nil
}

func (m *multiBuffer) Lookup(segid uint32) (*Segment, error) {
	if uint(segid) < uint(len(m.segments)) {
		return m.segments[segid], nil
	} else {
		return nil, ErrInvalidSegment
	}
}

// ReadFromStream reads a non-packed serialized stream from r. buf is used to
// buffer the read contents, can be nil, and is provided so that the buffer
// can be reused between messages. The returned segment is the first segment
// read, which contains the root pointer.
func ReadFromStream(r io.Reader, buf *bytes.Buffer) (*Segment, error) {
	if buf == nil {
		buf = new(bytes.Buffer)
	} else {
		buf.Reset()
	}

	if _, err := io.CopyN(buf, r, 4); err != nil {
		return nil, err
	}

	if binary.LittleEndian.Uint32(buf.Bytes()[:]) >= uint32(MaxSegmentNumber) {
		return nil, ErrTooMuchData
	}

	segnum := int(binary.LittleEndian.Uint32(buf.Bytes()[:]) + 1)
	hdrsz := 8*(segnum/2) + 4

	if _, err := io.CopyN(buf, r, int64(hdrsz)); err != nil {
		return nil, err
	}

	total := 0
	for i := 0; i < segnum; i++ {
		sz := binary.LittleEndian.Uint32(buf.Bytes()[4*i+4:])
		if uint64(total)+uint64(sz)*8 > uint64(MaxTotalSize) {
			return nil, ErrTooMuchData
		}
		total += int(sz) * 8
	}

	if _, err := io.CopyN(buf, r, int64(total)); err != nil {
		return nil, err
	}

	hdrv := buf.Bytes()[4 : hdrsz+4]
	datav := buf.Bytes()[hdrsz+4:]

	if segnum == 1 {
		sz := int(binary.LittleEndian.Uint32(hdrv)) * 8
		return NewBuffer(datav[:sz]), nil
	}

	m := &multiBuffer{segments: make([]*Segment, segnum)}
	for i := 0; i < segnum; i++ {
		sz := int(binary.LittleEndian.Uint32(hdrv[4*i:])) * 8
		m.segments[i] = &Segment{m, datav[:sz], uint32(i), false}
		datav = datav[sz:]
	}

	return m.segments[0], nil
}

// ReadFromMemoryZeroCopy: like ReadFromStream, but reads a non-packed
// serialized stream that already resides in memory in the argument data.
// The returned segment is the first segment read, which contains
// the root pointer. The returned bytesRead says how many bytes were
// consumed from data in making seg. The caller should advance the
// data slice by doing data = data[bytesRead:] between successive calls
// to ReadFromMemoryZeroCopy().
func ReadFromMemoryZeroCopy(data []byte) (seg *Segment, bytesRead int64, err error) {

	if len(data) < 4 {
		return nil, 0, io.EOF
	}

	if binary.LittleEndian.Uint32(data[0:4]) >= uint32(MaxSegmentNumber) {
		return nil, 0, ErrTooMuchData
	}

	segnum := int(binary.LittleEndian.Uint32(data[0:4]) + 1)
	hdrsz := 8*(segnum/2) + 4

	b := data[0:(hdrsz + 4)]

	total := 0
	for i := 0; i < segnum; i++ {
		sz := binary.LittleEndian.Uint32(b[4*i+4:])
		if uint64(total)+uint64(sz)*8 > uint64(MaxTotalSize) {
			return nil, 0, ErrTooMuchData
		}
		total += int(sz) * 8
	}
	if total == 0 {
		return nil, 0, io.EOF
	}

	hdrv := data[4:(hdrsz + 4)]
	datav := data[hdrsz+4:]
	m := &multiBuffer{segments: make([]*Segment, segnum)}
	for i := 0; i < segnum; i++ {
		sz := int(binary.LittleEndian.Uint32(hdrv[4*i:])) * 8
		m.segments[i] = &Segment{m, datav[:sz], uint32(i), false}
		datav = datav[sz:]
	}

	return m.segments[0], int64(4 + hdrsz + total), nil
}

// WriteTo writes the message that the segment is part of to the
// provided stream in serialized form.
func (s *Segment) WriteTo(w io.Writer) (int64, error) {
	segnum := uint32(1)
	for {
		if seg, _ := s.Message.Lookup(segnum); seg == nil {
			break
		}
		segnum++
	}

	hdrv := make([]uint8, 8*(segnum/2)+8)
	binary.LittleEndian.PutUint32(hdrv, segnum-1)
	for i := uint32(0); i < segnum; i++ {
		seg, _ := s.Message.Lookup(i)
		binary.LittleEndian.PutUint32(hdrv[4*i+4:], uint32(len(seg.Data)/8))
	}

	if n, err := w.Write(hdrv); err != nil {
		return int64(n), err
	}
	written := int64(len(hdrv))

	for i := uint32(0); i < segnum; i++ {
		seg, _ := s.Message.Lookup(i)
		if n, err := w.Write(seg.Data); err != nil {
			return written + int64(n), err
		} else {
			written += int64(n)
		}
	}

	return written, nil
}

type capTable []Client

func (tab capTable) CapTable() []Client {
	return []Client(tab)
}

func (tab *capTable) AddCap(c Client) uint32 {
	n := uint32(len(*tab))
	*tab = append(*tab, c)
	return n
}
<|endoftext|>


<commit_before>package vm

import (
	"fmt"
	"github.com/goby-lang/goby/bytecode"
	"github.com/goby-lang/goby/parser"
	"io/ioutil"
	"path"
	"path/filepath"
	"strings"
)

var stackTrace int

type isIndexTable struct {
	Data map[string]int
}

func newISIndexTable() *isIndexTable {
	return &isIndexTable{Data: make(map[string]int)}
}

type isTable map[string][]*instructionSet

type filename string

type errorMessage string

var standardLibraries = map[string]func(*VM){
	"file":              initializeFileClass,
	"net/http":          initializeHTTPClass,
	"net/simple_server": initializeSimpleServerClass,
	"uri":               initializeURIClass,
}

// VM represents a stack based virtual machine.
type VM struct {
	mainThread *thread
	// a map holds pointers of constants
	constants map[string]*Pointer
	// a map holds different types of label tables
	isTables map[labelType]isTable
	// method instruction set table
	methodISIndexTables map[filename]*isIndexTable
	// class instruction set table
	classISIndexTables map[filename]*isIndexTable
	// block instruction set table
	blockTables map[filename]map[string]*instructionSet
	// fileDir indicates executed file's directory
	fileDir string
	// args are command line arguments
	args []string
	// projectRoot is goby root's absolute path, which is $GOROOT/src/github.com/goby-lang/goby
	projectRoot string

	threadCount int
}

// New initializes a vm to initialize state and returns it.
func New(fileDir string, args []string) *VM {
	vm := &VM{args: args}
	vm.mainThread = vm.newThread()

	vm.initConstants()
	vm.methodISIndexTables = map[filename]*isIndexTable{
		filename(fileDir): newISIndexTable(),
	}
	vm.classISIndexTables = map[filename]*isIndexTable{
		filename(fileDir): newISIndexTable(),
	}
	vm.blockTables = make(map[filename]map[string]*instructionSet)
	vm.isTables = map[labelType]isTable{
		bytecode.LabelDef:      make(isTable),
		bytecode.LabelDefClass: make(isTable),
	}
	vm.fileDir = fileDir

	p, _ := filepath.Abs("../")

	if !strings.HasSuffix(p, "goby") {
		vm.projectRoot = path.Join(p, "goby")
	} else {
		vm.projectRoot = p
	}

	return vm
}

func (vm *VM) newThread() *thread {
	s := &stack{}
	cfs := &callFrameStack{callFrames: []*callFrame{}}
	t := &thread{stack: s, callFrameStack: cfs, sp: 0, cfp: 0}
	s.thread = t
	cfs.thread = t
	t.vm = vm
	vm.threadCount++
	return t
}

// ExecBytecodes accepts a sequence of bytecodes and use vm to evaluate them.
func (vm *VM) ExecBytecodes(bytecodes, fn string) {
	filename := filename(fn)
	p := newBytecodeParser(filename)
	p.vm = vm
	p.parseBytecode(bytecodes)

	// Keep update label table after parsed new files.
	// TODO: Find more efficient way to do this.
	for labelType, table := range p.labelTable {
		for labelName, is := range table {
			vm.isTables[labelType][labelName] = is
		}
	}

	vm.blockTables[p.filename] = p.blockTable
	vm.classISIndexTables[filename] = newISIndexTable()
	vm.methodISIndexTables[filename] = newISIndexTable()

	defer func() {
		if p := recover(); p != nil {
			switch p.(type) {
			case errorMessage:
				return
			default:
				panic(p)
			}
		}
	}()

	cf := newCallFrame(p.program)
	cf.self = mainObj
	vm.mainThread.callFrameStack.push(cf)
	vm.startFromTopFrame()
}

// GetExecResult returns stack's top most value. Normally it's used in tests.
func (vm *VM) GetExecResult() Object {
	return vm.mainThread.stack.top().Target
}

func (vm *VM) initConstants() {
	vm.constants = make(map[string]*Pointer)
	constants := make(map[string]*Pointer)

	builtInClasses := []Class{
		integerClass,
		stringClass,
		booleanClass,
		nullClass,
		arrayClass,
		hashClass,
		classClass,
		methodClass,
	}

	args := []Object{}

	for _, arg := range vm.args {
		args = append(args, initializeString(arg))
	}

	for _, c := range builtInClasses {
		p := &Pointer{Target: c}
		constants[c.ReturnName()] = p
	}

	constants["ARGV"] = &Pointer{Target: initializeArray(args)}
	objectClass.constants = constants
	vm.constants["Object"] = &Pointer{objectClass}
}

// Start evaluation from top most call frame
func (vm *VM) startFromTopFrame() {
	vm.mainThread.startFromTopFrame()
}

func (vm *VM) currentFilePath() string {
	return string(vm.mainThread.callFrameStack.top().instructionSet.filename)
}

func (vm *VM) printDebugInfo(i *instruction) {
	fmt.Println(i.inspect())
}

func (vm *VM) getBlock(name string, filename filename) *instructionSet {
	// The "name" here is actually an index from label
	// for example <Block:1>'s name is "1"
	is, ok := vm.blockTables[filename][name]

	if !ok {
		panic(fmt.Sprintf("Can't find block %s", name))
	}

	return is
}

func (vm *VM) getMethodIS(name string, filename filename) (*instructionSet, bool) {
	iss, ok := vm.isTables[bytecode.LabelDef][name]

	if !ok {
		return nil, false
	}

	is := iss[vm.methodISIndexTables[filename].Data[name]]

	vm.methodISIndexTables[filename].Data[name]++
	return is, ok
}

func (vm *VM) getClassIS(name string, filename filename) *instructionSet {
	iss, ok := vm.isTables[bytecode.LabelDefClass][name]

	if !ok {
		panic(fmt.Sprintf("Can't find class %s's instructions", name))
	}

	is := iss[vm.classISIndexTables[filename].Data[name]]

	vm.classISIndexTables[filename].Data[name]++
	return is
}

func (vm *VM) loadConstant(name string, isModule bool) *RClass {
	var c *RClass
	var ptr *Pointer

	ptr = objectClass.constants[name]

	if ptr == nil {
		c = initializeClass(name, isModule)
		objectClass.constants[name] = &Pointer{Target: c}
	} else {
		c = ptr.Target.(*RClass)
	}

	return c
}

func (vm *VM) lookupConstant(cf *callFrame, constName string) *Pointer {
	var constant *Pointer
	var namespace Class
	var hasNamespace bool

	top := vm.mainThread.stack.top()

	if top == nil {
		hasNamespace = false
	} else {
		namespace, hasNamespace = top.Target.(Class)
	}

	if hasNamespace {
		if namespace != cf.self {
			vm.mainThread.stack.pop()
		}

		constant = namespace.lookupConstant(constName, true)

		if constant != nil {
			return constant
		}
	}

	switch s := cf.self.(type) {
	case Class:
		constant = s.lookupConstant(constName, true)
		if constant != nil {
			return constant
		}
	default:
		c := s.returnClass()

		constant = c.lookupConstant(constName, true)
		if constant != nil {
			return constant
		}
	}

	constant = vm.constants[constName]
	return constant
}

func (vm *VM) execGobyLib(libName string) {
	libPath := path.Join(vm.projectRoot, "lib", libName)
	file, err := ioutil.ReadFile(libPath)

	if err != nil {
		vm.mainThread.returnError(err.Error())
	}

	vm.execRequiredFile(libPath, file)
}

func (vm *VM) execRequiredFile(filepath string, file []byte) {
	program := parser.BuildAST(file)
	g := bytecode.NewGenerator(program)
	bytecodes := g.GenerateByteCode(program)

	oldMethodTable := isTable{}
	oldClassTable := isTable{}

	// Copy current file's instruction sets.
	for name, is := range vm.isTables[bytecode.LabelDef] {
		oldMethodTable[name] = is
	}

	for name, is := range vm.isTables[bytecode.LabelDefClass] {
		oldClassTable[name] = is
	}

	// This creates new execution environments for required file, including new instruction set table.
	// So we need to copy old instruction sets and restore them later, otherwise current program's instruction set would be overwrite.
	vm.ExecBytecodes(bytecodes, filepath)

	// Restore instruction sets.
	vm.isTables[bytecode.LabelDef] = oldMethodTable
	vm.isTables[bytecode.LabelDefClass] = oldClassTable
}

func newError(format string, args ...interface{}) *Error {
	return &Error{Message: fmt.Sprintf(format, args...)}
}
<commit_msg>Remove thread count since it's for debugging.<commit_after>package vm

import (
	"fmt"
	"github.com/goby-lang/goby/bytecode"
	"github.com/goby-lang/goby/parser"
	"io/ioutil"
	"path"
	"path/filepath"
	"strings"
)

var stackTrace int

type isIndexTable struct {
	Data map[string]int
}

func newISIndexTable() *isIndexTable {
	return &isIndexTable{Data: make(map[string]int)}
}

type isTable map[string][]*instructionSet

type filename string

type errorMessage string

var standardLibraries = map[string]func(*VM){
	"file":              initializeFileClass,
	"net/http":          initializeHTTPClass,
	"net/simple_server": initializeSimpleServerClass,
	"uri":               initializeURIClass,
}

// VM represents a stack based virtual machine.
type VM struct {
	mainThread *thread
	// a map holds pointers of constants
	constants map[string]*Pointer
	// a map holds different types of label tables
	isTables map[labelType]isTable
	// method instruction set table
	methodISIndexTables map[filename]*isIndexTable
	// class instruction set table
	classISIndexTables map[filename]*isIndexTable
	// block instruction set table
	blockTables map[filename]map[string]*instructionSet
	// fileDir indicates executed file's directory
	fileDir string
	// args are command line arguments
	args []string
	// projectRoot is goby root's absolute path, which is $GOROOT/src/github.com/goby-lang/goby
	projectRoot string
}

// New initializes a vm to initialize state and returns it.
func New(fileDir string, args []string) *VM {
	vm := &VM{args: args}
	vm.mainThread = vm.newThread()

	vm.initConstants()
	vm.methodISIndexTables = map[filename]*isIndexTable{
		filename(fileDir): newISIndexTable(),
	}
	vm.classISIndexTables = map[filename]*isIndexTable{
		filename(fileDir): newISIndexTable(),
	}
	vm.blockTables = make(map[filename]map[string]*instructionSet)
	vm.isTables = map[labelType]isTable{
		bytecode.LabelDef:      make(isTable),
		bytecode.LabelDefClass: make(isTable),
	}
	vm.fileDir = fileDir

	p, _ := filepath.Abs("../")

	if !strings.HasSuffix(p, "goby") {
		vm.projectRoot = path.Join(p, "goby")
	} else {
		vm.projectRoot = p
	}

	return vm
}

func (vm *VM) newThread() *thread {
	s := &stack{}
	cfs := &callFrameStack{callFrames: []*callFrame{}}
	t := &thread{stack: s, callFrameStack: cfs, sp: 0, cfp: 0}
	s.thread = t
	cfs.thread = t
	t.vm = vm
	return t
}

// ExecBytecodes accepts a sequence of bytecodes and use vm to evaluate them.
func (vm *VM) ExecBytecodes(bytecodes, fn string) {
	filename := filename(fn)
	p := newBytecodeParser(filename)
	p.vm = vm
	p.parseBytecode(bytecodes)

	// Keep update label table after parsed new files.
	// TODO: Find more efficient way to do this.
	for labelType, table := range p.labelTable {
		for labelName, is := range table {
			vm.isTables[labelType][labelName] = is
		}
	}

	vm.blockTables[p.filename] = p.blockTable
	vm.classISIndexTables[filename] = newISIndexTable()
	vm.methodISIndexTables[filename] = newISIndexTable()

	defer func() {
		if p := recover(); p != nil {
			switch p.(type) {
			case errorMessage:
				return
			default:
				panic(p)
			}
		}
	}()

	cf := newCallFrame(p.program)
	cf.self = mainObj
	vm.mainThread.callFrameStack.push(cf)
	vm.startFromTopFrame()
}

// GetExecResult returns stack's top most value. Normally it's used in tests.
func (vm *VM) GetExecResult() Object {
	return vm.mainThread.stack.top().Target
}

func (vm *VM) initConstants() {
	vm.constants = make(map[string]*Pointer)
	constants := make(map[string]*Pointer)

	builtInClasses := []Class{
		integerClass,
		stringClass,
		booleanClass,
		nullClass,
		arrayClass,
		hashClass,
		classClass,
		methodClass,
	}

	args := []Object{}

	for _, arg := range vm.args {
		args = append(args, initializeString(arg))
	}

	for _, c := range builtInClasses {
		p := &Pointer{Target: c}
		constants[c.ReturnName()] = p
	}

	constants["ARGV"] = &Pointer{Target: initializeArray(args)}
	objectClass.constants = constants
	vm.constants["Object"] = &Pointer{objectClass}
}

// Start evaluation from top most call frame
func (vm *VM) startFromTopFrame() {
	vm.mainThread.startFromTopFrame()
}

func (vm *VM) currentFilePath() string {
	return string(vm.mainThread.callFrameStack.top().instructionSet.filename)
}

func (vm *VM) printDebugInfo(i *instruction) {
	fmt.Println(i.inspect())
}

func (vm *VM) getBlock(name string, filename filename) *instructionSet {
	// The "name" here is actually an index from label
	// for example <Block:1>'s name is "1"
	is, ok := vm.blockTables[filename][name]

	if !ok {
		panic(fmt.Sprintf("Can't find block %s", name))
	}

	return is
}

func (vm *VM) getMethodIS(name string, filename filename) (*instructionSet, bool) {
	iss, ok := vm.isTables[bytecode.LabelDef][name]

	if !ok {
		return nil, false
	}

	is := iss[vm.methodISIndexTables[filename].Data[name]]

	vm.methodISIndexTables[filename].Data[name]++
	return is, ok
}

func (vm *VM) getClassIS(name string, filename filename) *instructionSet {
	iss, ok := vm.isTables[bytecode.LabelDefClass][name]

	if !ok {
		panic(fmt.Sprintf("Can't find class %s's instructions", name))
	}

	is := iss[vm.classISIndexTables[filename].Data[name]]

	vm.classISIndexTables[filename].Data[name]++
	return is
}

func (vm *VM) loadConstant(name string, isModule bool) *RClass {
	var c *RClass
	var ptr *Pointer

	ptr = objectClass.constants[name]

	if ptr == nil {
		c = initializeClass(name, isModule)
		objectClass.constants[name] = &Pointer{Target: c}
	} else {
		c = ptr.Target.(*RClass)
	}

	return c
}

func (vm *VM) lookupConstant(cf *callFrame, constName string) *Pointer {
	var constant *Pointer
	var namespace Class
	var hasNamespace bool

	top := vm.mainThread.stack.top()

	if top == nil {
		hasNamespace = false
	} else {
		namespace, hasNamespace = top.Target.(Class)
	}

	if hasNamespace {
		if namespace != cf.self {
			vm.mainThread.stack.pop()
		}

		constant = namespace.lookupConstant(constName, true)

		if constant != nil {
			return constant
		}
	}

	switch s := cf.self.(type) {
	case Class:
		constant = s.lookupConstant(constName, true)
		if constant != nil {
			return constant
		}
	default:
		c := s.returnClass()

		constant = c.lookupConstant(constName, true)
		if constant != nil {
			return constant
		}
	}

	constant = vm.constants[constName]
	return constant
}

func (vm *VM) execGobyLib(libName string) {
	libPath := path.Join(vm.projectRoot, "lib", libName)
	file, err := ioutil.ReadFile(libPath)

	if err != nil {
		vm.mainThread.returnError(err.Error())
	}

	vm.execRequiredFile(libPath, file)
}

func (vm *VM) execRequiredFile(filepath string, file []byte) {
	program := parser.BuildAST(file)
	g := bytecode.NewGenerator(program)
	bytecodes := g.GenerateByteCode(program)

	oldMethodTable := isTable{}
	oldClassTable := isTable{}

	// Copy current file's instruction sets.
	for name, is := range vm.isTables[bytecode.LabelDef] {
		oldMethodTable[name] = is
	}

	for name, is := range vm.isTables[bytecode.LabelDefClass] {
		oldClassTable[name] = is
	}

	// This creates new execution environments for required file, including new instruction set table.
	// So we need to copy old instruction sets and restore them later, otherwise current program's instruction set would be overwrite.
	vm.ExecBytecodes(bytecodes, filepath)

	// Restore instruction sets.
	vm.isTables[bytecode.LabelDef] = oldMethodTable
	vm.isTables[bytecode.LabelDefClass] = oldClassTable
}

func newError(format string, args ...interface{}) *Error {
	return &Error{Message: fmt.Sprintf(format, args...)}
}
<|endoftext|>