<commit_before>package config

import (
	"errors"
	"regexp"
	"strconv"
	"strings"

	"github.com/zyedidia/tcell/v2"
)

// DefStyle is Micro's default style
var DefStyle tcell.Style = tcell.StyleDefault

// Colorscheme is the current colorscheme
var Colorscheme map[string]tcell.Style

// GetColor takes in a syntax group and returns the colorscheme's style for that group
func GetColor(color string) tcell.Style {
	st := DefStyle
	if color == "" {
		return st
	}
	groups := strings.Split(color, ".")
	if len(groups) > 1 {
		curGroup := ""
		for i, g := range groups {
			if i != 0 {
				curGroup += "."
			}
			curGroup += g
			if style, ok := Colorscheme[curGroup]; ok {
				st = style
			}
		}
	} else if style, ok := Colorscheme[color]; ok {
		st = style
	} else {
		st = StringToStyle(color)
	}

	return st
}

// ColorschemeExists checks if a given colorscheme exists
func ColorschemeExists(colorschemeName string) bool {
	return FindRuntimeFile(RTColorscheme, colorschemeName) != nil
}

// InitColorscheme picks and initializes the colorscheme when micro starts
func InitColorscheme() error {
	Colorscheme = make(map[string]tcell.Style)
	DefStyle = tcell.StyleDefault

	return LoadDefaultColorscheme()
}

// LoadDefaultColorscheme loads the default colorscheme from $(ConfigDir)/colorschemes
func LoadDefaultColorscheme() error {
	return LoadColorscheme(GlobalSettings["colorscheme"].(string))
}

// LoadColorscheme loads the given colorscheme from a directory
func LoadColorscheme(colorschemeName string) error {
	file := FindRuntimeFile(RTColorscheme, colorschemeName)
	if file == nil {
		return errors.New(colorschemeName + " is not a valid colorscheme")
	}
	if data, err := file.Data(); err != nil {
		return errors.New("Error loading colorscheme: " + err.Error())
	} else {
		Colorscheme, err = ParseColorscheme(string(data))
		if err != nil {
			return err
		}
	}
	return nil
}

// ParseColorscheme parses the text definition for a colorscheme and returns the corresponding object
// Colorschemes are made up of color-link statements linking a color group to a list of colors
// For example, color-link keyword (blue,red) makes all keywords have a blue foreground and
// red background
func ParseColorscheme(text string) (map[string]tcell.Style, error) {
	var err error
	parser := regexp.MustCompile(`color-link\s+(\S*)\s+"(.*)"`)

	lines := strings.Split(text, "\n")

	c := make(map[string]tcell.Style)

	for _, line := range lines {
		if strings.TrimSpace(line) == "" ||
			strings.TrimSpace(line)[0] == '#' {
			// Ignore this line
			continue
		}

		matches := parser.FindSubmatch([]byte(line))
		if len(matches) == 3 {
			link := string(matches[1])
			colors := string(matches[2])

			style := StringToStyle(colors)
			c[link] = style

			if link == "default" {
				DefStyle = style
			}
		} else {
			err = errors.New("Color-link statement is not valid: " + line)
		}
	}

	return c, err
}

// StringToStyle returns a style from a string
// The strings must be in the format "extra foregroundcolor,backgroundcolor"
// The 'extra' can be bold, reverse, italic or underline
func StringToStyle(str string) tcell.Style {
	var fg, bg string
	spaceSplit := strings.Split(str, " ")
	split := strings.Split(spaceSplit[len(spaceSplit)-1], ",")
	if len(split) > 1 {
		fg, bg = split[0], split[1]
	} else {
		fg = split[0]
	}
	fg = strings.TrimSpace(fg)
	bg = strings.TrimSpace(bg)

	var fgColor, bgColor tcell.Color
	if fg == "" {
		fgColor, _, _ = DefStyle.Decompose()
	} else {
		fgColor = StringToColor(fg)
	}
	if bg == "" {
		_, bgColor, _ = DefStyle.Decompose()
	} else {
		bgColor = StringToColor(bg)
	}

	style := DefStyle.Foreground(fgColor).Background(bgColor)
	if strings.Contains(str, "bold") {
		style = style.Bold(true)
	}
	if strings.Contains(str, "italic") {
		style = style.Italic(true)
	}
	if strings.Contains(str, "reverse") {
		style = style.Reverse(true)
	}
	if strings.Contains(str, "underline") {
		style = style.Underline(true)
	}
	return style
}

// StringToColor returns a tcell color from a string representation of a color
// We accept either bright... or light... to mean the brighter version of a color
func StringToColor(str string) tcell.Color {
	switch str {
	case "black":
		return tcell.ColorBlack
	case "red":
		return tcell.ColorMaroon
	case "green":
		return tcell.ColorGreen
	case "yellow":
		return tcell.ColorOlive
	case "blue":
		return tcell.ColorNavy
	case "magenta":
		return tcell.ColorPurple
	case "cyan":
		return tcell.ColorTeal
	case "white":
		return tcell.ColorSilver
	case "brightblack", "lightblack":
		return tcell.ColorGray
	case "brightred", "lightred":
		return tcell.ColorRed
	case "brightgreen", "lightgreen":
		return tcell.ColorLime
	case "brightyellow", "lightyellow":
		return tcell.ColorYellow
	case "brightblue", "lightblue":
		return tcell.ColorBlue
	case "brightmagenta", "lightmagenta":
		return tcell.ColorFuchsia
	case "brightcyan", "lightcyan":
		return tcell.ColorAqua
	case "brightwhite", "lightwhite":
		return tcell.ColorWhite
	case "default":
		return tcell.ColorDefault
	default:
		// Check if this is a 256 color
		if num, err := strconv.Atoi(str); err == nil {
			return GetColor256(num)
		}
		// Probably a truecolor hex value
		return tcell.GetColor(str)
	}
}

// GetColor256 returns the tcell color for a number between 0 and 255
func GetColor256(color int) tcell.Color {
	if color == 0 {
		return tcell.ColorDefault
	}
	return tcell.PaletteColor(color)
}
<commit_msg>Fix some issues with default colors in colorschemes (#2225)<commit_after>package config

import (
	"errors"
	"regexp"
	"strconv"
	"strings"

	"github.com/zyedidia/tcell/v2"
)

// DefStyle is Micro's default style
var DefStyle tcell.Style = tcell.StyleDefault

// Colorscheme is the current colorscheme
var Colorscheme map[string]tcell.Style

// GetColor takes in a syntax group and returns the colorscheme's style for that group
func GetColor(color string) tcell.Style {
	st := DefStyle
	if color == "" {
		return st
	}
	groups := strings.Split(color, ".")
	if len(groups) > 1 {
		curGroup := ""
		for i, g := range groups {
			if i != 0 {
				curGroup += "."
			}
			curGroup += g
			if style, ok := Colorscheme[curGroup]; ok {
				st = style
			}
		}
	} else if style, ok := Colorscheme[color]; ok {
		st = style
	}

	return st
}

// ColorschemeExists checks if a given colorscheme exists
func ColorschemeExists(colorschemeName string) bool {
	return FindRuntimeFile(RTColorscheme, colorschemeName) != nil
}

// InitColorscheme picks and initializes the colorscheme when micro starts
func InitColorscheme() error {
	Colorscheme = make(map[string]tcell.Style)
	DefStyle = tcell.StyleDefault

	return LoadDefaultColorscheme()
}

// LoadDefaultColorscheme loads the default colorscheme from $(ConfigDir)/colorschemes
func LoadDefaultColorscheme() error {
	return LoadColorscheme(GlobalSettings["colorscheme"].(string))
}

// LoadColorscheme loads the given colorscheme from a directory
func LoadColorscheme(colorschemeName string) error {
	file := FindRuntimeFile(RTColorscheme, colorschemeName)
	if file == nil {
		return errors.New(colorschemeName + " is not a valid colorscheme")
	}
	if data, err := file.Data(); err != nil {
		return errors.New("Error loading colorscheme: " + err.Error())
	} else {
		Colorscheme, err = ParseColorscheme(string(data))
		if err != nil {
			return err
		}
	}
	return nil
}

// ParseColorscheme parses the text definition for a colorscheme and returns the corresponding object
// Colorschemes are made up of color-link statements linking a color group to a list of colors
// For example, color-link keyword (blue,red) makes all keywords have a blue foreground and
// red background
func ParseColorscheme(text string) (map[string]tcell.Style, error) {
	var err error
	parser := regexp.MustCompile(`color-link\s+(\S*)\s+"(.*)"`)

	lines := strings.Split(text, "\n")

	c := make(map[string]tcell.Style)

	for _, line := range lines {
		if strings.TrimSpace(line) == "" ||
			strings.TrimSpace(line)[0] == '#' {
			// Ignore this line
			continue
		}

		matches := parser.FindSubmatch([]byte(line))
		if len(matches) == 3 {
			link := string(matches[1])
			colors := string(matches[2])

			style := StringToStyle(colors)
			c[link] = style

			if link == "default" {
				DefStyle = style
			}
		} else {
			err = errors.New("Color-link statement is not valid: " + line)
		}
	}

	return c, err
}

// StringToStyle returns a style from a string
// The strings must be in the format "extra foregroundcolor,backgroundcolor"
// The 'extra' can be bold, reverse, italic or underline
func StringToStyle(str string) tcell.Style {
	var fg, bg string
	spaceSplit := strings.Split(str, " ")
	split := strings.Split(spaceSplit[len(spaceSplit)-1], ",")
	if len(split) > 1 {
		fg, bg = split[0], split[1]
	} else {
		fg = split[0]
	}
	fg = strings.TrimSpace(fg)
	bg = strings.TrimSpace(bg)

	var fgColor, bgColor tcell.Color
	if fg == "" || fg == "default" {
		fgColor, _, _ = DefStyle.Decompose()
	} else {
		fgColor = StringToColor(fg)
	}
	if bg == "" || bg == "default" {
		_, bgColor, _ = DefStyle.Decompose()
	} else {
		bgColor = StringToColor(bg)
	}

	style := DefStyle.Foreground(fgColor).Background(bgColor)
	if strings.Contains(str, "bold") {
		style = style.Bold(true)
	}
	if strings.Contains(str, "italic") {
		style = style.Italic(true)
	}
	if strings.Contains(str, "reverse") {
		style = style.Reverse(true)
	}
	if strings.Contains(str, "underline") {
		style = style.Underline(true)
	}
	return style
}

// StringToColor returns a tcell color from a string representation of a color
// We accept either bright... or light... to mean the brighter version of a color
func StringToColor(str string) tcell.Color {
	switch str {
	case "black":
		return tcell.ColorBlack
	case "red":
		return tcell.ColorMaroon
	case "green":
		return tcell.ColorGreen
	case "yellow":
		return tcell.ColorOlive
	case "blue":
		return tcell.ColorNavy
	case "magenta":
		return tcell.ColorPurple
	case "cyan":
		return tcell.ColorTeal
	case "white":
		return tcell.ColorSilver
	case "brightblack", "lightblack":
		return tcell.ColorGray
	case "brightred", "lightred":
		return tcell.ColorRed
	case "brightgreen", "lightgreen":
		return tcell.ColorLime
	case "brightyellow", "lightyellow":
		return tcell.ColorYellow
	case "brightblue", "lightblue":
		return tcell.ColorBlue
	case "brightmagenta", "lightmagenta":
		return tcell.ColorFuchsia
	case "brightcyan", "lightcyan":
		return tcell.ColorAqua
	case "brightwhite", "lightwhite":
		return tcell.ColorWhite
	case "default":
		return tcell.ColorDefault
	default:
		// Check if this is a 256 color
		if num, err := strconv.Atoi(str); err == nil {
			return GetColor256(num)
		}
		// Probably a truecolor hex value
		return tcell.GetColor(str)
	}
}

// GetColor256 returns the tcell color for a number between 0 and 255
func GetColor256(color int) tcell.Color {
	if color == 0 {
		return tcell.ColorDefault
	}
	return tcell.PaletteColor(color)
}
<|endoftext|>


<commit_before>package schema_test

import (
	"testing"

	"github.com/graph-gophers/graphql-go/internal/schema"
)

type parseTestCase struct {
	description string
	sdl         string
	expected    *schema.Schema
	err         error
}

var parseTests = []parseTestCase{{
	description: "Parses interface definition",
	sdl:         "interface Greeting { message: String! }",
	expected: &schema.Schema{
		Types: map[string]schema.NamedType{
			"Greeting": &schema.Interface{
				Name:   "Greeting",
				Fields: []*schema.Field{{Name: "message"}},
			}},
	}}, {
	description: "Parses type with description string",
	sdl: `
	"Single line description."
	type Type {
		field: String
	}`,
	expected: &schema.Schema{
		Types: map[string]schema.NamedType{
			"Type": &schema.Object{
				Name: "Type",
				Desc: "Single line description.",
			}},
	}}, {
	description: "Parses type with multi-line description string",
	sdl: `
	"""
	Multi-line description.
	"""
	type Type {
		field: String
	}`,
	expected: &schema.Schema{
		Types: map[string]schema.NamedType{
			"Type": &schema.Object{
				Name: "Type",
				Desc: "Multi-line description.",
			}},
	}}, {
	description: "Parses type with multi-line description and ignores comments",
	sdl: `
	"""
	Multi-line description with ignored comments.
	"""
	# This comment should be ignored.
	type Type {
		field: String
	}`,
	expected: &schema.Schema{
		Types: map[string]schema.NamedType{
			"Type": &schema.Object{
				Name: "Type",
				Desc: "Multi-line description with ignored comments.",
			}},
	}},
}

func TestParse(t *testing.T) {
	setup := func(t *testing.T) *schema.Schema {
		t.Helper()
		return schema.New()
	}

	for _, test := range parseTests {
		t.Run(test.description, func(t *testing.T) {
			t.Skip("TODO: add support for descriptions")
			schema := setup(t)

			err := schema.Parse(test.sdl, false)
			if err != nil {
				t.Fatal(err)
			}

			// TODO: verify schema is the same as expected.
		})
	}
}

func TestInvalidInterfaceImpl(t *testing.T) {
	var tests = []parseTestCase{{
		description: "Parses type Welcome that implements interface Greeting without providing required fields",
		sdl:         "interface Greeting { message: String! } type Welcome implements Greeting {}",
		err:         errors.Errorf(`interface "Greeting" expects field "message" but "Welcome" does not provide it`),
	}}

	setup := func(t *testing.T) *schema.Schema {
		t.Helper()
		return schema.New()
	}
	for _, test := range tests {
		t.Run(test.description, func(t *testing.T) {
			schema := setup(t)
			err := schema.Parse(test.sdl, false)
			if err == nil || err.Error() != test.err.Error() {
				t.Fatal(err)
			}
		})
	}
}
<commit_msg>Update schema_test.go<commit_after>package schema_test

import (
	"testing"

	"github.com/graph-gophers/graphql-go/errors"
	"github.com/graph-gophers/graphql-go/internal/schema"
)

type parseTestCase struct {
	description string
	sdl         string
	expected    *schema.Schema
	err         error
}

var parseTests = []parseTestCase{{
	description: "Parses interface definition",
	sdl:         "interface Greeting { message: String! }",
	expected: &schema.Schema{
		Types: map[string]schema.NamedType{
			"Greeting": &schema.Interface{
				Name:   "Greeting",
				Fields: []*schema.Field{{Name: "message"}},
			}},
	}}, {
	description: "Parses type with description string",
	sdl: `
	"Single line description."
	type Type {
		field: String
	}`,
	expected: &schema.Schema{
		Types: map[string]schema.NamedType{
			"Type": &schema.Object{
				Name: "Type",
				Desc: "Single line description.",
			}},
	}}, {
	description: "Parses type with multi-line description string",
	sdl: `
	"""
	Multi-line description.
	"""
	type Type {
		field: String
	}`,
	expected: &schema.Schema{
		Types: map[string]schema.NamedType{
			"Type": &schema.Object{
				Name: "Type",
				Desc: "Multi-line description.",
			}},
	}}, {
	description: "Parses type with multi-line description and ignores comments",
	sdl: `
	"""
	Multi-line description with ignored comments.
	"""
	# This comment should be ignored.
	type Type {
		field: String
	}`,
	expected: &schema.Schema{
		Types: map[string]schema.NamedType{
			"Type": &schema.Object{
				Name: "Type",
				Desc: "Multi-line description with ignored comments.",
			}},
	}},
}

func TestParse(t *testing.T) {
	setup := func(t *testing.T) *schema.Schema {
		t.Helper()
		return schema.New()
	}

	for _, test := range parseTests {
		t.Run(test.description, func(t *testing.T) {
			t.Skip("TODO: add support for descriptions")
			schema := setup(t)

			err := schema.Parse(test.sdl, false)
			if err != nil {
				t.Fatal(err)
			}

			// TODO: verify schema is the same as expected.
		})
	}
}

func TestInvalidInterfaceImpl(t *testing.T) {
	var tests = []parseTestCase{{
		description: "Parses type Welcome that implements interface Greeting without providing required fields",
		sdl:         "interface Greeting { message: String! } type Welcome implements Greeting {}",
		err:         errors.Errorf(`interface "Greeting" expects field "message" but "Welcome" does not provide it`),
	}}

	setup := func(t *testing.T) *schema.Schema {
		t.Helper()
		return schema.New()
	}
	for _, test := range tests {
		t.Run(test.description, func(t *testing.T) {
			schema := setup(t)
			err := schema.Parse(test.sdl, false)
			if err == nil || err.Error() != test.err.Error() {
				t.Fatal(err)
			}
		})
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package sourcecache provides a cache of code found in Git repositories.
// It takes directly to the Gerrit instance at go.googlesource.com.
// If RegisterGitMirrorDial is called, it will first try to get code from gitmirror before falling back on Gerrit.
package sourcecache

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"time"

	"golang.org/x/build/cmd/coordinator/spanlog"
	"golang.org/x/build/internal/lru"
	"golang.org/x/build/internal/singleflight"
)

var processStartTime = time.Now()

var sourceGroup singleflight.Group

var sourceCache = lru.New(40) // git rev -> []byte

// GetSourceTgz returns a Reader that provides a tgz of the requested source revision.
// repo is go.googlesource.com repo ("go", "net", etc)
// rev is git revision.
func GetSourceTgz(sl spanlog.Logger, repo, rev string) (tgz io.Reader, err error) {
	sp := sl.CreateSpan("get_source")
	defer func() { sp.Done(err) }()

	key := fmt.Sprintf("%v-%v", repo, rev)
	vi, err, _ := sourceGroup.Do(key, func() (interface{}, error) {
		if tgzBytes, ok := sourceCache.Get(key); ok {
			return tgzBytes, nil
		}

		if gitMirrorClient != nil {
			sp := sl.CreateSpan("get_source_from_gitmirror")
			tgzBytes, err := getSourceTgzFromGitMirror(repo, rev)
			if err == nil {
				sourceCache.Add(key, tgzBytes)
				sp.Done(nil)
				return tgzBytes, nil
			}
			log.Printf("Error fetching source %s/%s from watcher (after %v uptime): %v",
				repo, rev, time.Since(processStartTime), err)
			sp.Done(errors.New("timeout"))
		}

		sp := sl.CreateSpan("get_source_from_gerrit", fmt.Sprintf("%v from gerrit", key))
		tgzBytes, err := getSourceTgzFromGerrit(repo, rev)
		sp.Done(err)
		if err == nil {
			sourceCache.Add(key, tgzBytes)
		}
		return tgzBytes, err
	})
	if err != nil {
		return nil, err
	}
	return bytes.NewReader(vi.([]byte)), nil
}

var gitMirrorClient *http.Client

// RegisterGitMirrorDial registers a dial function which will be used to reach gitmirror.
// If used, this function must be called before GetSourceTgz.
func RegisterGitMirrorDial(dial func(context.Context) (net.Conn, error)) {
	gitMirrorClient = &http.Client{
		Timeout: 30 * time.Second,
		Transport: &http.Transport{
			IdleConnTimeout: 30 * time.Second,
			DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
				return dial(ctx)
			},
		},
	}
}

var gerritHTTPClient = &http.Client{
	Timeout: 30 * time.Second,
}

func getSourceTgzFromGerrit(repo, rev string) (tgz []byte, err error) {
	return getSourceTgzFromURL(gerritHTTPClient, "gerrit", repo, rev, "https://go.googlesource.com/"+repo+"/+archive/"+rev+".tar.gz")
}

func getSourceTgzFromGitMirror(repo, rev string) (tgz []byte, err error) {
	for i := 0; i < 2; i++ { // two tries; different pods maybe?
		if i > 0 {
			time.Sleep(1 * time.Second)
		}
		// The "gitmirror" hostname is unused:
		tgz, err = getSourceTgzFromURL(gitMirrorClient, "gitmirror", repo, rev, "http://gitmirror/"+repo+".tar.gz?rev="+rev)
		if err == nil {
			return tgz, nil
		}
		if tr, ok := http.DefaultTransport.(*http.Transport); ok {
			tr.CloseIdleConnections()
		}
	}
	return nil, err
}

func getSourceTgzFromURL(hc *http.Client, source, repo, rev, urlStr string) (tgz []byte, err error) {
	res, err := hc.Get(urlStr)
	if err != nil {
		return nil, fmt.Errorf("fetching %s/%s from %s: %v", repo, rev, source, err)
	}
	defer res.Body.Close()
	if res.StatusCode/100 != 2 {
		slurp, _ := ioutil.ReadAll(io.LimitReader(res.Body, 4<<10))
		return nil, fmt.Errorf("fetching %s/%s from %s: %v; body: %s", repo, rev, source, res.Status, slurp)
	}
	// TODO(bradfitz): finish golang.org/issue/11224
	const maxSize = 50 << 20 // talks repo is over 25MB; go source is 7.8MB on 2015-06-15
	slurp, err := ioutil.ReadAll(io.LimitReader(res.Body, maxSize+1))
	if len(slurp) > maxSize && err == nil {
		err = fmt.Errorf("body over %d bytes", maxSize)
	}
	if err != nil {
		return nil, fmt.Errorf("reading %s/%s from %s: %v", repo, rev, source, err)
	}
	return slurp, nil
}
<commit_msg>internal/sourcecache: add more logging of source fetches<commit_after>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package sourcecache provides a cache of code found in Git repositories.
// It takes directly to the Gerrit instance at go.googlesource.com.
// If RegisterGitMirrorDial is called, it will first try to get code from gitmirror before falling back on Gerrit.
package sourcecache

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"time"

	"golang.org/x/build/cmd/coordinator/spanlog"
	"golang.org/x/build/internal/lru"
	"golang.org/x/build/internal/singleflight"
)

var processStartTime = time.Now()

var sourceGroup singleflight.Group

var sourceCache = lru.New(40) // git rev -> []byte

// GetSourceTgz returns a Reader that provides a tgz of the requested source revision.
// repo is go.googlesource.com repo ("go", "net", etc)
// rev is git revision.
func GetSourceTgz(sl spanlog.Logger, repo, rev string) (tgz io.Reader, err error) {
	sp := sl.CreateSpan("get_source", repo+"@"+rev)
	defer func() { sp.Done(err) }()

	key := fmt.Sprintf("%v-%v", repo, rev)
	vi, err, _ := sourceGroup.Do(key, func() (interface{}, error) {
		if tgzBytes, ok := sourceCache.Get(key); ok {
			return tgzBytes, nil
		}

		if gitMirrorClient != nil {
			sp := sl.CreateSpan("get_source_from_gitmirror")
			tgzBytes, err := getSourceTgzFromGitMirror(repo, rev)
			if err == nil {
				sourceCache.Add(key, tgzBytes)
				sp.Done(nil)
				return tgzBytes, nil
			}
			log.Printf("Error fetching source %s/%s from watcher (after %v uptime): %v",
				repo, rev, time.Since(processStartTime), err)
			sp.Done(errors.New("timeout"))
		}

		sp := sl.CreateSpan("get_source_from_gerrit", fmt.Sprintf("%v from gerrit", key))
		tgzBytes, err := getSourceTgzFromGerrit(repo, rev)
		sp.Done(err)
		if err == nil {
			sourceCache.Add(key, tgzBytes)
		}
		return tgzBytes, err
	})
	if err != nil {
		return nil, err
	}
	return bytes.NewReader(vi.([]byte)), nil
}

var gitMirrorClient *http.Client

// RegisterGitMirrorDial registers a dial function which will be used to reach gitmirror.
// If used, this function must be called before GetSourceTgz.
func RegisterGitMirrorDial(dial func(context.Context) (net.Conn, error)) {
	gitMirrorClient = &http.Client{
		Timeout: 30 * time.Second,
		Transport: &http.Transport{
			IdleConnTimeout: 30 * time.Second,
			DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
				return dial(ctx)
			},
		},
	}
}

var gerritHTTPClient = &http.Client{
	Timeout: 30 * time.Second,
}

func getSourceTgzFromGerrit(repo, rev string) (tgz []byte, err error) {
	return getSourceTgzFromURL(gerritHTTPClient, "gerrit", repo, rev, "https://go.googlesource.com/"+repo+"/+archive/"+rev+".tar.gz")
}

func getSourceTgzFromGitMirror(repo, rev string) (tgz []byte, err error) {
	for i := 0; i < 2; i++ { // two tries; different pods maybe?
		if i > 0 {
			time.Sleep(1 * time.Second)
		}
		// The "gitmirror" hostname is unused:
		tgz, err = getSourceTgzFromURL(gitMirrorClient, "gitmirror", repo, rev, "http://gitmirror/"+repo+".tar.gz?rev="+rev)
		if err == nil {
			return tgz, nil
		}
		if tr, ok := http.DefaultTransport.(*http.Transport); ok {
			tr.CloseIdleConnections()
		}
	}
	return nil, err
}

func getSourceTgzFromURL(hc *http.Client, source, repo, rev, urlStr string) (tgz []byte, err error) {
	res, err := hc.Get(urlStr)
	if err != nil {
		return nil, fmt.Errorf("fetching %s/%s from %s: %v", repo, rev, source, err)
	}
	defer res.Body.Close()
	if res.StatusCode/100 != 2 {
		slurp, _ := ioutil.ReadAll(io.LimitReader(res.Body, 4<<10))
		return nil, fmt.Errorf("fetching %s/%s from %s: %v; body: %s", repo, rev, source, res.Status, slurp)
	}
	// TODO(bradfitz): finish golang.org/issue/11224
	const maxSize = 50 << 20 // talks repo is over 25MB; go source is 7.8MB on 2015-06-15
	slurp, err := ioutil.ReadAll(io.LimitReader(res.Body, maxSize+1))
	if len(slurp) > maxSize && err == nil {
		err = fmt.Errorf("body over %d bytes", maxSize)
	}
	if err != nil {
		return nil, fmt.Errorf("reading %s/%s from %s: %v", repo, rev, source, err)
	}
	return slurp, nil
}
<|endoftext|>


<commit_before>package cache

import (
	"os"
	"path/filepath"
	"strings"
)

func Check(path string) {
	resultq := make(chan bool)

	dirname, basename := filepath.Split(path)
	requestq <- func() {
		for _, p := range executables[dirname] {
			if p == basename {
				resultq <- true
				break
			}
		}
		close(resultq)
	}

	if <-resultq {
		Files(dirname)
	}
}

func Executables(dirname string) []string {
	resultq := make(chan []string)

	requestq <- func() {
		resultq <- executables[dirname]
		close(resultq)
	}

	go Files(dirname)

	return <-resultq
}

func Files(dirname string) []string {
	max := strings.Count(dirname, pathSeparator)

	resultq := make(chan []string)

	requestq <- func() {
		e := []string{}
		f := []string{}

		_ = filepath.Walk(dirname, func(p string, i os.FileInfo, err error) error {
			if p == dirname {
				return nil
			}

			depth := strings.Count(p, pathSeparator)
			if depth > max {
				if i.IsDir() {
					return filepath.SkipDir
				}

				return nil
			} else if depth < max {
				return nil
			}

			if p != pathSeparator && i.IsDir() {
				p += pathSeparator

				e = append(e, p)
				f = append(f, p)
			} else if i.Mode()&0111 != 0 {
				e = append(e, p)
			} else {
				f = append(f, p)
			}

			return nil
		})

		executables[dirname] = e

		resultq <- f
		close(resultq)
	}

	return <-resultq
}

func Populate(dirnames string) {
	for _, dirname := range strings.Split(dirnames, pathListSeparator) {
		if dirname == "" {
			dirname = "."
		} else {
			dirname = filepath.Clean(dirname)
		}

		stat, err := os.Stat(dirname)
		if err != nil || !stat.IsDir() {
			continue
		}

		Files(dirname)
	}
}

var (
	executables       = map[string][]string{}
	pathListSeparator = string(os.PathListSeparator)
	pathSeparator     = string(os.PathSeparator)
	requestq          chan func()
)

func init() {
	requestq = make(chan func(), 1)

	go service()
}

func service() {
	for {
		(<-requestq)()
	}
}
<commit_msg>Improving completion.<commit_after>package cache

import (
	"os"
	"path/filepath"
	"strings"
)

func Check(path string) {
	resultq := make(chan bool)

	dirname, basename := filepath.Split(path)
	requestq <- func() {
		for _, p := range executables[dirname] {
			if p == basename {
				resultq <- true
				break
			}
		}
		close(resultq)
	}

	if <-resultq {
		Files(dirname)
	}
}

func Executables(dirname string) []string {
	resultq := make(chan []string)

	dirname = filepath.Clean(dirname)
	requestq <- func() {
		resultq <- executables[dirname]
		close(resultq)
	}

	return <-resultq
}

func Files(dirname string) []string {
	dirname = filepath.Clean(dirname)

	max := strings.Count(dirname, pathSeparator) + 1

	e := []string{}
	f := []string{}

	_ = filepath.Walk(dirname, func(p string, i os.FileInfo, err error) error {
		if p == dirname {
			return nil
		}

		depth := strings.Count(p, pathSeparator)
		if depth > max {
			if i.IsDir() {
				return filepath.SkipDir
			}

			return nil
		} else if depth < max {
			return nil
		}

		if p != pathSeparator && i.IsDir() {
			p += pathSeparator

			e = append(e, p)
			f = append(f, p)
		} else if i.Mode()&0111 != 0 {
			e = append(e, p)
		} else {
			f = append(f, p)
		}

		return nil
	})

	requestq <- func() {
		executables[dirname] = e
	}

	return f
}

func Populate(dirnames string) {
	for _, dirname := range strings.Split(dirnames, pathListSeparator) {
		if dirname == "" {
			dirname = "."
		} else {
			dirname = filepath.Clean(dirname)
		}

		stat, err := os.Stat(dirname)
		if err != nil || !stat.IsDir() {
			continue
		}

		Files(dirname)
	}
}

var (
	executables       = map[string][]string{}
	pathListSeparator = string(os.PathListSeparator)
	pathSeparator     = string(os.PathSeparator)
	requestq          chan func()
)

func init() {
	requestq = make(chan func(), 1)

	go service()
}

func service() {
	for {
		(<-requestq)()
	}
}
<|endoftext|>


<commit_before>package fastly

import (
	"fmt"
	"net/http"
	"reflect"
	"time"

	"github.com/mitchellh/mapstructure"
)

// mapToHTTPHeaderHookFunc returns a function that converts maps into an
// http.Header value.
func mapToHTTPHeaderHookFunc() mapstructure.DecodeHookFunc {
	return func(
		f reflect.Type,
		t reflect.Type,
		data interface{}) (interface{}, error) {
		if f.Kind() != reflect.Map {
			return data, nil
		}
		if t != reflect.TypeOf(new(http.Header)) {
			return data, nil
		}

		typed, ok := data.(map[string]interface{})
		if !ok {
			return nil, fmt.Errorf("cannot convert %T to http.Header", data)
		}

		n := map[string][]string{}
		for k, v := range typed {
			switch v.(type) {
			case string:
				n[k] = []string{v.(string)}
			case []string:
				n[k] = v.([]string)
			default:
				return nil, fmt.Errorf("cannot convert %T to http.Header", v)
			}
		}

		return n, nil
	}
}

// stringToTimeHookFunc returns a function that converts strings to a time.Time
// value.
func stringToTimeHookFunc() mapstructure.DecodeHookFunc {
	return func(
		f reflect.Type,
		t reflect.Type,
		data interface{}) (interface{}, error) {
		if f.Kind() != reflect.String {
			return data, nil
		}
		if t != reflect.TypeOf(time.Now()) {
			return data, nil
		}

		// Convert it by parsing
		return time.Parse(time.RFC3339, data.(string))
	}
}
<commit_msg>Fix broken decode hook<commit_after>package fastly

import (
	"fmt"
	"net/http"
	"reflect"
	"time"

	"github.com/mitchellh/mapstructure"
)

// mapToHTTPHeaderHookFunc returns a function that converts maps into an
// http.Header value.
func mapToHTTPHeaderHookFunc() mapstructure.DecodeHookFunc {
	return func(
		f reflect.Type,
		t reflect.Type,
		data interface{}) (interface{}, error) {
		if f.Kind() != reflect.Map {
			return data, nil
		}
		if t != reflect.TypeOf(new(http.Header)) {
			return data, nil
		}

		typed, ok := data.(map[string]interface{})
		if !ok {
			return nil, fmt.Errorf("cannot convert %T to http.Header", data)
		}

		n := map[string][]string{}
		for k, v := range typed {
			switch v.(type) {
			case string:
				n[k] = []string{v.(string)}
			case []string:
				n[k] = v.([]string)
			case int, int8, int16, int32, int64:
				n[k] = []string{fmt.Sprintf("%d", v.(int))}
			case float32, float64:
				n[k] = []string{fmt.Sprintf("%f", v.(float64))}
			default:
				return nil, fmt.Errorf("cannot convert %T to http.Header", v)
			}
		}

		return n, nil
	}
}

// stringToTimeHookFunc returns a function that converts strings to a time.Time
// value.
func stringToTimeHookFunc() mapstructure.DecodeHookFunc {
	return func(
		f reflect.Type,
		t reflect.Type,
		data interface{}) (interface{}, error) {
		if f.Kind() != reflect.String {
			return data, nil
		}
		if t != reflect.TypeOf(time.Now()) {
			return data, nil
		}

		// Convert it by parsing
		return time.Parse(time.RFC3339, data.(string))
	}
}
<|endoftext|>


<commit_before>package dbf

import (
	"bytes"
	"testing"
)

func TestUTF8Decoder_Decode(t *testing.T) {
	dec := new(UTF8Decoder)
	in := []byte("Tésting ㇹ Д")
	b, err := dec.Decode(in)
	if err != nil {
		t.Fatalf("error in decode: %s", err)
	}
	if bytes.Equal(in, b) == false {
		t.Errorf("Want %s, have %s", string(in), string(b))
	}
}

func TestWin1250Decoder_Decode (t *testing.T) {
	dec := new(Win1250Decoder)
	in := []byte{0xC4, 0xF5}
	b, err := dec.Decode(in)
	if err != nil {
		t.Fatalf("error in decode: %s", err)
	}
	want := "Äő"
	if string(b) != want {
		t.Errorf("Want %s, have %s", want, string(b))
	}
}<commit_msg>rerun gofmt on new test<commit_after>package dbf

import (
	"bytes"
	"testing"
)

func TestUTF8Decoder_Decode(t *testing.T) {
	dec := new(UTF8Decoder)
	in := []byte("Tésting ㇹ Д")
	b, err := dec.Decode(in)
	if err != nil {
		t.Fatalf("error in decode: %s", err)
	}
	if bytes.Equal(in, b) == false {
		t.Errorf("Want %s, have %s", string(in), string(b))
	}
}

func TestWin1250Decoder_Decode(t *testing.T) {
	dec := new(Win1250Decoder)
	in := []byte{0xC4, 0xF5}
	b, err := dec.Decode(in)
	if err != nil {
		t.Fatalf("error in decode: %s", err)
	}
	want := "Äő"
	if string(b) != want {
		t.Errorf("Want %s, have %s", want, string(b))
	}
}
<|endoftext|>


<commit_before>package define

import (
	"io"
	"time"

	"github.com/containers/image/v5/types"
	encconfig "github.com/containers/ocicrypt/config"
	"github.com/containers/storage/pkg/archive"
	"golang.org/x/sync/semaphore"
)

// CommonBuildOptions are resources that can be defined by flags for both buildah from and build
type CommonBuildOptions struct {
	// AddHost is the list of hostnames to add to the build container's /etc/hosts.
	AddHost []string
	// CgroupParent is the path to cgroups under which the cgroup for the container will be created.
	CgroupParent string
	// CPUPeriod limits the CPU CFS (Completely Fair Scheduler) period
	CPUPeriod uint64
	// CPUQuota limits the CPU CFS (Completely Fair Scheduler) quota
	CPUQuota int64
	// CPUShares (relative weight
	CPUShares uint64
	// CPUSetCPUs in which to allow execution (0-3, 0,1)
	CPUSetCPUs string
	// CPUSetMems memory nodes (MEMs) in which to allow execution (0-3, 0,1). Only effective on NUMA systems.
	CPUSetMems string
	// HTTPProxy determines whether *_proxy env vars from the build host are passed into the container.
	HTTPProxy bool
	// Memory is the upper limit (in bytes) on how much memory running containers can use.
	Memory int64
	// DNSSearch is the list of DNS search domains to add to the build container's /etc/resolv.conf
	DNSSearch []string
	// DNSServers is the list of DNS servers to add to the build container's /etc/resolv.conf
	DNSServers []string
	// DNSOptions is the list of DNS
	DNSOptions []string
	// MemorySwap limits the amount of memory and swap together.
	MemorySwap int64
	// LabelOpts is the a slice of fields of an SELinux context, given in "field:pair" format, or "disable".
	// Recognized field names are "role", "type", and "level".
	LabelOpts []string
	// OmitTimestamp forces epoch 0 as created timestamp to allow for
	// deterministic, content-addressable builds.
	OmitTimestamp bool
	// SeccompProfilePath is the pathname of a seccomp profile.
	SeccompProfilePath string
	// ApparmorProfile is the name of an apparmor profile.
	ApparmorProfile string
	// ShmSize is the "size" value to use when mounting an shmfs on the container's /dev/shm directory.
	ShmSize string
	// Ulimit specifies resource limit options, in the form type:softlimit[:hardlimit].
	// These types are recognized:
	// "core": maximum core dump size (ulimit -c)
	// "cpu": maximum CPU time (ulimit -t)
	// "data": maximum size of a process's data segment (ulimit -d)
	// "fsize": maximum size of new files (ulimit -f)
	// "locks": maximum number of file locks (ulimit -x)
	// "memlock": maximum amount of locked memory (ulimit -l)
	// "msgqueue": maximum amount of data in message queues (ulimit -q)
	// "nice": niceness adjustment (nice -n, ulimit -e)
	// "nofile": maximum number of open files (ulimit -n)
	// "nproc": maximum number of processes (ulimit -u)
	// "rss": maximum size of a process's (ulimit -m)
	// "rtprio": maximum real-time scheduling priority (ulimit -r)
	// "rttime": maximum amount of real-time execution between blocking syscalls
	// "sigpending": maximum number of pending signals (ulimit -i)
	// "stack": maximum stack size (ulimit -s)
	Ulimit []string
	// Volumes to bind mount into the container
	Volumes []string
	// Secrets are the available secrets to use in a build
	Secrets []string
	// SSHSources is the available ssh agent connections to forward in the build
	SSHSources []string
}

// BuildOptions can be used to alter how an image is built.
type BuildOptions struct {
	// ContextDirectory is the default source location for COPY and ADD
	// commands.
	ContextDirectory string
	// PullPolicy controls whether or not we pull images.  It should be one
	// of PullIfMissing, PullAlways, PullIfNewer, or PullNever.
	PullPolicy PullPolicy
	// Registry is a value which is prepended to the image's name, if it
	// needs to be pulled and the image name alone can not be resolved to a
	// reference to a source image.  No separator is implicitly added.
	Registry string
	// IgnoreUnrecognizedInstructions tells us to just log instructions we
	// don't recognize, and try to keep going.
	IgnoreUnrecognizedInstructions bool
	// Manifest Name to which the image will be added.
	Manifest string
	// Quiet tells us whether or not to announce steps as we go through them.
	Quiet bool
	// Isolation controls how Run() runs things.
	Isolation Isolation
	// Runtime is the name of the command to run for RUN instructions when
	// Isolation is either IsolationDefault or IsolationOCI.  It should
	// accept the same arguments and flags that runc does.
	Runtime string
	// RuntimeArgs adds global arguments for the runtime.
	RuntimeArgs []string
	// TransientMounts is a list of mounts that won't be kept in the image.
	TransientMounts []string
	// Compression specifies the type of compression which is applied to
	// layer blobs.  The default is to not use compression, but
	// archive.Gzip is recommended.
	Compression archive.Compression
	// Arguments which can be interpolated into Dockerfiles
	Args map[string]string
	// Name of the image to write to.
	Output string
	// Additional tags to add to the image that we write, if we know of a
	// way to add them.
	AdditionalTags []string
	// Log is a callback that will print a progress message.  If no value
	// is supplied, the message will be sent to Err (or os.Stderr, if Err
	// is nil) by default.
	Log func(format string, args ...interface{})
	// In is connected to stdin for RUN instructions.
	In io.Reader
	// Out is a place where non-error log messages are sent.
	Out io.Writer
	// Err is a place where error log messages should be sent.
	Err io.Writer
	// SignaturePolicyPath specifies an override location for the signature
	// policy which should be used for verifying the new image as it is
	// being written.  Except in specific circumstances, no value should be
	// specified, indicating that the shared, system-wide default policy
	// should be used.
	SignaturePolicyPath string
	// ReportWriter is an io.Writer which will be used to report the
	// progress of the (possible) pulling of the source image and the
	// writing of the new image.
	ReportWriter io.Writer
	// OutputFormat is the format of the output image's manifest and
	// configuration data.
	// Accepted values are buildah.OCIv1ImageManifest and buildah.Dockerv2ImageManifest.
	OutputFormat string
	// SystemContext holds parameters used for authentication.
	SystemContext *types.SystemContext
	// NamespaceOptions controls how we set up namespaces processes that we
	// might need when handling RUN instructions.
	NamespaceOptions []NamespaceOption
	// ConfigureNetwork controls whether or not network interfaces and
	// routing are configured for a new network namespace (i.e., when not
	// joining another's namespace and not just using the host's
	// namespace), effectively deciding whether or not the process has a
	// usable network.
	ConfigureNetwork NetworkConfigurationPolicy
	// CNIPluginPath is the location of CNI plugin helpers, if they should be
	// run from a location other than the default location.
	CNIPluginPath string
	// CNIConfigDir is the location of CNI configuration files, if the files in
	// the default configuration directory shouldn't be used.
	CNIConfigDir string
	// ID mapping options to use if we're setting up our own user namespace
	// when handling RUN instructions.
	IDMappingOptions *IDMappingOptions
	// AddCapabilities is a list of capabilities to add to the default set when
	// handling RUN instructions.
	AddCapabilities []string
	// DropCapabilities is a list of capabilities to remove from the default set
	// when handling RUN instructions. If a capability appears in both lists, it
	// will be dropped.
	DropCapabilities []string
	// CommonBuildOpts is *required*.
	CommonBuildOpts *CommonBuildOptions
	// DefaultMountsFilePath is the file path holding the mounts to be mounted in "host-path:container-path" format
	DefaultMountsFilePath string
	// IIDFile tells the builder to write the image ID to the specified file
	IIDFile string
	// Squash tells the builder to produce an image with a single layer
	// instead of with possibly more than one layer.
	Squash bool
	// Labels metadata for an image
	Labels []string
	// Annotation metadata for an image
	Annotations []string
	// OnBuild commands to be run by images based on this image
	OnBuild []string
	// Layers tells the builder to create a cache of images for each step in the Dockerfile
	Layers bool
	// NoCache tells the builder to build the image from scratch without checking for a cache.
	// It creates a new set of cached images for the build.
	NoCache bool
	// RemoveIntermediateCtrs tells the builder whether to remove intermediate containers used
	// during the build process. Default is true.
	RemoveIntermediateCtrs bool
	// ForceRmIntermediateCtrs tells the builder to remove all intermediate containers even if
	// the build was unsuccessful.
	ForceRmIntermediateCtrs bool
	// BlobDirectory is a directory which we'll use for caching layer blobs.
	BlobDirectory string
	// Target the targeted FROM in the Dockerfile to build.
	Target string
	// Devices are the additional devices to add to the containers.
	Devices []string
	// SignBy is the fingerprint of a GPG key to use for signing images.
	SignBy string
	// Architecture specifies the target architecture of the image to be built.
	Architecture string
	// Timestamp sets the created timestamp to the specified time, allowing
	// for deterministic, content-addressable builds.
	Timestamp *time.Time
	// OS is the specifies the operating system of the image to be built.
	OS string
	// MaxPullPushRetries is the maximum number of attempts we'll make to pull or push any one
	// image from or to an external registry if the first attempt fails.
	MaxPullPushRetries int
	// PullPushRetryDelay is how long to wait before retrying a pull or push attempt.
	PullPushRetryDelay time.Duration
	// OciDecryptConfig contains the config that can be used to decrypt an image if it is
	// encrypted if non-nil. If nil, it does not attempt to decrypt an image.
	OciDecryptConfig *encconfig.DecryptConfig
	// Jobs is the number of stages to run in parallel.  If not specified it defaults to 1.
	// Ignored if a JobSemaphore is provided.
	Jobs *int
	// JobSemaphore, for when you want Jobs to be shared with more than just this build.
	JobSemaphore *semaphore.Weighted
	// LogRusage logs resource usage for each step.
	LogRusage bool
	// File to which the Rusage logs will be saved to instead of stdout
	RusageLogFile string
	// Excludes is a list of excludes to be used instead of the .dockerignore file.
	Excludes []string
	// IgnoreFile is a name of the .containerignore file
	IgnoreFile string
	// From is the image name to use to replace the value specified in the first
	// FROM instruction in the Containerfile
	From string
	// Platforms is the list of parsed OS/Arch/Variant triples that we want
	// to build the image for.  If this slice has items in it, the OS and
	// Architecture fields above are ignored.
	Platforms []struct{ OS, Arch, Variant string }
	// AllPlatforms tells the builder to set the list of target platforms
	// to match the set of platforms for which all of the build's base
	// images are available.  If this field is set, Platforms is ignored.
	AllPlatforms bool
	// UnsetEnvs is a list of environments to not add to final image.
	UnsetEnvs []string
}
<commit_msg>Expand the godoc for CommonBuildOptions.Secrets<commit_after>package define

import (
	"io"
	"time"

	"github.com/containers/image/v5/types"
	encconfig "github.com/containers/ocicrypt/config"
	"github.com/containers/storage/pkg/archive"
	"golang.org/x/sync/semaphore"
)

// CommonBuildOptions are resources that can be defined by flags for both buildah from and build
type CommonBuildOptions struct {
	// AddHost is the list of hostnames to add to the build container's /etc/hosts.
	AddHost []string
	// CgroupParent is the path to cgroups under which the cgroup for the container will be created.
	CgroupParent string
	// CPUPeriod limits the CPU CFS (Completely Fair Scheduler) period
	CPUPeriod uint64
	// CPUQuota limits the CPU CFS (Completely Fair Scheduler) quota
	CPUQuota int64
	// CPUShares (relative weight
	CPUShares uint64
	// CPUSetCPUs in which to allow execution (0-3, 0,1)
	CPUSetCPUs string
	// CPUSetMems memory nodes (MEMs) in which to allow execution (0-3, 0,1). Only effective on NUMA systems.
	CPUSetMems string
	// HTTPProxy determines whether *_proxy env vars from the build host are passed into the container.
	HTTPProxy bool
	// Memory is the upper limit (in bytes) on how much memory running containers can use.
	Memory int64
	// DNSSearch is the list of DNS search domains to add to the build container's /etc/resolv.conf
	DNSSearch []string
	// DNSServers is the list of DNS servers to add to the build container's /etc/resolv.conf
	DNSServers []string
	// DNSOptions is the list of DNS
	DNSOptions []string
	// MemorySwap limits the amount of memory and swap together.
	MemorySwap int64
	// LabelOpts is the a slice of fields of an SELinux context, given in "field:pair" format, or "disable".
	// Recognized field names are "role", "type", and "level".
	LabelOpts []string
	// OmitTimestamp forces epoch 0 as created timestamp to allow for
	// deterministic, content-addressable builds.
	OmitTimestamp bool
	// SeccompProfilePath is the pathname of a seccomp profile.
	SeccompProfilePath string
	// ApparmorProfile is the name of an apparmor profile.
	ApparmorProfile string
	// ShmSize is the "size" value to use when mounting an shmfs on the container's /dev/shm directory.
	ShmSize string
	// Ulimit specifies resource limit options, in the form type:softlimit[:hardlimit].
	// These types are recognized:
	// "core": maximum core dump size (ulimit -c)
	// "cpu": maximum CPU time (ulimit -t)
	// "data": maximum size of a process's data segment (ulimit -d)
	// "fsize": maximum size of new files (ulimit -f)
	// "locks": maximum number of file locks (ulimit -x)
	// "memlock": maximum amount of locked memory (ulimit -l)
	// "msgqueue": maximum amount of data in message queues (ulimit -q)
	// "nice": niceness adjustment (nice -n, ulimit -e)
	// "nofile": maximum number of open files (ulimit -n)
	// "nproc": maximum number of processes (ulimit -u)
	// "rss": maximum size of a process's (ulimit -m)
	// "rtprio": maximum real-time scheduling priority (ulimit -r)
	// "rttime": maximum amount of real-time execution between blocking syscalls
	// "sigpending": maximum number of pending signals (ulimit -i)
	// "stack": maximum stack size (ulimit -s)
	Ulimit []string
	// Volumes to bind mount into the container
	Volumes []string
	// Secrets are the available secrets to use in a build.  Each item in the
	// slice takes the form "id=foo,src=bar", where both "id" and "src" are
	// required, in that order, and "bar" is the name of a file.
	Secrets []string
	// SSHSources is the available ssh agent connections to forward in the build
	SSHSources []string
}

// BuildOptions can be used to alter how an image is built.
type BuildOptions struct {
	// ContextDirectory is the default source location for COPY and ADD
	// commands.
	ContextDirectory string
	// PullPolicy controls whether or not we pull images.  It should be one
	// of PullIfMissing, PullAlways, PullIfNewer, or PullNever.
	PullPolicy PullPolicy
	// Registry is a value which is prepended to the image's name, if it
	// needs to be pulled and the image name alone can not be resolved to a
	// reference to a source image.  No separator is implicitly added.
	Registry string
	// IgnoreUnrecognizedInstructions tells us to just log instructions we
	// don't recognize, and try to keep going.
	IgnoreUnrecognizedInstructions bool
	// Manifest Name to which the image will be added.
	Manifest string
	// Quiet tells us whether or not to announce steps as we go through them.
	Quiet bool
	// Isolation controls how Run() runs things.
	Isolation Isolation
	// Runtime is the name of the command to run for RUN instructions when
	// Isolation is either IsolationDefault or IsolationOCI.  It should
	// accept the same arguments and flags that runc does.
	Runtime string
	// RuntimeArgs adds global arguments for the runtime.
	RuntimeArgs []string
	// TransientMounts is a list of mounts that won't be kept in the image.
	TransientMounts []string
	// Compression specifies the type of compression which is applied to
	// layer blobs.  The default is to not use compression, but
	// archive.Gzip is recommended.
	Compression archive.Compression
	// Arguments which can be interpolated into Dockerfiles
	Args map[string]string
	// Name of the image to write to.
	Output string
	// Additional tags to add to the image that we write, if we know of a
	// way to add them.
	AdditionalTags []string
	// Log is a callback that will print a progress message.  If no value
	// is supplied, the message will be sent to Err (or os.Stderr, if Err
	// is nil) by default.
	Log func(format string, args ...interface{})
	// In is connected to stdin for RUN instructions.
	In io.Reader
	// Out is a place where non-error log messages are sent.
	Out io.Writer
	// Err is a place where error log messages should be sent.
	Err io.Writer
	// SignaturePolicyPath specifies an override location for the signature
	// policy which should be used for verifying the new image as it is
	// being written.  Except in specific circumstances, no value should be
	// specified, indicating that the shared, system-wide default policy
	// should be used.
	SignaturePolicyPath string
	// ReportWriter is an io.Writer which will be used to report the
	// progress of the (possible) pulling of the source image and the
	// writing of the new image.
	ReportWriter io.Writer
	// OutputFormat is the format of the output image's manifest and
	// configuration data.
	// Accepted values are buildah.OCIv1ImageManifest and buildah.Dockerv2ImageManifest.
	OutputFormat string
	// SystemContext holds parameters used for authentication.
	SystemContext *types.SystemContext
	// NamespaceOptions controls how we set up namespaces processes that we
	// might need when handling RUN instructions.
	NamespaceOptions []NamespaceOption
	// ConfigureNetwork controls whether or not network interfaces and
	// routing are configured for a new network namespace (i.e., when not
	// joining another's namespace and not just using the host's
	// namespace), effectively deciding whether or not the process has a
	// usable network.
	ConfigureNetwork NetworkConfigurationPolicy
	// CNIPluginPath is the location of CNI plugin helpers, if they should be
	// run from a location other than the default location.
	CNIPluginPath string
	// CNIConfigDir is the location of CNI configuration files, if the files in
	// the default configuration directory shouldn't be used.
	CNIConfigDir string
	// ID mapping options to use if we're setting up our own user namespace
	// when handling RUN instructions.
	IDMappingOptions *IDMappingOptions
	// AddCapabilities is a list of capabilities to add to the default set when
	// handling RUN instructions.
	AddCapabilities []string
	// DropCapabilities is a list of capabilities to remove from the default set
	// when handling RUN instructions. If a capability appears in both lists, it
	// will be dropped.
	DropCapabilities []string
	// CommonBuildOpts is *required*.
	CommonBuildOpts *CommonBuildOptions
	// DefaultMountsFilePath is the file path holding the mounts to be mounted in "host-path:container-path" format
	DefaultMountsFilePath string
	// IIDFile tells the builder to write the image ID to the specified file
	IIDFile string
	// Squash tells the builder to produce an image with a single layer
	// instead of with possibly more than one layer.
	Squash bool
	// Labels metadata for an image
	Labels []string
	// Annotation metadata for an image
	Annotations []string
	// OnBuild commands to be run by images based on this image
	OnBuild []string
	// Layers tells the builder to create a cache of images for each step in the Dockerfile
	Layers bool
	// NoCache tells the builder to build the image from scratch without checking for a cache.
	// It creates a new set of cached images for the build.
	NoCache bool
	// RemoveIntermediateCtrs tells the builder whether to remove intermediate containers used
	// during the build process. Default is true.
	RemoveIntermediateCtrs bool
	// ForceRmIntermediateCtrs tells the builder to remove all intermediate containers even if
	// the build was unsuccessful.
	ForceRmIntermediateCtrs bool
	// BlobDirectory is a directory which we'll use for caching layer blobs.
	BlobDirectory string
	// Target the targeted FROM in the Dockerfile to build.
	Target string
	// Devices are the additional devices to add to the containers.
	Devices []string
	// SignBy is the fingerprint of a GPG key to use for signing images.
	SignBy string
	// Architecture specifies the target architecture of the image to be built.
	Architecture string
	// Timestamp sets the created timestamp to the specified time, allowing
	// for deterministic, content-addressable builds.
	Timestamp *time.Time
	// OS is the specifies the operating system of the image to be built.
	OS string
	// MaxPullPushRetries is the maximum number of attempts we'll make to pull or push any one
	// image from or to an external registry if the first attempt fails.
	MaxPullPushRetries int
	// PullPushRetryDelay is how long to wait before retrying a pull or push attempt.
	PullPushRetryDelay time.Duration
	// OciDecryptConfig contains the config that can be used to decrypt an image if it is
	// encrypted if non-nil. If nil, it does not attempt to decrypt an image.
	OciDecryptConfig *encconfig.DecryptConfig
	// Jobs is the number of stages to run in parallel.  If not specified it defaults to 1.
	// Ignored if a JobSemaphore is provided.
	Jobs *int
	// JobSemaphore, for when you want Jobs to be shared with more than just this build.
	JobSemaphore *semaphore.Weighted
	// LogRusage logs resource usage for each step.
	LogRusage bool
	// File to which the Rusage logs will be saved to instead of stdout
	RusageLogFile string
	// Excludes is a list of excludes to be used instead of the .dockerignore file.
	Excludes []string
	// IgnoreFile is a name of the .containerignore file
	IgnoreFile string
	// From is the image name to use to replace the value specified in the first
	// FROM instruction in the Containerfile
	From string
	// Platforms is the list of parsed OS/Arch/Variant triples that we want
	// to build the image for.  If this slice has items in it, the OS and
	// Architecture fields above are ignored.
	Platforms []struct{ OS, Arch, Variant string }
	// AllPlatforms tells the builder to set the list of target platforms
	// to match the set of platforms for which all of the build's base
	// images are available.  If this field is set, Platforms is ignored.
	AllPlatforms bool
	// UnsetEnvs is a list of environments to not add to final image.
	UnsetEnvs []string
}
<|endoftext|>


<commit_before>package bitmask

import "testing"

func TestBitMask(t *testing.T) {
	tests := []struct {
		pos     int
		data    bool
		recheck bool
	}{
		{0, true, true},
		{1, true, true},
		{2, true, true},
		{3, false, false},
		{4, true, true},
		{5, true, false},
		{6, true, true},
		{7, true, true},
		{3, true, true},
		{5, false, true},
		{8, false, true},
		{9, true, true},
		{15, true, false},
		{15, false, true},
		{16, true, false},
		{16, false, false},
		{16, false, true},
		{31, true, true},
		{30, true, true},
		{29, true, true},
		{24, true, true},
	}
	b := New(32)
	for n, test := range tests {
		b.Set(test.pos, test.data)
		if got := b.Get(test.pos); got != test.data {
			t.Errorf("test 1-%d: expecting %v, got %v", n+1, test.data, got)
		}
	}
	for n, test := range tests {
		if !test.recheck {
			continue
		}
		if got := b.Get(test.pos); got != test.data {
			t.Errorf("test 2-%d: expecting %v, got %v", n+1, test.data, got)
		}
	}
}

func TestSetIfNot(t *testing.T) {
	tests := []struct {
		pos  int
		data bool
		ret  bool
	}{
		{0, true, true},
		{1, true, true},
		{2, true, true},
		{3, false, false},
		{4, true, true},
		{5, true, true},
		{6, true, true},
		{7, true, true},
		{3, true, true},
		{5, false, true},
		{8, false, false},
		{9, true, true},
		{15, true, true},
		{15, false, true},
		{16, true, true},
		{16, false, true},
		{16, false, false},
		{31, true, true},
		{30, true, true},
		{29, true, true},
		{24, true, true},
	}
	b := New(32)
	for n, test := range tests {
		if got := b.SetIfNot(test.pos, test.data); got != test.ret {
			t.Errorf("test d: expecting %v, got %v", n+1, test.ret, got)
		}
	}
}
<commit_msg>corrected test output formatting<commit_after>package bitmask

import "testing"

func TestBitMask(t *testing.T) {
	tests := []struct {
		pos     int
		data    bool
		recheck bool
	}{
		{0, true, true},
		{1, true, true},
		{2, true, true},
		{3, false, false},
		{4, true, true},
		{5, true, false},
		{6, true, true},
		{7, true, true},
		{3, true, true},
		{5, false, true},
		{8, false, true},
		{9, true, true},
		{15, true, false},
		{15, false, true},
		{16, true, false},
		{16, false, false},
		{16, false, true},
		{31, true, true},
		{30, true, true},
		{29, true, true},
		{24, true, true},
	}
	b := New(32)
	for n, test := range tests {
		b.Set(test.pos, test.data)
		if got := b.Get(test.pos); got != test.data {
			t.Errorf("test 1-%d: expecting %v, got %v", n+1, test.data, got)
		}
	}
	for n, test := range tests {
		if !test.recheck {
			continue
		}
		if got := b.Get(test.pos); got != test.data {
			t.Errorf("test 2-%d: expecting %v, got %v", n+1, test.data, got)
		}
	}
}

func TestSetIfNot(t *testing.T) {
	tests := []struct {
		pos  int
		data bool
		ret  bool
	}{
		{0, true, true},
		{1, true, true},
		{2, true, true},
		{3, false, false},
		{4, true, true},
		{5, true, true},
		{6, true, true},
		{7, true, true},
		{3, true, true},
		{5, false, true},
		{8, false, false},
		{9, true, true},
		{15, true, true},
		{15, false, true},
		{16, true, true},
		{16, false, true},
		{16, false, false},
		{31, true, true},
		{30, true, true},
		{29, true, true},
		{24, true, true},
	}
	b := New(32)
	for n, test := range tests {
		if got := b.SetIfNot(test.pos, test.data); got != test.ret {
			t.Errorf("test %d: expecting %v, got %v", n+1, test.ret, got)
		}
	}
}
<|endoftext|>


<commit_before>package bkdtree

import (
	"testing"
)

func TestBkdInsert(t *testing.T) {
	t0mCap := 1000
	treesCap := 5
	bkdCap := t0mCap<<uint(treesCap) - 1
	numDims := 2
	bytesPerDim := 4
	leafCap := 50
	intraCap := 4
	dir := "/tmp"
	prefix := "bkd"
	bkd := NewBkdTree(t0mCap, bkdCap, numDims, bytesPerDim, leafCap, intraCap, dir, prefix)
	if bkd == nil {
		t.Fatalf("bkd is nil")
	}

	maxVal := uint64(1000)
	size := bkdCap + 1
	points := NewRandPoints(numDims, maxVal, size)

	//insert points
	for i := 0; i < bkdCap; i++ {
		err := bkd.Insert(points[i])
		if err != nil {
			t.Fatalf("bkd.Insert failed, i=%v, err: %v", i, err)
		}
		if bkd.NumPoints != i+1 {
			t.Fatalf("incorrect numPoints. numPoints=%v, i=%v", bkd.NumPoints, i)
		}
		remained := bkd.NumPoints % bkd.t0mCap
		quotient := bkd.NumPoints / bkd.t0mCap
		if len(bkd.t0m) != remained {
			t.Fatalf("bkd.numPoints %d, len(bkd.t0m) %d is incorect, want %d", bkd.NumPoints, len(bkd.t0m), remained)
		}
		for i := 0; i < len(bkd.trees); i++ {
			tiCap := bkd.t0mCap << uint(i)
			want := tiCap * (quotient % 2)
			if bkd.trees[i].numPoints != uint64(want) {
				t.Fatalf("bkd.numPoints %d, bkd.tree[%d].numPoints %d is incorrect, want %d", bkd.NumPoints, i, bkd.trees[i].numPoints, want)
			}
			quotient >>= 1
		}
	}
	err := bkd.Insert(points[bkdCap])
	if err == nil {
		t.Fatalf("bkd.Insert shall fail if tree is full")
	}
}

func prepareBkdTree(t *testing.T, maxVal uint64) (bkd *BkdTree, points []Point) {
	t0mCap := 1000
	treesCap := 5
	bkdCap := t0mCap<<uint(treesCap) - 1
	numDims := 2
	bytesPerDim := 4
	leafCap := 50
	intraCap := 4
	dir := "/tmp"
	prefix := "bkd"
	bkd = NewBkdTree(t0mCap, bkdCap, numDims, bytesPerDim, leafCap, intraCap, dir, prefix)
	if bkd == nil {
		t.Fatalf("bkd is nil")
	}

	size := bkdCap
	points = NewRandPoints(numDims, maxVal, size)
	for i := 0; i < bkdCap; i++ {
		err := bkd.Insert(points[i])
		if err != nil {
			t.Fatalf("bkd.Insert failed, i=%v, err: %v", i, err)
		}
	}
	return
}

func TestBkdIntersect(t *testing.T) {
	var maxVal uint64 = 1000
	bkd, points := prepareBkdTree(t, maxVal)
	var lowPoint, highPoint Point
	var visitor *IntersectCollector
	var err error

	//some intersect
	lowPoint = points[7]
	highPoint = lowPoint
	visitor = &IntersectCollector{lowPoint, highPoint, make([]Point, 0)}
	err = bkd.Intersect(visitor)
	if err != nil {
		t.Fatalf("%v", err)
	}
	if len(visitor.points) <= 0 {
		t.Errorf("found 0 matchs, however some expected")
	}
	var matched int
	for _, point := range visitor.points {
		isInside := point.Inside(lowPoint, highPoint)
		if !isInside {
			t.Errorf("point %v is ouside of range", point)
		}
		if point.Equal(lowPoint) {
			matched++
		}
	}
	if matched != 1 {
		t.Errorf("found %d points equal to %v", matched, lowPoint)
	}

	//all intersect
	lowPoint = Point{[]uint64{0, 0, 0}, 0}
	highPoint = Point{[]uint64{maxVal, maxVal, maxVal}, 0}
	visitor = &IntersectCollector{lowPoint, highPoint, make([]Point, 0)}
	err = bkd.Intersect(visitor)
	if err != nil {
		t.Fatalf("%v", err)
	}
	if len(visitor.points) != len(points) {
		t.Errorf("found %d matchs, want %d", len(visitor.points), len(points))
	}
}

func countPoint(bkd *BkdTree, point Point) (cnt int, err error) {
	lowPoint, highPoint := point, point
	visitor := &IntersectCollector{lowPoint, highPoint, make([]Point, 0)}
	err = bkd.Intersect(visitor)
	if err != nil {
		return
	}
	for _, p := range visitor.points {
		if p.Equal(point) {
			cnt++
		}
	}
	return
}

func TestBkdErase(t *testing.T) {
	var maxVal uint64 = 1000
	bkd, points := prepareBkdTree(t, maxVal)
	var target Point
	var cnt int
	var found bool
	var err error

	//erase an non-existing point
	target = points[17]
	target.UserData = uint64(len(points))
	found, err = bkd.Erase(target)
	if err != nil {
		t.Fatalf("%v", err)
	} else if found {
		t.Fatalf("point %v found, want non-existing", target)
	}

	//erase an existing point, verify the point really erased.
	target = points[13]
	found, err = bkd.Erase(target)
	if err != nil {
		t.Fatalf("%v", err)
	} else if !found {
		t.Fatalf("point %v not found", target)
	} else if bkd.NumPoints != len(points)-1 {
		t.Fatalf("incorrect bkd.numPoints %d, want %d", bkd.NumPoints, len(points)-1)
	}
	cnt, err = countPoint(bkd, target)
	if err != nil {
		t.Fatalf("%v", err)
	} else if cnt != 0 {
		t.Errorf("point %v still exists", target)
	}

	//there's room for insertion
	err = bkd.Insert(target)
	if err != nil {
		t.Fatalf("bkd.Insert failed, err: %v", err)
	} else if bkd.NumPoints != len(points) {
		t.Fatalf("incorrect bkd.numPoints %d, want %d", bkd.NumPoints, len(points))
	}
	cnt, err = countPoint(bkd, target)
	if err != nil {
		t.Fatalf("%v", err)
	} else if cnt != 1 {
		t.Errorf("point %v still exists", target)
	}
}
<commit_msg>benchmark<commit_after>package bkdtree

import (
	"testing"

	"github.com/pkg/errors"
)

func TestBkdInsert(t *testing.T) {
	t0mCap := 1000
	treesCap := 5
	bkdCap := t0mCap<<uint(treesCap) - 1
	numDims := 2
	bytesPerDim := 4
	leafCap := 50
	intraCap := 4
	dir := "/tmp"
	prefix := "bkd"
	bkd := NewBkdTree(t0mCap, bkdCap, numDims, bytesPerDim, leafCap, intraCap, dir, prefix)
	if bkd == nil {
		t.Fatalf("bkd is nil")
	}

	maxVal := uint64(1000)
	size := bkdCap + 1
	points := NewRandPoints(numDims, maxVal, size)

	//insert points
	for i := 0; i < bkdCap; i++ {
		err := bkd.Insert(points[i])
		if err != nil {
			t.Fatalf("bkd.Insert failed, i=%v, err: %v", i, err)
		}
		if bkd.NumPoints != i+1 {
			t.Fatalf("incorrect numPoints. numPoints=%v, i=%v", bkd.NumPoints, i)
		}
		remained := bkd.NumPoints % bkd.t0mCap
		quotient := bkd.NumPoints / bkd.t0mCap
		if len(bkd.t0m) != remained {
			t.Fatalf("bkd.numPoints %d, len(bkd.t0m) %d is incorect, want %d", bkd.NumPoints, len(bkd.t0m), remained)
		}
		for i := 0; i < len(bkd.trees); i++ {
			tiCap := bkd.t0mCap << uint(i)
			want := tiCap * (quotient % 2)
			if bkd.trees[i].numPoints != uint64(want) {
				t.Fatalf("bkd.numPoints %d, bkd.tree[%d].numPoints %d is incorrect, want %d", bkd.NumPoints, i, bkd.trees[i].numPoints, want)
			}
			quotient >>= 1
		}
	}
	err := bkd.Insert(points[bkdCap])
	if err == nil {
		t.Fatalf("bkd.Insert shall fail if tree is full")
	}
}

func prepareBkdTree(maxVal uint64) (bkd *BkdTree, points []Point, err error) {
	t0mCap := 1000
	treesCap := 5
	bkdCap := t0mCap<<uint(treesCap) - 1
	numDims := 2
	bytesPerDim := 4
	leafCap := 50
	intraCap := 4
	dir := "/tmp"
	prefix := "bkd"
	bkd = NewBkdTree(t0mCap, bkdCap, numDims, bytesPerDim, leafCap, intraCap, dir, prefix)
	if bkd == nil {
		err = errors.Errorf("bkd is nil")
		return
	}
	//fmt.Printf("created BkdTree %v\n", bkd)

	size := bkdCap
	points = NewRandPoints(numDims, maxVal, size)
	for i := 0; i < bkdCap; i++ {
		err = bkd.Insert(points[i])
		if err != nil {
			err = errors.Errorf("bkd.Insert failed, i=%v, err: %v", i, err)
		}
	}
	return
}

func TestBkdIntersect(t *testing.T) {
	var maxVal uint64 = 1000
	bkd, points, err := prepareBkdTree(maxVal)
	if err != nil {
		t.Fatalf("%+v", err)
	}
	var lowPoint, highPoint Point
	var visitor *IntersectCollector

	//some intersect
	lowPoint = points[7]
	highPoint = lowPoint
	visitor = &IntersectCollector{lowPoint, highPoint, make([]Point, 0)}
	err = bkd.Intersect(visitor)
	if err != nil {
		t.Fatalf("%+v", err)
	}
	if len(visitor.points) <= 0 {
		t.Errorf("found 0 matchs, however some expected")
	}
	var matched int
	for _, point := range visitor.points {
		isInside := point.Inside(lowPoint, highPoint)
		if !isInside {
			t.Errorf("point %v is ouside of range", point)
		}
		if point.Equal(lowPoint) {
			matched++
		}
	}
	if matched != 1 {
		t.Errorf("found %d points equal to %v", matched, lowPoint)
	}

	//all intersect
	lowPoint = Point{[]uint64{0, 0, 0}, 0}
	highPoint = Point{[]uint64{maxVal, maxVal, maxVal}, 0}
	visitor = &IntersectCollector{lowPoint, highPoint, make([]Point, 0)}
	err = bkd.Intersect(visitor)
	if err != nil {
		t.Fatalf("%+v", err)
	}
	if len(visitor.points) != len(points) {
		t.Errorf("found %d matchs, want %d", len(visitor.points), len(points))
	}
}

func countPoint(bkd *BkdTree, point Point) (cnt int, err error) {
	lowPoint, highPoint := point, point
	visitor := &IntersectCollector{lowPoint, highPoint, make([]Point, 0)}
	err = bkd.Intersect(visitor)
	if err != nil {
		return
	}
	for _, p := range visitor.points {
		if p.Equal(point) {
			cnt++
		}
	}
	return
}

func TestBkdErase(t *testing.T) {
	var maxVal uint64 = 1000
	bkd, points, err := prepareBkdTree(maxVal)
	if err != nil {
		t.Fatalf("%+v", err)
	}
	var target Point
	var cnt int
	var found bool

	//erase an non-existing point
	target = points[17]
	target.UserData = uint64(len(points))
	found, err = bkd.Erase(target)
	if err != nil {
		t.Fatalf("%v", err)
	} else if found {
		t.Fatalf("point %v found, want non-existing", target)
	}

	//erase an existing point, verify the point really erased.
	target = points[13]
	found, err = bkd.Erase(target)
	if err != nil {
		t.Fatalf("%v", err)
	} else if !found {
		t.Fatalf("point %v not found", target)
	} else if bkd.NumPoints != len(points)-1 {
		t.Fatalf("incorrect bkd.numPoints %d, want %d", bkd.NumPoints, len(points)-1)
	}
	cnt, err = countPoint(bkd, target)
	if err != nil {
		t.Fatalf("%v", err)
	} else if cnt != 0 {
		t.Errorf("point %v still exists", target)
	}

	//there's room for insertion
	err = bkd.Insert(target)
	if err != nil {
		t.Fatalf("bkd.Insert failed, err: %v", err)
	} else if bkd.NumPoints != len(points) {
		t.Fatalf("incorrect bkd.numPoints %d, want %d", bkd.NumPoints, len(points))
	}
	cnt, err = countPoint(bkd, target)
	if err != nil {
		t.Fatalf("%v", err)
	} else if cnt != 1 {
		t.Errorf("point %v still exists", target)
	}
}

func BenchmarkBkdInsert(b *testing.B) {
	t0mCap := 1000
	treesCap := 20
	bkdCap := t0mCap<<uint(treesCap) - 1
	numDims := 2
	bytesPerDim := 4
	leafCap := 50
	intraCap := 4
	dir := "/tmp"
	prefix := "bkd"
	bkd := NewBkdTree(t0mCap, bkdCap, numDims, bytesPerDim, leafCap, intraCap, dir, prefix)
	if bkd == nil {
		b.Fatalf("bkd is nil")
	}
	//fmt.Printf("created BkdTree %v\n", bkd)

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		err := bkd.Insert(Point{[]uint64{uint64(i), uint64(i)}, uint64(i)})
		if err != nil {
			b.Fatalf("bkd.Insert failed, i=%v, err: %v", i, err)
		}
	}
	return
}

func BenchmarkBkdErase(b *testing.B) {
	var maxVal uint64 = 1000
	bkd, points, err := prepareBkdTree(maxVal)
	if err != nil {
		b.Fatalf("%+v", err)
	}

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		_, err = bkd.Erase(points[i])
		if err != nil {
			b.Fatalf("%v", err)
		}
	}
}

func BenchmarkBkdIntersect(b *testing.B) {
	var maxVal uint64 = 1000
	bkd, points, err := prepareBkdTree(maxVal)
	if err != nil {
		b.Fatalf("%+v", err)
	}
	var lowPoint, highPoint Point
	var visitor *IntersectCollector
	lowPoint = points[7]
	highPoint = lowPoint
	visitor = &IntersectCollector{lowPoint, highPoint, make([]Point, 0)}

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		err = bkd.Intersect(visitor)
		if err != nil {
			b.Fatalf("%v", err)
		} else if len(visitor.points) <= 0 {
			b.Errorf("found 0 matchs, however some expected")
		}
	}
}
<|endoftext|>


<commit_before>// Use and distribution licensed under the Apache license version 2.
//
// See the COPYING file in the root project directory for full text.
//

package ghw

import (
	"fmt"
	"os"
	"os/exec"
	"strings"

	"github.com/pkg/errors"
	"howett.net/plist"
)

type diskOrPartitionPlistNode struct {
	Content          string
	DeviceIdentifier string
	DiskUUID         string
	VolumeName       string
	VolumeUUID       string
	Size             int64
	MountPoint       string
	Partitions       []diskOrPartitionPlistNode
	APFSVolumes      []diskOrPartitionPlistNode
}

type diskUtilListPlist struct {
	AllDisks              []string
	AllDisksAndPartitions []diskOrPartitionPlistNode
	VolumesFromDisks      []string
	WholeDisks            []string
}

type diskUtilInfoPlist struct {
	AESHardware                                 bool   // true
	Bootable                                    bool   // true
	BooterDeviceIdentifier                      string // disk1s2
	BusProtocol                                 string // PCI-Express
	CanBeMadeBootable                           bool   // false
	CanBeMadeBootableRequiresDestroy            bool   // false
	Content                                     string // some-uuid-foo-bar
	DeviceBlockSize                             int64  // 4096
	DeviceIdentifier                            string // disk1s1
	DeviceNode                                  string // /dev/disk1s1
	DeviceTreePath                              string // IODeviceTree:/PCI0@0/RP17@1B/ANS2@0/AppleANS2Controller
	DiskUUID                                    string // some-uuid-foo-bar
	Ejectable                                   bool   // false
	EjectableMediaAutomaticUnderSoftwareControl bool   // false
	EjectableOnly                               bool   // false
	FilesystemName                              string // APFS
	FilesystemType                              string // apfs
	FilesystemUserVisibleName                   string // APFS
	FreeSpace                                   int64  // 343975677952
	GlobalPermissionsEnabled                    bool   // true
	IOKitSize                                   int64  // 499963174912
	IORegistryEntryName                         string // Macintosh HD
	Internal                                    bool   // true
	MediaName                                   string //
	MediaType                                   string // Generic
	MountPoint                                  string // /
	ParentWholeDisk                             string // disk1
	PartitionMapPartition                       bool   // false
	RAIDMaster                                  bool   // false
	RAIDSlice                                   bool   // false
	RecoveryDeviceIdentifier                    string // disk1s3
	Removable                                   bool   // false
	RemovableMedia                              bool   // false
	RemovableMediaOrExternalDevice              bool   // false
	SMARTStatus                                 string // Verified
	Size                                        int64  // 499963174912
	SolidState                                  bool   // true
	SupportsGlobalPermissionsDisable            bool   // true
	SystemImage                                 bool   // false
	TotalSize                                   int64  // 499963174912
	VolumeAllocationBlockSize                   int64  // 4096
	VolumeName                                  string // Macintosh HD
	VolumeSize                                  int64  // 499963174912
	VolumeUUID                                  string // some-uuid-foo-bar
	WholeDisk                                   bool   // false
	Writable                                    bool   // true
	WritableMedia                               bool   // true
	WritableVolume                              bool   // true
	// also has a SMARTDeviceSpecificKeysMayVaryNotGuaranteed dict with various info
	// NOTE: VolumeUUID sometimes == DiskUUID, but not always. So far Content is always a different UUID.
}

type ioregPlist struct {
	// there's a lot more than just this...
	ModelNumber  string `plist:"Model Number"`
	SerialNumber string `plist:"Serial Number"`
	VendorName   string `plist:"Vendor Name"`
}

func (ctx *context) getDiskUtilListPlist() (*diskUtilListPlist, error) {
	out, err := exec.Command("diskutil", "list", "-plist").Output()
	if err != nil {
		return nil, errors.Wrap(err, "diskutil list failed")
	}

	var data diskUtilListPlist
	if _, err := plist.Unmarshal(out, &data); err != nil {
		return nil, errors.Wrap(err, "diskutil list plist unmarshal failed")
	}

	return &data, nil
}

func (ctx *context) getDiskUtilInfoPlist(device string) (*diskUtilInfoPlist, error) {
	out, err := exec.Command("diskutil", "info", "-plist", device).Output()
	if err != nil {
		return nil, errors.Wrapf(err, "diskutil info for %q failed", device)
	}

	var data diskUtilInfoPlist
	if _, err := plist.Unmarshal(out, &data); err != nil {
		return nil, errors.Wrapf(err, "diskutil info plist unmarshal for %q failed", device)
	}

	return &data, nil
}

func (ctx *context) getIoregPlist(ioDeviceTreePath string) (*ioregPlist, error) {
	elems := strings.Split(ioDeviceTreePath, "/")
	name := elems[len(elems)-1]

	args := []string{
		"ioreg",
		"-a",      // use XML output
		"-d", "1", // limit device tree output depth to root node
		"-r",       // root device tree at matched node
		"-n", name, // match by name
	}
	out, err := exec.Command(args[0], args[1:]...).Output()
	if err != nil {
		return nil, errors.Wrapf(err, "ioreg query for %q failed", ioDeviceTreePath)
	}

	var data []ioregPlist
	if _, err := plist.Unmarshal(out, &data); err != nil {
		return nil, errors.Wrapf(err, "ioreg unmarshal for %q failed", ioDeviceTreePath)
	}
	if len(data) != 1 {
		err := errors.Errorf("ioreg unmarshal resulted in %d I/O device tree nodes (expected 1)", len(data))
		return nil, err
	}

	return &data[0], nil
}

func (ctx *context) makePartition(disk, s diskOrPartitionPlistNode, isAPFS bool) (*Partition, error) {
	if s.Size < 0 {
		return nil, errors.Errorf("invalid size %q of partition %q", s.Size, s.DeviceIdentifier)
	}

	var partType string
	if isAPFS {
		partType = "APFS Volume"
	} else {
		partType = s.Content
	}

	info, err := ctx.getDiskUtilInfoPlist(s.DeviceIdentifier)
	if err != nil {
		return nil, err
	}

	return &Partition{
		Disk:       nil, // filled in later
		Name:       s.DeviceIdentifier,
		Label:      s.VolumeName,
		MountPoint: s.MountPoint,
		SizeBytes:  uint64(s.Size),
		Type:       partType,
		IsReadOnly: !info.WritableVolume,
	}, nil
}

func (ctx *context) blockFillInfo(info *BlockInfo) error {
	listPlist, err := ctx.getDiskUtilListPlist()
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		return err
	}

	info.TotalPhysicalBytes = 0
	info.Disks = make([]*Disk, 0, len(listPlist.AllDisksAndPartitions))
	info.Partitions = []*Partition{}

	for _, disk := range listPlist.AllDisksAndPartitions {
		if disk.Size < 0 {
			return errors.Errorf("invalid size %q of disk %q", disk.Size, disk.DeviceIdentifier)
		}

		infoPlist, err := ctx.getDiskUtilInfoPlist(disk.DeviceIdentifier)
		if err != nil {
			return err
		}
		if infoPlist.DeviceBlockSize < 0 {
			return errors.Errorf("invalid block size %q of disk %q", infoPlist.DeviceBlockSize, disk.DeviceIdentifier)
		}

		busPath := strings.TrimPrefix(infoPlist.DeviceTreePath, "IODeviceTree:")

		ioregPlist, err := ctx.getIoregPlist(infoPlist.DeviceTreePath)
		if err != nil {
			return err
		}

		// The NUMA node & WWN don't seem to be reported by any tools available by default in macOS.
		diskReport := &Disk{
			Name:                   disk.DeviceIdentifier,
			SizeBytes:              uint64(disk.Size),
			PhysicalBlockSizeBytes: uint64(infoPlist.DeviceBlockSize),
			BusType:                infoPlist.BusProtocol,
			BusPath:                busPath,
			NUMANodeID:             -1,
			Vendor:                 ioregPlist.VendorName,
			Model:                  ioregPlist.ModelNumber,
			SerialNumber:           ioregPlist.SerialNumber,
			WWN:                    "",
			Partitions:             make([]*Partition, 0, len(disk.Partitions)+len(disk.APFSVolumes)),
		}

		for _, partition := range disk.Partitions {
			part, err := ctx.makePartition(disk, partition, false)
			if err != nil {
				return err
			}
			part.Disk = diskReport
			diskReport.Partitions = append(diskReport.Partitions, part)
		}
		for _, volume := range disk.APFSVolumes {
			part, err := ctx.makePartition(disk, volume, true)
			if err != nil {
				return err
			}
			part.Disk = diskReport
			diskReport.Partitions = append(diskReport.Partitions, part)
		}

		info.TotalPhysicalBytes += uint64(disk.Size)
		info.Disks = append(info.Disks, diskReport)
		info.Partitions = append(info.Partitions, diskReport.Partitions...)
	}

	return nil
}
<commit_msg>Use path.Base() to split IOKit device tree path<commit_after>// Use and distribution licensed under the Apache license version 2.
//
// See the COPYING file in the root project directory for full text.
//

package ghw

import (
	"fmt"
	"os"
	"os/exec"
	"path"
	"strings"

	"github.com/pkg/errors"
	"howett.net/plist"
)

type diskOrPartitionPlistNode struct {
	Content          string
	DeviceIdentifier string
	DiskUUID         string
	VolumeName       string
	VolumeUUID       string
	Size             int64
	MountPoint       string
	Partitions       []diskOrPartitionPlistNode
	APFSVolumes      []diskOrPartitionPlistNode
}

type diskUtilListPlist struct {
	AllDisks              []string
	AllDisksAndPartitions []diskOrPartitionPlistNode
	VolumesFromDisks      []string
	WholeDisks            []string
}

type diskUtilInfoPlist struct {
	AESHardware                                 bool   // true
	Bootable                                    bool   // true
	BooterDeviceIdentifier                      string // disk1s2
	BusProtocol                                 string // PCI-Express
	CanBeMadeBootable                           bool   // false
	CanBeMadeBootableRequiresDestroy            bool   // false
	Content                                     string // some-uuid-foo-bar
	DeviceBlockSize                             int64  // 4096
	DeviceIdentifier                            string // disk1s1
	DeviceNode                                  string // /dev/disk1s1
	DeviceTreePath                              string // IODeviceTree:/PCI0@0/RP17@1B/ANS2@0/AppleANS2Controller
	DiskUUID                                    string // some-uuid-foo-bar
	Ejectable                                   bool   // false
	EjectableMediaAutomaticUnderSoftwareControl bool   // false
	EjectableOnly                               bool   // false
	FilesystemName                              string // APFS
	FilesystemType                              string // apfs
	FilesystemUserVisibleName                   string // APFS
	FreeSpace                                   int64  // 343975677952
	GlobalPermissionsEnabled                    bool   // true
	IOKitSize                                   int64  // 499963174912
	IORegistryEntryName                         string // Macintosh HD
	Internal                                    bool   // true
	MediaName                                   string //
	MediaType                                   string // Generic
	MountPoint                                  string // /
	ParentWholeDisk                             string // disk1
	PartitionMapPartition                       bool   // false
	RAIDMaster                                  bool   // false
	RAIDSlice                                   bool   // false
	RecoveryDeviceIdentifier                    string // disk1s3
	Removable                                   bool   // false
	RemovableMedia                              bool   // false
	RemovableMediaOrExternalDevice              bool   // false
	SMARTStatus                                 string // Verified
	Size                                        int64  // 499963174912
	SolidState                                  bool   // true
	SupportsGlobalPermissionsDisable            bool   // true
	SystemImage                                 bool   // false
	TotalSize                                   int64  // 499963174912
	VolumeAllocationBlockSize                   int64  // 4096
	VolumeName                                  string // Macintosh HD
	VolumeSize                                  int64  // 499963174912
	VolumeUUID                                  string // some-uuid-foo-bar
	WholeDisk                                   bool   // false
	Writable                                    bool   // true
	WritableMedia                               bool   // true
	WritableVolume                              bool   // true
	// also has a SMARTDeviceSpecificKeysMayVaryNotGuaranteed dict with various info
	// NOTE: VolumeUUID sometimes == DiskUUID, but not always. So far Content is always a different UUID.
}

type ioregPlist struct {
	// there's a lot more than just this...
	ModelNumber  string `plist:"Model Number"`
	SerialNumber string `plist:"Serial Number"`
	VendorName   string `plist:"Vendor Name"`
}

func (ctx *context) getDiskUtilListPlist() (*diskUtilListPlist, error) {
	out, err := exec.Command("diskutil", "list", "-plist").Output()
	if err != nil {
		return nil, errors.Wrap(err, "diskutil list failed")
	}

	var data diskUtilListPlist
	if _, err := plist.Unmarshal(out, &data); err != nil {
		return nil, errors.Wrap(err, "diskutil list plist unmarshal failed")
	}

	return &data, nil
}

func (ctx *context) getDiskUtilInfoPlist(device string) (*diskUtilInfoPlist, error) {
	out, err := exec.Command("diskutil", "info", "-plist", device).Output()
	if err != nil {
		return nil, errors.Wrapf(err, "diskutil info for %q failed", device)
	}

	var data diskUtilInfoPlist
	if _, err := plist.Unmarshal(out, &data); err != nil {
		return nil, errors.Wrapf(err, "diskutil info plist unmarshal for %q failed", device)
	}

	return &data, nil
}

func (ctx *context) getIoregPlist(ioDeviceTreePath string) (*ioregPlist, error) {
	name := path.Base(ioDeviceTreePath)

	args := []string{
		"ioreg",
		"-a",      // use XML output
		"-d", "1", // limit device tree output depth to root node
		"-r",       // root device tree at matched node
		"-n", name, // match by name
	}
	out, err := exec.Command(args[0], args[1:]...).Output()
	if err != nil {
		return nil, errors.Wrapf(err, "ioreg query for %q failed", ioDeviceTreePath)
	}

	var data []ioregPlist
	if _, err := plist.Unmarshal(out, &data); err != nil {
		return nil, errors.Wrapf(err, "ioreg unmarshal for %q failed", ioDeviceTreePath)
	}
	if len(data) != 1 {
		err := errors.Errorf("ioreg unmarshal resulted in %d I/O device tree nodes (expected 1)", len(data))
		return nil, err
	}

	return &data[0], nil
}

func (ctx *context) makePartition(disk, s diskOrPartitionPlistNode, isAPFS bool) (*Partition, error) {
	if s.Size < 0 {
		return nil, errors.Errorf("invalid size %q of partition %q", s.Size, s.DeviceIdentifier)
	}

	var partType string
	if isAPFS {
		partType = "APFS Volume"
	} else {
		partType = s.Content
	}

	info, err := ctx.getDiskUtilInfoPlist(s.DeviceIdentifier)
	if err != nil {
		return nil, err
	}

	return &Partition{
		Disk:       nil, // filled in later
		Name:       s.DeviceIdentifier,
		Label:      s.VolumeName,
		MountPoint: s.MountPoint,
		SizeBytes:  uint64(s.Size),
		Type:       partType,
		IsReadOnly: !info.WritableVolume,
	}, nil
}

func (ctx *context) blockFillInfo(info *BlockInfo) error {
	listPlist, err := ctx.getDiskUtilListPlist()
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		return err
	}

	info.TotalPhysicalBytes = 0
	info.Disks = make([]*Disk, 0, len(listPlist.AllDisksAndPartitions))
	info.Partitions = []*Partition{}

	for _, disk := range listPlist.AllDisksAndPartitions {
		if disk.Size < 0 {
			return errors.Errorf("invalid size %q of disk %q", disk.Size, disk.DeviceIdentifier)
		}

		infoPlist, err := ctx.getDiskUtilInfoPlist(disk.DeviceIdentifier)
		if err != nil {
			return err
		}
		if infoPlist.DeviceBlockSize < 0 {
			return errors.Errorf("invalid block size %q of disk %q", infoPlist.DeviceBlockSize, disk.DeviceIdentifier)
		}

		busPath := strings.TrimPrefix(infoPlist.DeviceTreePath, "IODeviceTree:")

		ioregPlist, err := ctx.getIoregPlist(infoPlist.DeviceTreePath)
		if err != nil {
			return err
		}

		// The NUMA node & WWN don't seem to be reported by any tools available by default in macOS.
		diskReport := &Disk{
			Name:                   disk.DeviceIdentifier,
			SizeBytes:              uint64(disk.Size),
			PhysicalBlockSizeBytes: uint64(infoPlist.DeviceBlockSize),
			BusType:                infoPlist.BusProtocol,
			BusPath:                busPath,
			NUMANodeID:             -1,
			Vendor:                 ioregPlist.VendorName,
			Model:                  ioregPlist.ModelNumber,
			SerialNumber:           ioregPlist.SerialNumber,
			WWN:                    "",
			Partitions:             make([]*Partition, 0, len(disk.Partitions)+len(disk.APFSVolumes)),
		}

		for _, partition := range disk.Partitions {
			part, err := ctx.makePartition(disk, partition, false)
			if err != nil {
				return err
			}
			part.Disk = diskReport
			diskReport.Partitions = append(diskReport.Partitions, part)
		}
		for _, volume := range disk.APFSVolumes {
			part, err := ctx.makePartition(disk, volume, true)
			if err != nil {
				return err
			}
			part.Disk = diskReport
			diskReport.Partitions = append(diskReport.Partitions, part)
		}

		info.TotalPhysicalBytes += uint64(disk.Size)
		info.Disks = append(info.Disks, diskReport)
		info.Partitions = append(info.Partitions, diskReport.Partitions...)
	}

	return nil
}
<|endoftext|>