<commit_before>package main

import (
	"fmt"
	"io"
)

// Parser represents a parser.
type Parser struct {
	s   *Scanner
	buf struct {
		t   Token  // last read token
		lit string // last read literal
		n   int    // buffer size (max=1)
	}
}

// NewParser returns a new instance of Parser.
func NewParser(r io.Reader) *Parser {
	return &Parser{s: NewScanner(r)}
}

// scan returns the next token from the underlying scanner.
// if a token has been unscanned then read that instead.
func (p *Parser) scan() (t Token, lit string) {

	if p.buf.n != 0 {
		p.buf.n = 0
		return p.buf.t, p.buf.lit
	}

	t, lit = p.s.Scan()

	p.buf.t, p.buf.lit = t, lit
	return
}

// unscan pushes the previously read token back onto the buffer.
func (p *Parser) unscan() { p.buf.n = 1 }

// scanIgnoreWhitespace scans the next non-whitespace token.
func (p *Parser) scanIgnoreWhitespace() (t Token, lit string) {
	t, lit = p.scan()
	if t == Space {
		t, lit = p.scan()
	}
	return
}

// Expression is the interface for an expression.
type Expression interface {
	String() string
	Evaluate() string
}

// Statement represents a code statement a = 2.
type Statement struct {
	Left  string
	Right string
}

// String retuns the statement as a string.
func (s Statement) String() string {
	return fmt.Sprintf("%v%v", s.Left, s.Right)
}

// Evaluate evaluates the given statement.
func (s Statement) Evaluate() string {
	return fmt.Sprintf("%v%v", s.Left, s.Right)
}

// Num represents a number statement.
type Num struct {
	n string
}

// String returns the string of the number
func (n Num) String() string {
	return fmt.Sprintf("%v", n.n)
}

// Evaluate returns the number value
func (n Num) Evaluate() string {
	return n.String()
}

// Parse parse a assign statement a = b
func (p *Parser) Parse() (*Expression, error) {
	stmt := Statement{}
	tok, lit := p.scanIgnoreWhitespace()
	var lastTok Token
	if tok == Identifier {
		stmt.Left = lit
	} else if tok == Number {
		stmt.Left = lit
		lastTok = Number
	} else {
		return nil, fmt.Errorf("found %q, expected left", lit)
	}

	tok, lit = p.scanIgnoreWhitespace()
	if tok == EOF && lastTok == Number {
		e := Expression(Num{stmt.Left})
		return &e, nil
	}

	if tok != Assign {
		return nil, fmt.Errorf("found %q, expected '=' with tok: %v, expected %v", lit, tok, Assign)
	}

	tok, lit = p.scanIgnoreWhitespace()
	if tok != Identifier {
		return nil, fmt.Errorf("found %q, expected identifier name", lit)
	}
	stmt.Right = lit
	e := Expression(stmt)
	return &e, nil
}
<commit_msg>Add binary type<commit_after>package main

import (
	"fmt"
	"io"
)

// Parser represents a parser.
type Parser struct {
	s   *Scanner
	buf struct {
		t   Token  // last read token
		lit string // last read literal
		n   int    // buffer size (max=1)
	}
}

// NewParser returns a new instance of Parser.
func NewParser(r io.Reader) *Parser {
	return &Parser{s: NewScanner(r)}
}

// scan returns the next token from the underlying scanner.
// if a token has been unscanned then read that instead.
func (p *Parser) scan() (t Token, lit string) {

	if p.buf.n != 0 {
		p.buf.n = 0
		return p.buf.t, p.buf.lit
	}

	t, lit = p.s.Scan()

	p.buf.t, p.buf.lit = t, lit
	return
}

// unscan pushes the previously read token back onto the buffer.
func (p *Parser) unscan() { p.buf.n = 1 }

// scanIgnoreWhitespace scans the next non-whitespace token.
func (p *Parser) scanIgnoreWhitespace() (t Token, lit string) {
	t, lit = p.scan()
	if t == Space {
		t, lit = p.scan()
	}
	return
}

// Expression is the interface for an expression.
type Expression interface {
	String() string
	Evaluate() string
}

// Statement represents a code statement a = 2.
type Statement struct {
	Left  string
	Right string
}

// String retuns the statement as a string.
func (s Statement) String() string {
	return fmt.Sprintf("%v%v", s.Left, s.Right)
}

// Evaluate evaluates the given statement.
func (s Statement) Evaluate() string {
	return fmt.Sprintf("%v%v", s.Left, s.Right)
}

// Num represents a number statement.
type Num struct {
	n string
}

// String returns the string of the number
func (n Num) String() string {
	return fmt.Sprintf("%v", n.n)
}

// Evaluate returns the number value
func (n Num) Evaluate() string {
	return n.String()
}

// Binary represents a binary statement
// example: 12 + 3
type Binary struct {
	Left     string
	Right    string
	Operator string
}

// String returns the string of the number
func (b Binary) String() string {
	return fmt.Sprintf("%v %v %v", b.Left, b.Operator, b.Right)
}

// Evaluate returns the number value
func (b Binary) Evaluate() string {
	return b.String()
}

// Parse parse a assign statement a = b
func (p *Parser) Parse() (*Expression, error) {
	stmt := Statement{}
	tok, lit := p.scanIgnoreWhitespace()
	var lastTok Token
	if tok == Identifier {
		stmt.Left = lit
	} else if tok == Number {
		stmt.Left = lit
		lastTok = Number
	} else {
		return nil, fmt.Errorf("found %q, expected left", lit)
	}

	tok, lit = p.scanIgnoreWhitespace()
	if tok == EOF && lastTok == Number {
		e := Expression(Num{stmt.Left})
		return &e, nil
	}

	if tok != Assign {
		return nil, fmt.Errorf("found %q, expected '=' with tok: %v, expected %v", lit, tok, Assign)
	}

	tok, lit = p.scanIgnoreWhitespace()
	if tok != Identifier {
		return nil, fmt.Errorf("found %q, expected identifier name", lit)
	}
	stmt.Right = lit
	e := Expression(stmt)
	return &e, nil
}
<|endoftext|>


<commit_before>package patree

import (
	"bufio"
	"bytes"
	"errors"
	"io"
	"strings"
	"unicode/utf8"
)

var NoClosingBracket = errors.New("Invalid syntax: No closing bracket found")

func isMatchPattern(s string) bool {
	return len(s) > 2 && s[0] == '<' && s[len(s)-1] == '>'
}

func routeSplitFunc(data []byte, atEOF bool) (int, []byte, error) {
	if atEOF || data == nil {
		return 0, nil, io.EOF
	}

	r, _ := utf8.DecodeRune(data)

	// matcher
	if r == '<' {
		i := bytes.IndexRune(data, '>')
		if i == -1 {
			return 0, nil, NoClosingBracket
		}
		return (i + 1), data[:(i + 1)], nil
	}

	// one char
	if len(data) == 1 {
		return 1, data, nil
	}

	// should ignore first '/'
	slashIndex := bytes.IndexRune(data[1:], '/')
	if slashIndex != -1 {
		slashIndex++
	}

	matchIndex := bytes.IndexRune(data, '<')

	// remaining string would be a static entry
	if slashIndex == -1 && matchIndex == -1 {
		return len(data), data, nil
	}

	// split by '<'
	// return data before '<'
	if matchIndex != -1 && (slashIndex == -1 || slashIndex > matchIndex) {
		return matchIndex, data[:matchIndex], nil
	}

	// split by '/'
	// return data before '/' including '/'
	return slashIndex + 1, data[:(slashIndex + 1)], nil
}

// SplitPath splits the url pattern to entries.
func SplitPath(pat string) (routes []string, err error) {
	scanner := bufio.NewScanner(strings.NewReader(pat))
	scanner.Split(routeSplitFunc)
	for scanner.Scan() {
		routes = append(routes, scanner.Text())
	}
	err = scanner.Err()
	return
}

func isNextSuffixPattern(p []string) bool {
	if len(p) >= 2 && isMatchPattern(p[0]) && !isMatchPattern(p[1]) {
		matcher, _ := parseMatcher(p[0])
		r, _ := utf8.DecodeRuneInString(p[1])
		return matcher.MatchRune(r)
	}
	return false
}

// PeekNextPattern returns next entry pattern with offset size
func PeekNextPattern(p []string) (pat string, size int) {
	if isNextSuffixPattern(p) {
		pat, size = (p[0] + p[1]), 2
	} else {
		pat, size = p[0], 1
	}
	return
}
<commit_msg>add comments<commit_after>package patree

import (
	"bufio"
	"bytes"
	"errors"
	"io"
	"strings"
	"unicode/utf8"
)

// NoClosingBracket is the error returned by SplitPath when there is no closing
// bracket '>' after opening bracket '<'.
var NoClosingBracket = errors.New("Invalid syntax: No closing bracket found")

func isMatchPattern(s string) bool {
	return len(s) > 2 && s[0] == '<' && s[len(s)-1] == '>'
}

// routeSplitFunc is the SplitFunc to scan url pattern.
func routeSplitFunc(data []byte, atEOF bool) (int, []byte, error) {
	if atEOF || data == nil {
		return 0, nil, io.EOF
	}

	r, _ := utf8.DecodeRune(data)

	// matcher
	if r == '<' {
		i := bytes.IndexRune(data, '>')
		if i == -1 {
			return 0, nil, NoClosingBracket
		}
		return (i + 1), data[:(i + 1)], nil
	}

	// one char
	if len(data) == 1 {
		return 1, data, nil
	}

	// should ignore first '/'
	slashIndex := bytes.IndexRune(data[1:], '/')
	if slashIndex != -1 {
		slashIndex++
	}

	matchIndex := bytes.IndexRune(data, '<')

	// remaining string would be a static entry
	if slashIndex == -1 && matchIndex == -1 {
		return len(data), data, nil
	}

	// split by '<'
	// return data before '<'
	if matchIndex != -1 && (slashIndex == -1 || slashIndex > matchIndex) {
		return matchIndex, data[:matchIndex], nil
	}

	// split by '/'
	// return data before '/' including '/'
	return slashIndex + 1, data[:(slashIndex + 1)], nil
}

// SplitPath splits the url pattern.
func SplitPath(pat string) (routes []string, err error) {
	scanner := bufio.NewScanner(strings.NewReader(pat))
	scanner.Split(routeSplitFunc)
	for scanner.Scan() {
		routes = append(routes, scanner.Text())
	}
	err = scanner.Err()
	return
}

func isNextSuffixPattern(p []string) bool {
	if len(p) >= 2 && isMatchPattern(p[0]) && !isMatchPattern(p[1]) {
		matcher, _ := parseMatcher(p[0])
		r, _ := utf8.DecodeRuneInString(p[1])
		return matcher.MatchRune(r)
	}
	return false
}

// PeekNextPattern returns next entry pattern with offset size
func PeekNextPattern(p []string) (pat string, size int) {
	if isNextSuffixPattern(p) {
		pat, size = (p[0] + p[1]), 2
	} else {
		pat, size = p[0], 1
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"github.com/luisvinicius167/godux"
)

func main() {
	// Creating new Store
	store := godux.NewStore()
	store.SetState("count", 1)

	// Creating new Action

	increment := func(number int) godux.Action {
		return godux.Action{
			Type:  "INCREMENT",
			Value: number,
		}
	}

	decrement := func(number int) godux.Action {
		return godux.Action{
			Type:  "DECREMENT",
			Value: number,
		}
	}
	// reducer function
	reducer := func(action godux.Action) interface{} {
		switch action.Type {
		case "INCREMENT":
			return store.GetState("count").(int) + action.Value.(int)
		case "DECREMENT":
			return action.Value.(int) - store.GetState("count").(int)
		default:
			return store.GetAllState()
		}
	}
	// Add your reducer function to return new values basend on your state
	store.Reducer(reducer)
	// Receive new value
	newCount := store.Dispatch(increment(10))
	subCount := store.Dispatch(decrement(10))
	fmt.Printf("Your Store state is: %s. Your newCount is: %s. Your subCount is: %s", store.GetState("count"), newCount, subCount)
}
<commit_msg>change Setstate -> SetState method<commit_after>package main

import (
	"fmt"

	"github.com/luisvinicius167/godux"
)

func main() {
	// Creating new Store
	store := godux.NewStore()
	// Set state
	store.SetState("count", 1)

	// Creating new Action

	increment := func(number int) godux.Action {
		return godux.Action{
			Type:  "INCREMENT",
			Value: number,
		}
	}

	decrement := func(number int) godux.Action {
		return godux.Action{
			Type:  "DECREMENT",
			Value: number,
		}
	}
	// reducer function
	reducer := func(action godux.Action) interface{} {
		switch action.Type {
		case "INCREMENT":
			return store.GetState("count").(int) + action.Value.(int)
		case "DECREMENT":
			return action.Value.(int) - store.GetState("count").(int)
		default:
			return store.GetAllState()
		}
	}
	// Add your reducer function to return new values basend on your state
	store.Reducer(reducer)
	// Receive new value
	newCount := store.Dispatch(increment(10))
	subCount := store.Dispatch(decrement(10))
	fmt.Printf("Your Store state is: %s. Your newCount is: %s. Your subCount is: %s", store.GetState("count"), newCount, subCount)
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"
	"os/signal"
	"syscall"

	discovery "github.com/andy-trimble/go.discovery"
)

func main() {
	d := discovery.Discovery{}
	err := d.Start("server")
	if err != nil {
		log.Fatal(err)
	}
	defer d.Shutdown()

	go func() {
		for {
			actor := <-d.Discovered
			log.Printf("Discovered %+v", actor)
		}
	}()

	c := make(chan os.Signal, 2)
	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
	go func() {
		<-c
		d.Shutdown()
		os.Exit(0)
	}()

	for {
		err := <-d.Err
		log.Printf("%+v", err)
	}
}
<commit_msg>Fixed example exit criteria<commit_after>package main

import (
	"bufio"
	"log"
	"os"

	discovery "github.com/andy-trimble/go.discovery"
)

func main() {
	d := discovery.Discovery{}
	err := d.Start("server")
	if err != nil {
		log.Fatal(err)
	}
	defer d.Shutdown()

	go func() {
		for {
			actor := <-d.Discovered
			log.Printf("Discovered %+v", actor)
		}
	}()

	go func() {
		for {
			err := <-d.Err
			log.Printf("%+v", err)
		}
	}()

	log.Println("Press return to exit...")
	reader := bufio.NewReader(os.Stdin)
	reader.ReadString('\n')
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"regexp"

	"github.com/wcl48/valval"
)

func main() {
	personValidator := valval.Object(valval.M{
		"Name": valval.String(
			valval.MaxLength(20),
			valval.Regexp(regexp.MustCompile(`^[a-z ]+$`)),
		),
		"Attr": valval.Object(valval.M{
			"Age": valval.Number(
				valval.Min(0),
				valval.Max(120),
			),
			"Gender": valval.String(valval.In("male", "female")),
			"Tags": valval.Slice(
				valval.String(
					valval.MinLength(1),
					valval.MaxLength(10),
				),
			).Self(valval.MaxSliceLength(10)),
		}),
	})

	// Validate struct
	type PersonAttr struct {
		Age    int
		Gender string
		Tags   []string
	}
	type Person struct {
		Name string
		Attr PersonAttr
	}
	person := &Person{
		Name: "John!",
		Attr: PersonAttr{
			Age:    200,
			Gender: "otoko",
			Tags:   []string{"", "abcdefghijklmn"},
		},
	}
	if err := personValidator.Validate(person); err != nil {
		errs := valval.Errors(err)
		for _, errInfo := range errs {
			fmt.Printf("%s : %v\n", errInfo.Path, errInfo.Error)
		}
	}

	// validate map
	personMap := map[string]interface{}{
		"Name": "John!",
		"Attr": map[string]interface{}{
			"Age":    200,
			"Gender": "otoko",
			"Tags":   []string{"", "abcdefghijklmn"},
		},
	}
	if err := personValidator.Validate(personMap); err != nil {
		errs := valval.Errors(err)
		for _, errInfo := range errs {
			fmt.Printf("%s : %v\n", errInfo.Path, errInfo.Error)
		}
	}
}
<commit_msg>transferred wcl48 to wacul<commit_after>package main

import (
	"fmt"
	"regexp"

	"github.com/wacul/valval"
)

func main() {
	personValidator := valval.Object(valval.M{
		"Name": valval.String(
			valval.MaxLength(20),
			valval.Regexp(regexp.MustCompile(`^[a-z ]+$`)),
		),
		"Attr": valval.Object(valval.M{
			"Age": valval.Number(
				valval.Min(0),
				valval.Max(120),
			),
			"Gender": valval.String(valval.In("male", "female")),
			"Tags": valval.Slice(
				valval.String(
					valval.MinLength(1),
					valval.MaxLength(10),
				),
			).Self(valval.MaxSliceLength(10)),
		}),
	})

	// Validate struct
	type PersonAttr struct {
		Age    int
		Gender string
		Tags   []string
	}
	type Person struct {
		Name string
		Attr PersonAttr
	}
	person := &Person{
		Name: "John!",
		Attr: PersonAttr{
			Age:    200,
			Gender: "otoko",
			Tags:   []string{"", "abcdefghijklmn"},
		},
	}
	if err := personValidator.Validate(person); err != nil {
		errs := valval.Errors(err)
		for _, errInfo := range errs {
			fmt.Printf("%s : %v\n", errInfo.Path, errInfo.Error)
		}
	}

	// validate map
	personMap := map[string]interface{}{
		"Name": "John!",
		"Attr": map[string]interface{}{
			"Age":    200,
			"Gender": "otoko",
			"Tags":   []string{"", "abcdefghijklmn"},
		},
	}
	if err := personValidator.Validate(personMap); err != nil {
		errs := valval.Errors(err)
		for _, errInfo := range errs {
			fmt.Printf("%s : %v\n", errInfo.Path, errInfo.Error)
		}
	}
}
<|endoftext|>


<commit_before>// Example usage of papaBot.
package main

import (
	"flag"
	"fmt"
	"github.com/pawelszydlo/papa-bot"
	"github.com/pawelszydlo/papa-bot/events"
	"github.com/pawelszydlo/papa-bot/extensions"
	"time"
)

var (
	configFile = flag.String("c", "config.ini", "Path to TOML configuration file for the bot.")
	textsFile  = flag.String("t", "texts.ini", "Path to TOML configuration file with the bot texts.")
)

func init() {
	flag.Parse()
}

// Create your own extension.
type MyExtension struct {
	startTime time.Time
	bot       *papaBot.Bot
}

// Will be run on bot's init or when extension is registered after bot's init. This is the only require function
// that your extension must have.
func (ext *MyExtension) Init(bot *papaBot.Bot) error {
	ext.bot = bot
	ext.startTime = time.Now()
	bot.EventDispatcher.RegisterListener(events.EventTick, ext.TickListener)
	return nil
}

// Will be attached to a EventTick event that is happening every 5 minutes.
func (ext *MyExtension) TickListener(message events.EventMessage) {
	ext.bot.SendMassNotice(
		fmt.Sprintf("I have been running for %.0f minutes now.", time.Since(ext.startTime).Minutes()))
}

// Entry point
func main() {
	// This will create bot's structures. Feel free to modify what you need afterwards.
	bot := papaBot.New(*configFile, *textsFile)

	// As an example, change the name.
	bot.Config.Name = "David"

	// Add all built-in extensions.
	extensions.RegisterBuiltinExtensions(bot)

	// Add your own custom extension.
	bot.RegisterExtension(new(MyExtension))

	// This will init the bot's mechanisms and run the bot's main loop.
	bot.Run()
}
<commit_msg>Extend the example extension.<commit_after>// Example usage of papaBot.
package main

import (
	"flag"
	"fmt"
	"github.com/pawelszydlo/papa-bot"
	"github.com/pawelszydlo/papa-bot/events"
	"github.com/pawelszydlo/papa-bot/extensions"
	"time"
)

var (
	configFile = flag.String("c", "config.ini", "Path to TOML configuration file for the bot.")
	textsFile  = flag.String("t", "texts.ini", "Path to TOML configuration file with the bot texts.")
)

func init() {
	flag.Parse()
}

// Create your own extension.
type MyExtension struct {
	startTime time.Time
	bot       *papaBot.Bot
}

// Will be run on bot's init or when extension is registered after bot's init. This is the only required function
// that your extension must have. For a list of actions your extension can perform on the bot see api.go.
func (ext *MyExtension) Init(bot *papaBot.Bot) error {
	ext.bot = bot
	ext.startTime = time.Now()
	// This is an example event listener registration. You can find a list of events in the "events" package.
	bot.EventDispatcher.RegisterListener(events.EventTick, ext.TickListener)
	// Register new command. See the struct for field descriptions.
	bot.RegisterCommand(&papaBot.BotCommand{
		[]string{"hello"},
		false, false, false,
		"", "Say hello!",
		ext.commandHello})
	return nil
}

// TickListener will be attached to a EventTick event that is happening every 5 minutes.
func (ext *MyExtension) TickListener(message events.EventMessage) {
	ext.bot.SendMassNotice(
		fmt.Sprintf("I have been running for %.0f minutes now.", time.Since(ext.startTime).Minutes()))
}

// commandHello is a command for saying hello.
func (ext *MyExtension) commandHello(bot *papaBot.Bot, sourceEvent *events.EventMessage, params []string) {
	bot.SendMessage(sourceEvent, "Hello!")
}

// Entry point
func main() {
	// This will create bot's structures. Feel free to modify what you need afterwards.
	bot := papaBot.New(*configFile, *textsFile)

	// As an example, change the name.
	bot.Config.Name = "David"

	// Add all built-in extensions.
	extensions.RegisterBuiltinExtensions(bot)

	// Add your own custom extension.
	bot.RegisterExtension(new(MyExtension))

	// This will init the bot's mechanisms and run the bot's main loop.
	bot.Run()
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package shards

import (
	"context"
	"fmt"
	"log"
	"os"
	"runtime"
	"runtime/debug"
	"sort"
	"time"

	"golang.org/x/net/trace"
	"golang.org/x/sync/semaphore"

	"github.com/google/zoekt"
	"github.com/google/zoekt/query"
)

type rankedShard struct {
	zoekt.Searcher
	rank uint16
}

type shardedSearcher struct {
	// Limit the number of parallel queries. Since searching is
	// CPU bound, we can't do better than #CPU queries in
	// parallel.  If we do so, we just create more memory
	// pressure.
	throttle *semaphore.Weighted
	capacity int64

	shards map[string]rankedShard
}

func newShardedSearcher(n int64) *shardedSearcher {
	ss := &shardedSearcher{
		shards:   make(map[string]rankedShard),
		throttle: semaphore.NewWeighted(n),
		capacity: n,
	}
	return ss
}

// NewDirectorySearcher returns a searcher instance that loads all
// shards corresponding to a glob into memory.
func NewDirectorySearcher(dir string) (zoekt.Searcher, error) {
	ss := newShardedSearcher(int64(runtime.GOMAXPROCS(0)))
	tl := &throttledLoader{
		ss:       ss,
		throttle: make(chan struct{}, runtime.GOMAXPROCS(0)),
	}
	_, err := NewDirectoryWatcher(dir, tl)
	if err != nil {
		return nil, err
	}

	return ss, nil
}

// throttledLoader tries to load up to throttle shards in parallel.
type throttledLoader struct {
	ss       *shardedSearcher
	throttle chan struct{}
}

func (tl *throttledLoader) load(key string) {
	tl.throttle <- struct{}{}
	shard, err := loadShard(key)
	<-tl.throttle
	if err != nil {
		log.Printf("reloading: %s, err %v ", key, err)
		return
	}

	tl.ss.replace(key, shard)
}

func (tl *throttledLoader) drop(key string) {
	tl.ss.replace(key, nil)
}

func (ss *shardedSearcher) String() string {
	return "shardedSearcher"
}

// Close closes references to open files. It may be called only once.
func (ss *shardedSearcher) Close() {
	ss.lock(context.Background())
	defer ss.unlock()
	for _, s := range ss.shards {
		s.Close()
	}
}

func (ss *shardedSearcher) Search(ctx context.Context, q query.Q, opts *zoekt.SearchOptions) (sr *zoekt.SearchResult, err error) {
	tr := trace.New("shardedSearcher.Search", "")
	tr.LazyLog(q, true)
	tr.LazyPrintf("opts: %+v", opts)
	defer func() {
		if sr != nil {
			tr.LazyPrintf("num files: %d", len(sr.Files))
			tr.LazyPrintf("stats: %+v", sr.Stats)
		}
		if err != nil {
			tr.LazyPrintf("error: %v", err)
			tr.SetError()
		}
		tr.Finish()
	}()

	start := time.Now()

	aggregate := &zoekt.SearchResult{
		RepoURLs:      map[string]string{},
		LineFragments: map[string]string{},
	}

	// This critical section is large, but we don't want to deal with
	// searches on shards that have just been closed.
	if err := ss.rlock(ctx); err != nil {
		return aggregate, err
	}
	defer ss.runlock()
	tr.LazyPrintf("acquired lock")
	aggregate.Wait = time.Since(start)
	start = time.Now()

	shards := ss.getShards()
	all := make(chan shardResult, len(shards))

	var childCtx context.Context
	var cancel context.CancelFunc
	if opts.MaxWallTime == 0 {
		childCtx, cancel = context.WithCancel(ctx)
	} else {
		childCtx, cancel = context.WithTimeout(ctx, opts.MaxWallTime)
	}

	defer cancel()

	// For each query, throttle the number of parallel
	// actions. Since searching is mostly CPU bound, we limit the
	// number of parallel searches. This reduces the peak working
	// set, which hopefully stops https://cs.bazel.build from crashing
	// when looking for the string "com".
	feeder := make(chan zoekt.Searcher, len(shards))
	for _, s := range shards {
		feeder <- s
	}
	close(feeder)
	for i := 0; i < runtime.GOMAXPROCS(0); i++ {
		go func() {
			for s := range feeder {
				searchOneShard(childCtx, s, q, opts, all)
			}
		}()
	}

	for range shards {
		r := <-all
		if r.err != nil {
			return nil, r.err
		}
		aggregate.Files = append(aggregate.Files, r.sr.Files...)
		aggregate.Stats.Add(r.sr.Stats)

		if len(r.sr.Files) > 0 {
			for k, v := range r.sr.RepoURLs {
				aggregate.RepoURLs[k] = v
			}
			for k, v := range r.sr.LineFragments {
				aggregate.LineFragments[k] = v
			}
		}

		if cancel != nil && opts.TotalMaxMatchCount > 0 && aggregate.Stats.MatchCount > opts.TotalMaxMatchCount {
			cancel()
			cancel = nil
		}
	}

	zoekt.SortFilesByScore(aggregate.Files)
	if max := opts.MaxDocDisplayCount; max > 0 && len(aggregate.Files) > max {
		aggregate.Files = aggregate.Files[:max]
	}
	for i := range aggregate.Files {
		copySlice(&aggregate.Files[i].Content)
		copySlice(&aggregate.Files[i].Checksum)
		for l := range aggregate.Files[i].LineMatches {
			copySlice(&aggregate.Files[i].LineMatches[l].Line)
		}
	}

	aggregate.Duration = time.Since(start)
	return aggregate, nil
}

func copySlice(src *[]byte) {
	dst := make([]byte, len(*src))
	copy(dst, *src)
	*src = dst
}

type shardResult struct {
	sr  *zoekt.SearchResult
	err error
}

func searchOneShard(ctx context.Context, s zoekt.Searcher, q query.Q, opts *zoekt.SearchOptions, sink chan shardResult) {
	defer func() {
		if r := recover(); r != nil {
			log.Printf("crashed shard: %s: %s, %s", s.String(), r, debug.Stack())

			var r zoekt.SearchResult
			r.Stats.Crashes = 1
			sink <- shardResult{&r, nil}
		}
	}()

	ms, err := s.Search(ctx, q, opts)
	sink <- shardResult{ms, err}
}

func (ss *shardedSearcher) List(ctx context.Context, r query.Q) (rl *zoekt.RepoList, err error) {
	tr := trace.New("shardedSearcher.List", "")
	tr.LazyLog(r, true)
	defer func() {
		if rl != nil {
			tr.LazyPrintf("repos size: %d", len(rl.Repos))
			tr.LazyPrintf("crashes: %d", rl.Crashes)
		}
		if err != nil {
			tr.LazyPrintf("error: %v", err)
			tr.SetError()
		}
		tr.Finish()
	}()

	type res struct {
		rl  *zoekt.RepoList
		err error
	}

	if err := ss.rlock(ctx); err != nil {
		return nil, err
	}
	defer ss.runlock()
	tr.LazyPrintf("acquired lock")

	shards := ss.getShards()
	shardCount := len(shards)
	all := make(chan res, shardCount)
	tr.LazyPrintf("shardCount: %d", len(shards))

	for _, s := range shards {
		go func(s zoekt.Searcher) {
			defer func() {
				if r := recover(); r != nil {
					all <- res{
						&zoekt.RepoList{Crashes: 1}, nil,
					}
				}
			}()
			ms, err := s.List(ctx, r)
			all <- res{ms, err}
		}(s.Searcher)
	}

	crashes := 0
	uniq := map[string]*zoekt.RepoListEntry{}

	for i := 0; i < shardCount; i++ {
		r := <-all
		if r.err != nil {
			return nil, r.err
		}
		crashes += r.rl.Crashes
		for _, r := range r.rl.Repos {
			prev, ok := uniq[r.Repository.Name]
			if !ok {
				cp := *r
				uniq[r.Repository.Name] = &cp
			} else {
				prev.Stats.Add(&r.Stats)
			}
		}
	}

	aggregate := make([]*zoekt.RepoListEntry, 0, len(uniq))
	for _, v := range uniq {
		aggregate = append(aggregate, v)
	}
	return &zoekt.RepoList{
		Repos:   aggregate,
		Crashes: crashes,
	}, nil
}

func (s *shardedSearcher) rlock(ctx context.Context) error {
	return s.throttle.Acquire(ctx, 1)
}

// getShards returns the currently loaded shards. The shards must be
// accessed under a rlock call. The shards are sorted by decreasing
// rank.
func (s *shardedSearcher) getShards() []rankedShard {
	var res []rankedShard
	for _, sh := range s.shards {
		res = append(res, sh)
	}
	// TODO: precompute this.
	sort.Slice(res, func(i, j int) bool {
		return res[i].rank > res[j].rank
	})
	return res
}

func (s *shardedSearcher) runlock() {
	s.throttle.Release(1)
}

func (s *shardedSearcher) lock(ctx context.Context) error {
	return s.throttle.Acquire(ctx, s.capacity)
}

func (s *shardedSearcher) unlock() {
	s.throttle.Release(s.capacity)
}

func shardRank(s zoekt.Searcher) uint16 {
	q := query.Repo{}
	result, err := s.List(context.Background(), &q)
	if err != nil {
		return 0
	}
	if len(result.Repos) == 0 {
		return 0
	}
	return result.Repos[0].Repository.Rank
}

func (s *shardedSearcher) replace(key string, shard zoekt.Searcher) {
	var rank uint16
	if shard != nil {
		rank = shardRank(shard)
	}

	s.lock(context.Background())
	defer s.unlock()
	old := s.shards[key]
	if old.Searcher != nil {
		old.Close()
	}

	if shard == nil {
		delete(s.shards, key)
	} else {
		s.shards[key] = rankedShard{
			rank:     rank,
			Searcher: shard,
		}
	}
}

func loadShard(fn string) (zoekt.Searcher, error) {
	f, err := os.Open(fn)
	if err != nil {
		return nil, err
	}

	iFile, err := zoekt.NewIndexFile(f)
	if err != nil {
		return nil, err
	}
	s, err := zoekt.NewSearcher(iFile)
	if err != nil {
		iFile.Close()
		return nil, fmt.Errorf("NewSearcher(%s): %v", fn, err)
	}

	return s, nil
}
<commit_msg>shards: remove context from R/W lock helper<commit_after>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package shards

import (
	"context"
	"fmt"
	"log"
	"os"
	"runtime"
	"runtime/debug"
	"sort"
	"time"

	"golang.org/x/net/trace"
	"golang.org/x/sync/semaphore"

	"github.com/google/zoekt"
	"github.com/google/zoekt/query"
)

type rankedShard struct {
	zoekt.Searcher
	rank uint16
}

type shardedSearcher struct {
	// Limit the number of parallel queries. Since searching is
	// CPU bound, we can't do better than #CPU queries in
	// parallel.  If we do so, we just create more memory
	// pressure.
	throttle *semaphore.Weighted
	capacity int64

	shards map[string]rankedShard
}

func newShardedSearcher(n int64) *shardedSearcher {
	ss := &shardedSearcher{
		shards:   make(map[string]rankedShard),
		throttle: semaphore.NewWeighted(n),
		capacity: n,
	}
	return ss
}

// NewDirectorySearcher returns a searcher instance that loads all
// shards corresponding to a glob into memory.
func NewDirectorySearcher(dir string) (zoekt.Searcher, error) {
	ss := newShardedSearcher(int64(runtime.GOMAXPROCS(0)))
	tl := &throttledLoader{
		ss:       ss,
		throttle: make(chan struct{}, runtime.GOMAXPROCS(0)),
	}
	_, err := NewDirectoryWatcher(dir, tl)
	if err != nil {
		return nil, err
	}

	return ss, nil
}

// throttledLoader tries to load up to throttle shards in parallel.
type throttledLoader struct {
	ss       *shardedSearcher
	throttle chan struct{}
}

func (tl *throttledLoader) load(key string) {
	tl.throttle <- struct{}{}
	shard, err := loadShard(key)
	<-tl.throttle
	if err != nil {
		log.Printf("reloading: %s, err %v ", key, err)
		return
	}

	tl.ss.replace(key, shard)
}

func (tl *throttledLoader) drop(key string) {
	tl.ss.replace(key, nil)
}

func (ss *shardedSearcher) String() string {
	return "shardedSearcher"
}

// Close closes references to open files. It may be called only once.
func (ss *shardedSearcher) Close() {
	ss.lock()
	defer ss.unlock()
	for _, s := range ss.shards {
		s.Close()
	}
}

func (ss *shardedSearcher) Search(ctx context.Context, q query.Q, opts *zoekt.SearchOptions) (sr *zoekt.SearchResult, err error) {
	tr := trace.New("shardedSearcher.Search", "")
	tr.LazyLog(q, true)
	tr.LazyPrintf("opts: %+v", opts)
	defer func() {
		if sr != nil {
			tr.LazyPrintf("num files: %d", len(sr.Files))
			tr.LazyPrintf("stats: %+v", sr.Stats)
		}
		if err != nil {
			tr.LazyPrintf("error: %v", err)
			tr.SetError()
		}
		tr.Finish()
	}()

	start := time.Now()

	aggregate := &zoekt.SearchResult{
		RepoURLs:      map[string]string{},
		LineFragments: map[string]string{},
	}

	// This critical section is large, but we don't want to deal with
	// searches on shards that have just been closed.
	if err := ss.rlock(ctx); err != nil {
		return aggregate, err
	}
	defer ss.runlock()
	tr.LazyPrintf("acquired lock")
	aggregate.Wait = time.Since(start)
	start = time.Now()

	shards := ss.getShards()
	all := make(chan shardResult, len(shards))

	var childCtx context.Context
	var cancel context.CancelFunc
	if opts.MaxWallTime == 0 {
		childCtx, cancel = context.WithCancel(ctx)
	} else {
		childCtx, cancel = context.WithTimeout(ctx, opts.MaxWallTime)
	}

	defer cancel()

	// For each query, throttle the number of parallel
	// actions. Since searching is mostly CPU bound, we limit the
	// number of parallel searches. This reduces the peak working
	// set, which hopefully stops https://cs.bazel.build from crashing
	// when looking for the string "com".
	feeder := make(chan zoekt.Searcher, len(shards))
	for _, s := range shards {
		feeder <- s
	}
	close(feeder)
	for i := 0; i < runtime.GOMAXPROCS(0); i++ {
		go func() {
			for s := range feeder {
				searchOneShard(childCtx, s, q, opts, all)
			}
		}()
	}

	for range shards {
		r := <-all
		if r.err != nil {
			return nil, r.err
		}
		aggregate.Files = append(aggregate.Files, r.sr.Files...)
		aggregate.Stats.Add(r.sr.Stats)

		if len(r.sr.Files) > 0 {
			for k, v := range r.sr.RepoURLs {
				aggregate.RepoURLs[k] = v
			}
			for k, v := range r.sr.LineFragments {
				aggregate.LineFragments[k] = v
			}
		}

		if cancel != nil && opts.TotalMaxMatchCount > 0 && aggregate.Stats.MatchCount > opts.TotalMaxMatchCount {
			cancel()
			cancel = nil
		}
	}

	zoekt.SortFilesByScore(aggregate.Files)
	if max := opts.MaxDocDisplayCount; max > 0 && len(aggregate.Files) > max {
		aggregate.Files = aggregate.Files[:max]
	}
	for i := range aggregate.Files {
		copySlice(&aggregate.Files[i].Content)
		copySlice(&aggregate.Files[i].Checksum)
		for l := range aggregate.Files[i].LineMatches {
			copySlice(&aggregate.Files[i].LineMatches[l].Line)
		}
	}

	aggregate.Duration = time.Since(start)
	return aggregate, nil
}

func copySlice(src *[]byte) {
	dst := make([]byte, len(*src))
	copy(dst, *src)
	*src = dst
}

type shardResult struct {
	sr  *zoekt.SearchResult
	err error
}

func searchOneShard(ctx context.Context, s zoekt.Searcher, q query.Q, opts *zoekt.SearchOptions, sink chan shardResult) {
	defer func() {
		if r := recover(); r != nil {
			log.Printf("crashed shard: %s: %s, %s", s.String(), r, debug.Stack())

			var r zoekt.SearchResult
			r.Stats.Crashes = 1
			sink <- shardResult{&r, nil}
		}
	}()

	ms, err := s.Search(ctx, q, opts)
	sink <- shardResult{ms, err}
}

func (ss *shardedSearcher) List(ctx context.Context, r query.Q) (rl *zoekt.RepoList, err error) {
	tr := trace.New("shardedSearcher.List", "")
	tr.LazyLog(r, true)
	defer func() {
		if rl != nil {
			tr.LazyPrintf("repos size: %d", len(rl.Repos))
			tr.LazyPrintf("crashes: %d", rl.Crashes)
		}
		if err != nil {
			tr.LazyPrintf("error: %v", err)
			tr.SetError()
		}
		tr.Finish()
	}()

	type res struct {
		rl  *zoekt.RepoList
		err error
	}

	if err := ss.rlock(ctx); err != nil {
		return nil, err
	}
	defer ss.runlock()
	tr.LazyPrintf("acquired lock")

	shards := ss.getShards()
	shardCount := len(shards)
	all := make(chan res, shardCount)
	tr.LazyPrintf("shardCount: %d", len(shards))

	for _, s := range shards {
		go func(s zoekt.Searcher) {
			defer func() {
				if r := recover(); r != nil {
					all <- res{
						&zoekt.RepoList{Crashes: 1}, nil,
					}
				}
			}()
			ms, err := s.List(ctx, r)
			all <- res{ms, err}
		}(s.Searcher)
	}

	crashes := 0
	uniq := map[string]*zoekt.RepoListEntry{}

	for i := 0; i < shardCount; i++ {
		r := <-all
		if r.err != nil {
			return nil, r.err
		}
		crashes += r.rl.Crashes
		for _, r := range r.rl.Repos {
			prev, ok := uniq[r.Repository.Name]
			if !ok {
				cp := *r
				uniq[r.Repository.Name] = &cp
			} else {
				prev.Stats.Add(&r.Stats)
			}
		}
	}

	aggregate := make([]*zoekt.RepoListEntry, 0, len(uniq))
	for _, v := range uniq {
		aggregate = append(aggregate, v)
	}
	return &zoekt.RepoList{
		Repos:   aggregate,
		Crashes: crashes,
	}, nil
}

func (s *shardedSearcher) rlock(ctx context.Context) error {
	return s.throttle.Acquire(ctx, 1)
}

// getShards returns the currently loaded shards. The shards must be
// accessed under a rlock call. The shards are sorted by decreasing
// rank.
func (s *shardedSearcher) getShards() []rankedShard {
	var res []rankedShard
	for _, sh := range s.shards {
		res = append(res, sh)
	}
	// TODO: precompute this.
	sort.Slice(res, func(i, j int) bool {
		return res[i].rank > res[j].rank
	})
	return res
}

func (s *shardedSearcher) runlock() {
	s.throttle.Release(1)
}

func (s *shardedSearcher) lock() {
	// won't error since context.Background won't expire
	_ = s.throttle.Acquire(context.Background(), s.capacity)
}

func (s *shardedSearcher) unlock() {
	s.throttle.Release(s.capacity)
}

func shardRank(s zoekt.Searcher) uint16 {
	q := query.Repo{}
	result, err := s.List(context.Background(), &q)
	if err != nil {
		return 0
	}
	if len(result.Repos) == 0 {
		return 0
	}
	return result.Repos[0].Repository.Rank
}

func (s *shardedSearcher) replace(key string, shard zoekt.Searcher) {
	var rank uint16
	if shard != nil {
		rank = shardRank(shard)
	}

	s.lock()
	defer s.unlock()
	old := s.shards[key]
	if old.Searcher != nil {
		old.Close()
	}

	if shard == nil {
		delete(s.shards, key)
	} else {
		s.shards[key] = rankedShard{
			rank:     rank,
			Searcher: shard,
		}
	}
}

func loadShard(fn string) (zoekt.Searcher, error) {
	f, err := os.Open(fn)
	if err != nil {
		return nil, err
	}

	iFile, err := zoekt.NewIndexFile(f)
	if err != nil {
		return nil, err
	}
	s, err := zoekt.NewSearcher(iFile)
	if err != nil {
		iFile.Close()
		return nil, fmt.Errorf("NewSearcher(%s): %v", fn, err)
	}

	return s, nil
}
<|endoftext|>


<commit_before>// Copyright 2022 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bbfake

import (
	"fmt"
	"sync"

	bbpb "go.chromium.org/luci/buildbucket/proto"
	"google.golang.org/protobuf/proto"

	"go.chromium.org/luci/cv/internal/buildbucket"
)

type fakeApp struct {
	buildStoreMu sync.RWMutex
	buildStore   map[int64]*bbpb.Build // build ID -> build
}

type Fake struct {
	hostsMu sync.Mutex
	hosts   map[string]*fakeApp // hostname -> fakeApp
}

func (f *Fake) NewClientFactory() buildbucket.ClientFactory {
	return clientFactory{
		fake: f,
	}
}

// AddBuild adds a build to fake Buildbucket host.
//
// Reads Buildbucket hostname from `infra.buildbucket.hostname`.
// Overwrites the existing build if the build with same ID already exists.
func (f *Fake) AddBuild(build *bbpb.Build) {
	host := build.GetInfra().GetBuildbucket().GetHostname()
	if host == "" {
		panic(fmt.Errorf("missing host for build %d", build.Id))
	}
	fa := f.ensureApp(host)
	fa.buildStoreMu.Lock()
	fa.buildStore[build.GetId()] = build
	fa.buildStoreMu.Unlock()
}

func (f *Fake) ensureApp(host string) *fakeApp {
	f.hostsMu.Lock()
	defer f.hostsMu.Unlock()
	if _, ok := f.hosts[host]; !ok {
		if f.hosts == nil {
			f.hosts = make(map[string]*fakeApp)
		}
		f.hosts[host] = &fakeApp{
			buildStore: make(map[int64]*bbpb.Build),
		}
	}
	return f.hosts[host]
}

func (fa *fakeApp) getBuild(id int64) *bbpb.Build {
	fa.buildStoreMu.RLock()
	defer fa.buildStoreMu.RUnlock()
	if build, ok := fa.buildStore[id]; ok {
		return proto.Clone(build).(*bbpb.Build)
	}
	return nil
}

func (fa *fakeApp) iterBuildStore(cb func(*bbpb.Build)) {
	fa.buildStoreMu.RLock()
	defer fa.buildStoreMu.RUnlock()
	for _, build := range fa.buildStore {
		cb(proto.Clone(build).(*bbpb.Build))
	}
}

func (fa *fakeApp) updateBuild(id int64, cb func(*bbpb.Build)) *bbpb.Build {
	fa.buildStoreMu.Lock()
	defer fa.buildStoreMu.Unlock()
	if build, ok := fa.buildStore[id]; ok {
		cb(build)
		// store a copy to avoid cb keeps the reference to the build and mutate it
		// later.
		fa.buildStore[id] = proto.Clone(build).(*bbpb.Build)
		return build
	}
	return nil
}
<commit_msg>cv: add config store to buildbucket fake<commit_after>// Copyright 2022 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bbfake

import (
	"encoding/json"
	"fmt"
	"sync"

	bbpb "go.chromium.org/luci/buildbucket/proto"
	"google.golang.org/protobuf/proto"

	"go.chromium.org/luci/cv/internal/buildbucket"
)

type fakeApp struct {
	buildStoreMu  sync.RWMutex
	buildStore    map[int64]*bbpb.Build // build ID -> build
	configStoreMu sync.RWMutex
	configStore   map[string]*bbpb.BuildbucketCfg // project name -> config
}

type Fake struct {
	hostsMu sync.Mutex
	hosts   map[string]*fakeApp // hostname -> fakeApp
}

func (f *Fake) NewClientFactory() buildbucket.ClientFactory {
	return clientFactory{
		fake: f,
	}
}

// AddBuilder adds a new builder configuration to fake Buildbucket host.
//
// Overwrites the existing builder if the same builder already exists.
// `properties` should be marshallable by `encoding/json`.
func (f *Fake) AddBuilder(host string, builder *bbpb.BuilderID, properties interface{}) *Fake {
	fa := f.ensureApp(host)
	fa.configStoreMu.Lock()
	defer fa.configStoreMu.Unlock()
	if _, ok := fa.configStore[builder.GetProject()]; !ok {
		fa.configStore[builder.GetProject()] = &bbpb.BuildbucketCfg{}
	}
	cfg := fa.configStore[builder.GetProject()]
	var bucket *bbpb.Bucket
	for _, b := range cfg.GetBuckets() {
		if b.Name == builder.GetBucket() {
			bucket = b
			break
		}
	}
	if bucket == nil {
		bucket = &bbpb.Bucket{
			Name:     builder.GetBucket(),
			Swarming: &bbpb.Swarming{},
		}
		cfg.Buckets = append(cfg.GetBuckets(), bucket)
	}

	builderCfg := &bbpb.BuilderConfig{
		Name: builder.GetBuilder(),
	}
	if properties != nil {
		bProperties, err := json.Marshal(properties)
		if err != nil {
			panic(err)
		}
		builderCfg.Properties = string(bProperties)
	}
	for i, b := range bucket.GetSwarming().GetBuilders() {
		if b.Name == builder.GetBuilder() {
			bucket.GetSwarming().GetBuilders()[i] = builderCfg
			return f
		}
	}
	bucket.GetSwarming().Builders = append(bucket.GetSwarming().GetBuilders(), builderCfg)
	return f
}

// AddBuild adds a build to fake Buildbucket host.
//
// Reads Buildbucket hostname from `infra.buildbucket.hostname`.
// Overwrites the existing build if the build with same ID already exists.
func (f *Fake) AddBuild(build *bbpb.Build) {
	host := build.GetInfra().GetBuildbucket().GetHostname()
	if host == "" {
		panic(fmt.Errorf("missing host for build %d", build.Id))
	}
	fa := f.ensureApp(host)
	fa.buildStoreMu.Lock()
	fa.buildStore[build.GetId()] = build
	fa.buildStoreMu.Unlock()
}

func (f *Fake) ensureApp(host string) *fakeApp {
	f.hostsMu.Lock()
	defer f.hostsMu.Unlock()
	if _, ok := f.hosts[host]; !ok {
		if f.hosts == nil {
			f.hosts = make(map[string]*fakeApp)
		}
		f.hosts[host] = &fakeApp{
			buildStore: make(map[int64]*bbpb.Build),
		}
	}
	return f.hosts[host]
}

func (fa *fakeApp) getBuild(id int64) *bbpb.Build {
	fa.buildStoreMu.RLock()
	defer fa.buildStoreMu.RUnlock()
	if build, ok := fa.buildStore[id]; ok {
		return proto.Clone(build).(*bbpb.Build)
	}
	return nil
}

func (fa *fakeApp) iterBuildStore(cb func(*bbpb.Build)) {
	fa.buildStoreMu.RLock()
	defer fa.buildStoreMu.RUnlock()
	for _, build := range fa.buildStore {
		cb(proto.Clone(build).(*bbpb.Build))
	}
}

func (fa *fakeApp) updateBuild(id int64, cb func(*bbpb.Build)) *bbpb.Build {
	fa.buildStoreMu.Lock()
	defer fa.buildStoreMu.Unlock()
	if build, ok := fa.buildStore[id]; ok {
		cb(build)
		// store a copy to avoid cb keeps the reference to the build and mutate it
		// later.
		fa.buildStore[id] = proto.Clone(build).(*bbpb.Build)
		return build
	}
	return nil
}
<|endoftext|>


<commit_before>package solr

type SolrResponse struct {
	Status int `json:"status"`
	QTime  int `json:"qtime"`
	Params struct {
		Query  string `json:"q"`
		Indent string `json:"indent"`
		Wt     string `json:"wt"`
	} `json:"params"`
	Response       Response `json:"response"`
	NextCursorMark string   `json:"nextCursorMark"`
	Adds           Adds     `json:"adds"`
}

type Response struct {
	NumFound uint32                   `json:"numFound"`
	Start    int                      `json:"start"`
	Docs     []map[string]interface{} `json:"docs"`
}

func GetDocIdFromDoc(m map[string]interface{}) string {
	if v, ok := m["id"]; ok {
		return v.(string)
	}
	return ""
}

func GetVersionFromDoc(m map[string]interface{}) int {
	if v, ok := m["_version_"]; ok {
		return v.(int)
	}
	return 0

}

type Adds map[string]int

type UpdateResponse struct {
	Response struct {
		Status int `json:"status"`
		QTime  int `json:"QTime"`
		RF     int `json:"rf"`
		MinRF  int `json:"min_rf"`
	} `json:"responseHeader"`
	Error struct {
		Metadata []string `json:"metadata"`
		Msg      string   `json:"msg"`
		Code     int      `json:"code"`
	}
}

type DeleteRequest struct {
	Delete []string `json:"delete"`
}
<commit_msg>MCON-1193 solr cli<commit_after>package solr

type SolrResponse struct {
	Status int `json:"status"`
	QTime  int `json:"qtime"`
	Params struct {
		Query  string `json:"q"`
		Indent string `json:"indent"`
		Wt     string `json:"wt"`
	} `json:"params"`
	Response       Response `json:"response"`
	NextCursorMark string   `json:"nextCursorMark"`
	Adds           Adds     `json:"adds"`
}

type Response struct {
	NumFound uint32                   `json:"numFound"`
	Start    int                      `json:"start"`
	Docs     []map[string]interface{} `json:"docs"`
}

func GetDocIdFromDoc(m map[string]interface{}) string {
	if v, ok := m["id"]; ok {
		return v.(string)
	}
	return ""
}

func GetVersionFromDoc(m map[string]interface{}) int {
	if v, ok := m["_version_"]; ok {
		return int(v.(float64))
	}
	return 0
}

type Adds map[string]int

type UpdateResponse struct {
	Response struct {
		Status int `json:"status"`
		QTime  int `json:"QTime"`
		RF     int `json:"rf"`
		MinRF  int `json:"min_rf"`
	} `json:"responseHeader"`
	Error struct {
		Metadata []string `json:"metadata"`
		Msg      string   `json:"msg"`
		Code     int      `json:"code"`
	}
}

type DeleteRequest struct {
	Delete []string `json:"delete"`
}
<|endoftext|>


<commit_before>package redis

import (
	"crypto/tls"
	"errors"
	"fmt"
	"net"
	"net/url"
	"strconv"
	"strings"
	"time"

	"github.com/go-redis/redis/internal/pool"
)

type Options struct {
	// The network type, either tcp or unix.
	// Default is tcp.
	Network string
	// host:port address.
	Addr string

	// Dialer creates new network connection and has priority over
	// Network and Addr options.
	Dialer func() (net.Conn, error)

	// Optional password. Must match the password specified in the
	// requirepass server configuration option.
	Password string
	// Database to be selected after connecting to the server.
	DB int

	// Maximum number of retries before giving up.
	// Default is to not retry failed commands.
	MaxRetries int

	// Dial timeout for establishing new connections.
	// Default is 5 seconds.
	DialTimeout time.Duration
	// Timeout for socket reads. If reached, commands will fail
	// with a timeout instead of blocking.
	// Default is 3 seconds.
	ReadTimeout time.Duration
	// Timeout for socket writes. If reached, commands will fail
	// with a timeout instead of blocking.
	// Default is 3 seconds.
	WriteTimeout time.Duration

	// Maximum number of socket connections.
	// Default is 10 connections.
	PoolSize int
	// Amount of time client waits for connection if all connections
	// are busy before returning an error.
	// Default is ReadTimeout + 1 second.
	PoolTimeout time.Duration
	// Amount of time after which client closes idle connections.
	// Should be less than server's timeout.
	// Default is 5 minutes.
	IdleTimeout time.Duration
	// Frequency of idle checks.
	// Default is 1 minute.
	// When minus value is set, then idle check is disabled.
	IdleCheckFrequency time.Duration

	// Enables read only queries on slave nodes.
	ReadOnly bool

	// TLS Config to use. When set TLS will be negotiated.
	TLSConfig *tls.Config
}

func (opt *Options) init() {
	if opt.Network == "" {
		opt.Network = "tcp"
	}
	if opt.Dialer == nil {
		opt.Dialer = func() (net.Conn, error) {
			conn, err := net.DialTimeout(opt.Network, opt.Addr, opt.DialTimeout)
			if opt.TLSConfig == nil || err != nil {
				return conn, err
			}
			t := tls.Client(conn, opt.TLSConfig)
			return t, t.Handshake()
		}
	}
	if opt.PoolSize == 0 {
		opt.PoolSize = 100
	}
	if opt.DialTimeout == 0 {
		opt.DialTimeout = 5 * time.Second
	}
	if opt.ReadTimeout == 0 {
		opt.ReadTimeout = 3 * time.Second
	} else if opt.ReadTimeout == -1 {
		opt.ReadTimeout = 0
	}
	if opt.WriteTimeout == 0 {
		opt.WriteTimeout = opt.ReadTimeout
	} else if opt.WriteTimeout == -1 {
		opt.WriteTimeout = 0
	}
	if opt.PoolTimeout == 0 {
		opt.PoolTimeout = opt.ReadTimeout + time.Second
	}
	if opt.IdleTimeout == 0 {
		opt.IdleTimeout = 5 * time.Minute
	}
	if opt.IdleCheckFrequency == 0 {
		opt.IdleCheckFrequency = time.Minute
	}
}

// ParseURL parses a redis URL into options that can be used to connect to redis
func ParseURL(redisURL string) (*Options, error) {
	o := &Options{Network: "tcp"}
	u, err := url.Parse(redisURL)
	if err != nil {
		return nil, err
	}

	if u.Scheme != "redis" && u.Scheme != "rediss" {
		return nil, errors.New("invalid redis URL scheme: " + u.Scheme)
	}

	if u.User != nil {
		if p, ok := u.User.Password(); ok {
			o.Password = p
		}
	}

	if len(u.Query()) > 0 {
		return nil, errors.New("no options supported")
	}

	h, p, err := net.SplitHostPort(u.Host)
	if err != nil {
		h = u.Host
	}
	if h == "" {
		h = "localhost"
	}
	if p == "" {
		p = "6379"
	}
	o.Addr = net.JoinHostPort(h, p)

	f := strings.FieldsFunc(u.Path, func(r rune) bool {
		return r == '/'
	})
	switch len(f) {
	case 0:
		o.DB = 0
	case 1:
		if o.DB, err = strconv.Atoi(f[0]); err != nil {
			return nil, fmt.Errorf("invalid redis database number: %q", f[0])
		}
	default:
		return nil, errors.New("invalid redis URL path: " + u.Path)
	}

	if u.Scheme == "rediss" {
		o.TLSConfig = &tls.Config{ServerName: h}
	}
	return o, nil
}

func newConnPool(opt *Options) *pool.ConnPool {
	return pool.NewConnPool(
		opt.Dialer,
		opt.PoolSize,
		opt.PoolTimeout,
		opt.IdleTimeout,
		opt.IdleCheckFrequency,
	)
}

// PoolStats contains pool state information and accumulated stats.
type PoolStats struct {
	Requests uint32 // number of times a connection was requested by the pool
	Hits     uint32 // number of times free connection was found in the pool
	Timeouts uint32 // number of times a wait timeout occurred

	TotalConns uint32 // the number of total connections in the pool
	FreeConns  uint32 // the number of free connections in the pool
}
<commit_msg>doc: update more Options default value<commit_after>package redis

import (
	"crypto/tls"
	"errors"
	"fmt"
	"net"
	"net/url"
	"strconv"
	"strings"
	"time"

	"github.com/go-redis/redis/internal/pool"
)

type Options struct {
	// The network type, either tcp or unix.
	// Default is tcp.
	Network string
	// host:port address.
	Addr string

	// Dialer creates new network connection and has priority over
	// Network and Addr options.
	Dialer func() (net.Conn, error)

	// Optional password. Must match the password specified in the
	// requirepass server configuration option.
	Password string
	// Database to be selected after connecting to the server.
	DB int

	// Maximum number of retries before giving up.
	// Default is to not retry failed commands.
	MaxRetries int

	// Dial timeout for establishing new connections.
	// Default is 5 seconds.
	DialTimeout time.Duration
	// Timeout for socket reads. If reached, commands will fail
	// with a timeout instead of blocking.
	// Default is 3 seconds.
	ReadTimeout time.Duration
	// Timeout for socket writes. If reached, commands will fail
	// with a timeout instead of blocking.
	// Default is ReadTimeout.
	WriteTimeout time.Duration

	// Maximum number of socket connections.
	// Default is 100 connections.
	PoolSize int
	// Amount of time client waits for connection if all connections
	// are busy before returning an error.
	// Default is ReadTimeout + 1 second.
	PoolTimeout time.Duration
	// Amount of time after which client closes idle connections.
	// Should be less than server's timeout.
	// Default is 5 minutes.
	IdleTimeout time.Duration
	// Frequency of idle checks.
	// Default is 1 minute.
	// When minus value is set, then idle check is disabled.
	IdleCheckFrequency time.Duration

	// Enables read only queries on slave nodes.
	ReadOnly bool

	// TLS Config to use. When set TLS will be negotiated.
	TLSConfig *tls.Config
}

func (opt *Options) init() {
	if opt.Network == "" {
		opt.Network = "tcp"
	}
	if opt.Dialer == nil {
		opt.Dialer = func() (net.Conn, error) {
			conn, err := net.DialTimeout(opt.Network, opt.Addr, opt.DialTimeout)
			if opt.TLSConfig == nil || err != nil {
				return conn, err
			}
			t := tls.Client(conn, opt.TLSConfig)
			return t, t.Handshake()
		}
	}
	if opt.PoolSize == 0 {
		opt.PoolSize = 100
	}
	if opt.DialTimeout == 0 {
		opt.DialTimeout = 5 * time.Second
	}
	if opt.ReadTimeout == 0 {
		opt.ReadTimeout = 3 * time.Second
	} else if opt.ReadTimeout == -1 {
		opt.ReadTimeout = 0
	}
	if opt.WriteTimeout == 0 {
		opt.WriteTimeout = opt.ReadTimeout
	} else if opt.WriteTimeout == -1 {
		opt.WriteTimeout = 0
	}
	if opt.PoolTimeout == 0 {
		opt.PoolTimeout = opt.ReadTimeout + time.Second
	}
	if opt.IdleTimeout == 0 {
		opt.IdleTimeout = 5 * time.Minute
	}
	if opt.IdleCheckFrequency == 0 {
		opt.IdleCheckFrequency = time.Minute
	}
}

// ParseURL parses a redis URL into options that can be used to connect to redis
func ParseURL(redisURL string) (*Options, error) {
	o := &Options{Network: "tcp"}
	u, err := url.Parse(redisURL)
	if err != nil {
		return nil, err
	}

	if u.Scheme != "redis" && u.Scheme != "rediss" {
		return nil, errors.New("invalid redis URL scheme: " + u.Scheme)
	}

	if u.User != nil {
		if p, ok := u.User.Password(); ok {
			o.Password = p
		}
	}

	if len(u.Query()) > 0 {
		return nil, errors.New("no options supported")
	}

	h, p, err := net.SplitHostPort(u.Host)
	if err != nil {
		h = u.Host
	}
	if h == "" {
		h = "localhost"
	}
	if p == "" {
		p = "6379"
	}
	o.Addr = net.JoinHostPort(h, p)

	f := strings.FieldsFunc(u.Path, func(r rune) bool {
		return r == '/'
	})
	switch len(f) {
	case 0:
		o.DB = 0
	case 1:
		if o.DB, err = strconv.Atoi(f[0]); err != nil {
			return nil, fmt.Errorf("invalid redis database number: %q", f[0])
		}
	default:
		return nil, errors.New("invalid redis URL path: " + u.Path)
	}

	if u.Scheme == "rediss" {
		o.TLSConfig = &tls.Config{ServerName: h}
	}
	return o, nil
}

func newConnPool(opt *Options) *pool.ConnPool {
	return pool.NewConnPool(
		opt.Dialer,
		opt.PoolSize,
		opt.PoolTimeout,
		opt.IdleTimeout,
		opt.IdleCheckFrequency,
	)
}

// PoolStats contains pool state information and accumulated stats.
type PoolStats struct {
	Requests uint32 // number of times a connection was requested by the pool
	Hits     uint32 // number of times free connection was found in the pool
	Timeouts uint32 // number of times a wait timeout occurred

	TotalConns uint32 // the number of total connections in the pool
	FreeConns  uint32 // the number of free connections in the pool
}
<|endoftext|>


<commit_before>/*
 Copyright 2016 Padduck, LLC

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 	http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package sftp

import (
	"github.com/pkg/errors"
	"github.com/pufferpanel/pufferd/utils"
	"github.com/pufferpanel/sftp"
	"os"
	"path/filepath"
	"strings"
	"time"
)

type PrefixFileSystem struct {
	sftp.FileSystem
	prefix string
}

func CreateVirtualFs(prefix string) sftp.FileSystem {
	return PrefixFileSystem{prefix: prefix}
}

func (fs PrefixFileSystem) Stat(path string) (os.FileInfo, error) {
	p, err := fs.validate(path)
	if err != nil {
		return nil, err
	}
	fi, e := os.Stat(p)
	if e != nil {
		e = fs.maskError(e)
	}
	return fi, e
}

func (fs PrefixFileSystem) Lstat(path string) (os.FileInfo, error) {
	p, err := fs.validate(path)
	if err != nil {
		return nil, err
	}
	fi, e := os.Lstat(p)
	if e != nil {
		e = fs.maskError(e)
	}
	return fi, e
}

func (fs PrefixFileSystem) Mkdir(path string, mode os.FileMode) error {
	p, err := fs.validate(path)
	if err != nil {
		return err
	}
	e := os.Mkdir(p, mode)
	if e != nil {
		e = fs.maskError(e)
	}
	return e
}

func (fs PrefixFileSystem) Remove(path string) error {
	p, err := fs.validate(path)
	if err != nil {
		return err
	}
	e := os.Remove(p)
	if e != nil {
		e = fs.maskError(e)
	}
	return e
}

func (fs PrefixFileSystem) Symlink(target string, link string) error {
	t, err := fs.validate(target)
	if err != nil {
		return err
	}
	l, err := fs.validate(link)
	if err != nil {
		return err
	}
	e := os.Symlink(t, l)
	if e != nil {
		e = fs.maskError(e)
	}
	return e
}

func (fs PrefixFileSystem) Readlink(path string) (string, error) {
	p, err := fs.validate(path)
	if err != nil {
		return "", err
	}
	return os.Readlink(p)
}

func (fs PrefixFileSystem) OpenFile(path string, flag int, mode os.FileMode) (*os.File, error) {
	p, err := fs.validate(path)
	if err != nil {
		return nil, err
	}
	return os.OpenFile(p, flag, mode)
}

func (fs PrefixFileSystem) Truncate(path string, size int64) error {
	p, err := fs.validate(path)
	if err != nil {
		return err
	}
	return os.Truncate(p, size)
}

func (fs PrefixFileSystem) Chmod(path string, mode os.FileMode) error {
	p, err := fs.validate(path)
	if err != nil {
		return err
	}
	e := os.Chmod(p, mode)
	if e != nil {
		e = fs.maskError(e)
	}
	return e
}

func (fs PrefixFileSystem) Chtimes(path string, aTime, mTime time.Time) error {
	p, err := fs.validate(path)
	if err != nil {
		return err
	}
	e := os.Chtimes(p, aTime, mTime)
	if e != nil {
		e = fs.maskError(e)
	}
	return e
}

func (fs PrefixFileSystem) Chown(path string, uid, gid int) error {
	return nil
}

func (fs PrefixFileSystem) Rename(oldPath string, newPath string) error {
	path1, err := fs.validate(oldPath)
	if err != nil {
		return err
	}
	path2, err := fs.validate(newPath)
	if err != nil {
		return err
	}
	e := os.Rename(path1, path2)
	if e != nil {
		e = fs.maskError(e)
	}
	return e
}

func (fs PrefixFileSystem) validate(path string) (string, error) {
	ok, path := fs.tryPrefix(path)
	if !ok {
		return "", errors.New("Invalid path provided")
	}
	return path, nil
}

func (fs PrefixFileSystem) tryPrefix(path string) (bool, string) {
	newPath := filepath.Clean(filepath.Join(fs.prefix, path))
	if utils.EnsureAccess(newPath, fs.prefix) {
		return true, newPath
	} else {
		return false, ""
	}
}

func (fs PrefixFileSystem) maskError(err error) error {
	return errors.New(strings.Replace(err.Error(), fs.prefix, "", -1))
}
<commit_msg>This file was not used anymore by us<commit_after><|endoftext|>


<commit_before>// Copyright GoFrame gf Author(https://github.com/gogf/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://github.com/gogf/gf.

package gdb_test

import (
	"github.com/gogf/gf/frame/g"
	"testing"

	"github.com/gogf/gf/database/gdb"
	"github.com/gogf/gf/test/gtest"
)

func Test_Insert_Raw(t *testing.T) {
	table := createTable()
	defer dropTable(table)

	gtest.C(t, func(t *gtest.T) {
		user := db.Table(table)
		result, err := user.Filter().Data(g.Map{
			"id":          gdb.Raw("id+2"),
			"passport":    "port_1",
			"password":    "pass_1",
			"nickname":    "name_1",
			"create_time": gdb.Raw("now()"),
		}).Insert()
		t.Assert(err, nil)
		n, _ := result.LastInsertId()
		t.Assert(n, 2)
	})
}

func Test_BatchInsert_Raw(t *testing.T) {
	table := createTable()
	defer dropTable(table)

	gtest.C(t, func(t *gtest.T) {
		user := db.Table(table)
		result, err := user.Filter().Data(
			g.List{
				g.Map{
					"id":          gdb.Raw("id+2"),
					"passport":    "port_2",
					"password":    "pass_2",
					"nickname":    "name_2",
					"create_time": gdb.Raw("now()"),
				},
				g.Map{
					"id":          gdb.Raw("id+4"),
					"passport":    "port_4",
					"password":    "pass_4",
					"nickname":    "name_4",
					"create_time": gdb.Raw("now()"),
				},
			},
		).Insert()
		t.Assert(err, nil)
		n, _ := result.LastInsertId()
		t.Assert(n, 4)
	})
}

func Test_Update_Raw(t *testing.T) {
	table := createInitTable()
	defer dropTable(table)

	gtest.C(t, func(t *gtest.T) {
		user := db.Table(table)
		result, err := user.Data(g.Map{
			"id":          gdb.Raw("id+100"),
			"create_time": gdb.Raw("now()"),
		}).Where("id", 1).Insert()
		t.Assert(err, nil)
		n, _ := result.RowsAffected()
		t.Assert(n, 1)
	})
}
<commit_msg>improve unit testing case for package gdb<commit_after>// Copyright GoFrame gf Author(https://github.com/gogf/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://github.com/gogf/gf.

package gdb_test

import (
	"github.com/gogf/gf/frame/g"
	"testing"

	"github.com/gogf/gf/database/gdb"
	"github.com/gogf/gf/test/gtest"
)

func Test_Insert_Raw(t *testing.T) {
	table := createTable()
	defer dropTable(table)

	gtest.C(t, func(t *gtest.T) {
		user := db.Table(table)
		result, err := user.Filter().Data(g.Map{
			"id":          gdb.Raw("id+2"),
			"passport":    "port_1",
			"password":    "pass_1",
			"nickname":    "name_1",
			"create_time": gdb.Raw("now()"),
		}).Insert()
		t.Assert(err, nil)
		n, _ := result.LastInsertId()
		t.Assert(n, 2)
	})
}

func Test_BatchInsert_Raw(t *testing.T) {
	table := createTable()
	defer dropTable(table)

	gtest.C(t, func(t *gtest.T) {
		user := db.Table(table)
		result, err := user.Filter().Data(
			g.List{
				g.Map{
					"id":          gdb.Raw("id+2"),
					"passport":    "port_2",
					"password":    "pass_2",
					"nickname":    "name_2",
					"create_time": gdb.Raw("now()"),
				},
				g.Map{
					"id":          gdb.Raw("id+4"),
					"passport":    "port_4",
					"password":    "pass_4",
					"nickname":    "name_4",
					"create_time": gdb.Raw("now()"),
				},
			},
		).Insert()
		t.Assert(err, nil)
		n, _ := result.LastInsertId()
		t.Assert(n, 4)
	})
}

func Test_Update_Raw(t *testing.T) {
	table := createInitTable()
	defer dropTable(table)

	gtest.C(t, func(t *gtest.T) {
		user := db.Table(table)
		result, err := user.Data(g.Map{
			"id":          gdb.Raw("id+100"),
			"create_time": gdb.Raw("now()"),
		}).Where("id", 1).Update()
		t.Assert(err, nil)
		n, _ := result.RowsAffected()
		t.Assert(n, 1)
	})
	gtest.C(t, func(t *gtest.T) {
		user := db.Table(table)
		n, err := user.Where("id", 101).Count()
		t.Assert(err, nil)
		t.Assert(n, 1)
	})
}
<|endoftext|>


<commit_before>// Copyright (c) 2012 VMware, Inc.

package sigar

// #include <stdlib.h>
// #include <windows.h>
import "C"

import (
	"bytes"
	"fmt"
	"os"
	"path/filepath"
	"syscall"
	"time"
	"unsafe"
)

var (
	modpsapi    = syscall.NewLazyDLL("psapi.dll")
	modkernel32 = syscall.NewLazyDLL("kernel32.dll")

	procEnumProcesses            = modpsapi.NewProc("EnumProcesses")
	procGetProcessMemoryInfo     = modpsapi.NewProc("GetProcessMemoryInfo")
	procGetProcessTimes          = modkernel32.NewProc("GetProcessTimes")
	procGetProcessImageFileName  = modpsapi.NewProc("GetProcessImageFileNameA")
	procCreateToolhelp32Snapshot = modkernel32.NewProc("CreateToolhelp32Snapshot")
	procProcess32First           = modkernel32.NewProc("Process32FirstW")

	procGetDiskFreeSpaceExW     = modkernel32.NewProc("GetDiskFreeSpaceExW")
	procGetLogicalDriveStringsW = modkernel32.NewProc("GetLogicalDriveStringsW")
	procGetDriveType            = modkernel32.NewProc("GetDriveTypeW")
	provGetVolumeInformation    = modkernel32.NewProc("GetVolumeInformationW")
)

const (
	PROCESS_ALL_ACCESS = 0x001f0fff
	TH32CS_SNAPPROCESS = 0x02
	MAX_PATH           = 260
)

type PROCESS_MEMORY_COUNTERS_EX struct {
	CB                         uint32
	PageFaultCount             uint32
	PeakWorkingSetSize         uintptr
	WorkingSetSize             uintptr
	QuotaPeakPagedPoolUsage    uintptr
	QuotaPagedPoolUsage        uintptr
	QuotaPeakNonPagedPoolUsage uintptr
	QuotaNonPagedPoolUsage     uintptr
	PagefileUsage              uintptr
	PeakPagefileUsage          uintptr
	PrivateUsage               uintptr
}

// PROCESSENTRY32 is the Windows API structure that contains a process's
// information.
type PROCESSENTRY32 struct {
	Size              uint32
	CntUsage          uint32
	ProcessID         uint32
	DefaultHeapID     uintptr
	ModuleID          uint32
	CntThreads        uint32
	ParentProcessID   uint32
	PriorityClassBase int32
	Flags             uint32
	ExeFile           [MAX_PATH]uint16
}

func init() {
}

func (self *LoadAverage) Get() error {
	return nil
}

func (self *Uptime) Get() error {
	return nil
}

func (self *Mem) Get() error {
	var statex C.MEMORYSTATUSEX
	statex.dwLength = C.DWORD(unsafe.Sizeof(statex))

	succeeded := C.GlobalMemoryStatusEx(&statex)
	if succeeded == C.FALSE {
		return syscall.GetLastError()
	}

	self.Total = uint64(statex.ullTotalPhys)
	self.Free = uint64(statex.ullAvailPhys)
	self.Used = self.Total - self.Free
	vtotal := uint64(statex.ullTotalVirtual)
	self.ActualFree = uint64(statex.ullAvailVirtual)
	self.ActualUsed = vtotal - self.ActualFree

	return nil
}

func (self *Swap) Get() error {
	//return notImplemented()
	return nil
}

func (self *Cpu) Get() error {

	var lpIdleTime, lpKernelTime, lpUserTime C.FILETIME

	succeeded := C.GetSystemTimes(&lpIdleTime, &lpKernelTime, &lpUserTime)
	if succeeded == C.FALSE {
		return syscall.GetLastError()
	}

	LOT := float64(0.0000001)
	HIT := (LOT * 4294967296.0)

	idle := ((HIT * float64(lpIdleTime.dwHighDateTime)) + (LOT * float64(lpIdleTime.dwLowDateTime)))
	user := ((HIT * float64(lpUserTime.dwHighDateTime)) + (LOT * float64(lpUserTime.dwLowDateTime)))
	kernel := ((HIT * float64(lpKernelTime.dwHighDateTime)) + (LOT * float64(lpKernelTime.dwLowDateTime)))
	system := (kernel - idle)

	self.Idle = uint64(idle)
	self.User = uint64(user)
	self.Sys = uint64(system)
	return nil
}

func (self *CpuList) Get() error {
	return notImplemented()
}

func (self *FileSystemList) Get() error {

	/*
		Get a list of the disks:
		fsutil fsinfo drives

		Get driver type:
		fsutil fsinfo drivetype C:

		Get volume info:
		fsutil fsinfo volumeinfo C:
	*/

	NullTermToStrings := func(b []byte) []string {
		list := []string{}
		for _, x := range bytes.SplitN(b, []byte{0, 0}, -1) {
			x = bytes.Replace(x, []byte{0}, []byte{}, -1)
			if len(x) == 0 {
				break
			}
			list = append(list, string(x))
		}
		return list
	}

	GetDriveTypeString := func(drivetype uintptr) string {
		switch drivetype {
		case 1:
			return "Invalid"
		case 2:
			return "Removable drive"
		case 3:
			return "Fixed drive"
		case 4:
			return "Remote drive"
		case 5:
			return "CDROM"
		case 6:
			return "RAM disk"
		default:
			return "Unknown"
		}
	}

	lpBuffer := make([]byte, 254)

	ret, _, _ := procGetLogicalDriveStringsW.Call(
		uintptr(len(lpBuffer)),
		uintptr(unsafe.Pointer(&lpBuffer[0])))
	if ret == 0 {
		return fmt.Errorf("GetLogicalDriveStringsW %v", syscall.GetLastError())
	}
	fss := NullTermToStrings(lpBuffer)

	for _, fs := range fss {
		typepath, _ := syscall.UTF16PtrFromString(fs)
		typeret, _, _ := procGetDriveType.Call(uintptr(unsafe.Pointer(typepath)))
		if typeret == 0 {
			return fmt.Errorf("GetDriveTypeW %v", syscall.GetLastError())
		}
		/* TODO volumeinfo by calling GetVolumeInformationW */

		d := FileSystem{
			DirName:  fs,
			DevName:  fs,
			TypeName: GetDriveTypeString(typeret),
		}
		self.List = append(self.List, d)
	}
	return nil
}

// Retrieves the process identifier for each process object in the system.

func (self *ProcList) Get() error {

	var enumSize int
	var pids [1024]C.DWORD

	// If the function succeeds, the return value is nonzero.
	ret, _, _ := procEnumProcesses.Call(
		uintptr(unsafe.Pointer(&pids[0])),
		uintptr(unsafe.Sizeof(pids)),
		uintptr(unsafe.Pointer(&enumSize)),
	)
	if ret == 0 {
		return syscall.GetLastError()
	}

	results := []int{}

	pids_size := enumSize / int(unsafe.Sizeof(pids[0]))

	for _, pid := range pids[:pids_size] {
		results = append(results, int(pid))
	}

	self.List = results

	return nil
}

func FiletimeToDuration(ft *syscall.Filetime) time.Duration {
	n := int64(ft.HighDateTime)<<32 + int64(ft.LowDateTime) // in 100-nanosecond intervals
	return time.Duration(n*100) * time.Nanosecond
}

func CarrayToString(c [MAX_PATH]byte) string {
	end := 0
	for {
		if c[end] == 0 {
			break
		}
		end++
	}
	return string(c[:end])
}

func (self *ProcState) Get(pid int) error {

	var err error

	self.Name, err = GetProcName(pid)
	if err != nil {
		return err
	}

	self.State, err = GetProcStatus(pid)
	if err != nil {
		return err
	}

	self.Ppid, err = GetParentPid(pid)
	if err != nil {
		return err
	}
	return nil
}

func GetProcName(pid int) (string, error) {

	handle, err := syscall.OpenProcess(syscall.PROCESS_QUERY_INFORMATION, false, uint32(pid))

	defer syscall.CloseHandle(handle)

	if err != nil {
		return "", fmt.Errorf("OpenProcess fails with %v", err)
	}

	var nameProc [MAX_PATH]byte

	ret, _, _ := procGetProcessImageFileName.Call(
		uintptr(handle),
		uintptr(unsafe.Pointer(&nameProc)),
		uintptr(MAX_PATH),
	)
	if ret == 0 {
		return "", syscall.GetLastError()
	}

	return filepath.Base(CarrayToString(nameProc)), nil

}

func GetProcStatus(pid int) (RunState, error) {

	handle, err := syscall.OpenProcess(syscall.PROCESS_QUERY_INFORMATION, false, uint32(pid))

	defer syscall.CloseHandle(handle)

	if err != nil {
		return RunStateUnknown, fmt.Errorf("OpenProcess fails with %v", err)
	}

	var ec uint32
	e := syscall.GetExitCodeProcess(syscall.Handle(handle), &ec)
	if e != nil {
		return RunStateUnknown, os.NewSyscallError("GetExitCodeProcess", e)
	}
	if ec == 259 { //still active
		return RunStateRun, nil
	}
	return RunStateSleep, nil
}

func GetParentPid(pid int) (int, error) {

	handle, _, _ := procCreateToolhelp32Snapshot.Call(
		uintptr(TH32CS_SNAPPROCESS),
		uintptr(uint32(pid)),
	)
	if handle < 0 {
		return 0, syscall.GetLastError()
	}

	var entry PROCESSENTRY32
	entry.Size = uint32(unsafe.Sizeof(entry))

	ret, _, _ := procProcess32First.Call(handle, uintptr(unsafe.Pointer(&entry)))
	if ret == 0 {
		return 0, fmt.Errorf("Error retrieving process info.")
	}
	return int(entry.ParentProcessID), nil

}

func (self *ProcMem) Get(pid int) error {
	handle, err := syscall.OpenProcess(PROCESS_ALL_ACCESS, false, uint32(pid))

	defer syscall.CloseHandle(handle)

	if err != nil {
		return fmt.Errorf("OpenProcess fails with %v", err)
	}

	var mem PROCESS_MEMORY_COUNTERS_EX
	mem.CB = uint32(unsafe.Sizeof(mem))

	r1, _, e1 := procGetProcessMemoryInfo.Call(
		uintptr(handle),
		uintptr(unsafe.Pointer(&mem)),
		uintptr(mem.CB),
	)
	if r1 == 0 {
		if e1 != nil {
			return error(e1)
		} else {
			return syscall.EINVAL
		}
	}

	self.Resident = uint64(mem.WorkingSetSize)
	self.Size = uint64(mem.PrivateUsage)
	// Size contains only to the Private Bytes
	// Virtual Bytes are the Working Set plus paged Private Bytes and standby list.
	return nil
}

func (self *ProcTime) Get(pid int) error {
	handle, err := syscall.OpenProcess(syscall.PROCESS_QUERY_INFORMATION, false, uint32(pid))

	defer syscall.CloseHandle(handle)

	if err != nil {
		return fmt.Errorf("OpenProcess fails with %v", err)

	}
	var CPU syscall.Rusage
	if err := syscall.GetProcessTimes(handle, &CPU.CreationTime, &CPU.ExitTime, &CPU.KernelTime, &CPU.UserTime); err != nil {
		return fmt.Errorf("GetProcessTimes fails with %v", err)
	}

	// convert to millis
	self.StartTime = uint64(FiletimeToDuration(&CPU.CreationTime).Nanoseconds() / 1e6)

	self.User = uint64(FiletimeToDuration(&CPU.UserTime).Nanoseconds() / 1e6)

	self.Sys = uint64(FiletimeToDuration(&CPU.KernelTime).Nanoseconds() / 1e6)

	self.Total = self.User + self.Sys

	return nil
}

func (self *ProcArgs) Get(pid int) error {
	return notImplemented()
}

func (self *ProcExe) Get(pid int) error {
	return notImplemented()
}

func (self *FileSystemUsage) Get(path string) error {

	/*
		Get free, available, total free bytes:
		fsutil volume diskfree C:
	*/
	var availableBytes C.ULARGE_INTEGER
	var totalBytes C.ULARGE_INTEGER
	var totalFreeBytes C.ULARGE_INTEGER

	pathChars := C.CString(path)
	defer C.free(unsafe.Pointer(pathChars))

	succeeded := C.GetDiskFreeSpaceEx((*C.CHAR)(pathChars), &availableBytes, &totalBytes, &totalFreeBytes)
	if succeeded == C.FALSE {
		return syscall.GetLastError()
	}

	self.Total = *(*uint64)(unsafe.Pointer(&totalBytes))
	self.Free = *(*uint64)(unsafe.Pointer(&totalFreeBytes))
	self.Used = self.Total - self.Free
	self.Avail = *(*uint64)(unsafe.Pointer(&availableBytes))

	return nil
}

func notImplemented() error {
	panic("Not Implemented")
	return nil
}
<commit_msg>Add a deferred close for the handle returned by CreateToolhelp32Snapshot on Windows. The handle points to a snapshot of the process'es information and was previously being leaked when getting ProcState.<commit_after>// Copyright (c) 2012 VMware, Inc.

package sigar

// #include <stdlib.h>
// #include <windows.h>
import "C"

import (
	"bytes"
	"fmt"
	"os"
	"path/filepath"
	"syscall"
	"time"
	"unsafe"
)

var (
	modpsapi    = syscall.NewLazyDLL("psapi.dll")
	modkernel32 = syscall.NewLazyDLL("kernel32.dll")

	procEnumProcesses            = modpsapi.NewProc("EnumProcesses")
	procGetProcessMemoryInfo     = modpsapi.NewProc("GetProcessMemoryInfo")
	procGetProcessTimes          = modkernel32.NewProc("GetProcessTimes")
	procGetProcessImageFileName  = modpsapi.NewProc("GetProcessImageFileNameA")
	procCreateToolhelp32Snapshot = modkernel32.NewProc("CreateToolhelp32Snapshot")
	procProcess32First           = modkernel32.NewProc("Process32FirstW")

	procGetDiskFreeSpaceExW     = modkernel32.NewProc("GetDiskFreeSpaceExW")
	procGetLogicalDriveStringsW = modkernel32.NewProc("GetLogicalDriveStringsW")
	procGetDriveType            = modkernel32.NewProc("GetDriveTypeW")
	provGetVolumeInformation    = modkernel32.NewProc("GetVolumeInformationW")
)

const (
	PROCESS_ALL_ACCESS = 0x001f0fff
	TH32CS_SNAPPROCESS = 0x02
	MAX_PATH           = 260
)

type PROCESS_MEMORY_COUNTERS_EX struct {
	CB                         uint32
	PageFaultCount             uint32
	PeakWorkingSetSize         uintptr
	WorkingSetSize             uintptr
	QuotaPeakPagedPoolUsage    uintptr
	QuotaPagedPoolUsage        uintptr
	QuotaPeakNonPagedPoolUsage uintptr
	QuotaNonPagedPoolUsage     uintptr
	PagefileUsage              uintptr
	PeakPagefileUsage          uintptr
	PrivateUsage               uintptr
}

// PROCESSENTRY32 is the Windows API structure that contains a process's
// information.
type PROCESSENTRY32 struct {
	Size              uint32
	CntUsage          uint32
	ProcessID         uint32
	DefaultHeapID     uintptr
	ModuleID          uint32
	CntThreads        uint32
	ParentProcessID   uint32
	PriorityClassBase int32
	Flags             uint32
	ExeFile           [MAX_PATH]uint16
}

func init() {
}

func (self *LoadAverage) Get() error {
	return nil
}

func (self *Uptime) Get() error {
	return nil
}

func (self *Mem) Get() error {
	var statex C.MEMORYSTATUSEX
	statex.dwLength = C.DWORD(unsafe.Sizeof(statex))

	succeeded := C.GlobalMemoryStatusEx(&statex)
	if succeeded == C.FALSE {
		return syscall.GetLastError()
	}

	self.Total = uint64(statex.ullTotalPhys)
	self.Free = uint64(statex.ullAvailPhys)
	self.Used = self.Total - self.Free
	vtotal := uint64(statex.ullTotalVirtual)
	self.ActualFree = uint64(statex.ullAvailVirtual)
	self.ActualUsed = vtotal - self.ActualFree

	return nil
}

func (self *Swap) Get() error {
	//return notImplemented()
	return nil
}

func (self *Cpu) Get() error {

	var lpIdleTime, lpKernelTime, lpUserTime C.FILETIME

	succeeded := C.GetSystemTimes(&lpIdleTime, &lpKernelTime, &lpUserTime)
	if succeeded == C.FALSE {
		return syscall.GetLastError()
	}

	LOT := float64(0.0000001)
	HIT := (LOT * 4294967296.0)

	idle := ((HIT * float64(lpIdleTime.dwHighDateTime)) + (LOT * float64(lpIdleTime.dwLowDateTime)))
	user := ((HIT * float64(lpUserTime.dwHighDateTime)) + (LOT * float64(lpUserTime.dwLowDateTime)))
	kernel := ((HIT * float64(lpKernelTime.dwHighDateTime)) + (LOT * float64(lpKernelTime.dwLowDateTime)))
	system := (kernel - idle)

	self.Idle = uint64(idle)
	self.User = uint64(user)
	self.Sys = uint64(system)
	return nil
}

func (self *CpuList) Get() error {
	return notImplemented()
}

func (self *FileSystemList) Get() error {

	/*
		Get a list of the disks:
		fsutil fsinfo drives

		Get driver type:
		fsutil fsinfo drivetype C:

		Get volume info:
		fsutil fsinfo volumeinfo C:
	*/

	NullTermToStrings := func(b []byte) []string {
		list := []string{}
		for _, x := range bytes.SplitN(b, []byte{0, 0}, -1) {
			x = bytes.Replace(x, []byte{0}, []byte{}, -1)
			if len(x) == 0 {
				break
			}
			list = append(list, string(x))
		}
		return list
	}

	GetDriveTypeString := func(drivetype uintptr) string {
		switch drivetype {
		case 1:
			return "Invalid"
		case 2:
			return "Removable drive"
		case 3:
			return "Fixed drive"
		case 4:
			return "Remote drive"
		case 5:
			return "CDROM"
		case 6:
			return "RAM disk"
		default:
			return "Unknown"
		}
	}

	lpBuffer := make([]byte, 254)

	ret, _, _ := procGetLogicalDriveStringsW.Call(
		uintptr(len(lpBuffer)),
		uintptr(unsafe.Pointer(&lpBuffer[0])))
	if ret == 0 {
		return fmt.Errorf("GetLogicalDriveStringsW %v", syscall.GetLastError())
	}
	fss := NullTermToStrings(lpBuffer)

	for _, fs := range fss {
		typepath, _ := syscall.UTF16PtrFromString(fs)
		typeret, _, _ := procGetDriveType.Call(uintptr(unsafe.Pointer(typepath)))
		if typeret == 0 {
			return fmt.Errorf("GetDriveTypeW %v", syscall.GetLastError())
		}
		/* TODO volumeinfo by calling GetVolumeInformationW */

		d := FileSystem{
			DirName:  fs,
			DevName:  fs,
			TypeName: GetDriveTypeString(typeret),
		}
		self.List = append(self.List, d)
	}
	return nil
}

// Retrieves the process identifier for each process object in the system.

func (self *ProcList) Get() error {

	var enumSize int
	var pids [1024]C.DWORD

	// If the function succeeds, the return value is nonzero.
	ret, _, _ := procEnumProcesses.Call(
		uintptr(unsafe.Pointer(&pids[0])),
		uintptr(unsafe.Sizeof(pids)),
		uintptr(unsafe.Pointer(&enumSize)),
	)
	if ret == 0 {
		return syscall.GetLastError()
	}

	results := []int{}

	pids_size := enumSize / int(unsafe.Sizeof(pids[0]))

	for _, pid := range pids[:pids_size] {
		results = append(results, int(pid))
	}

	self.List = results

	return nil
}

func FiletimeToDuration(ft *syscall.Filetime) time.Duration {
	n := int64(ft.HighDateTime)<<32 + int64(ft.LowDateTime) // in 100-nanosecond intervals
	return time.Duration(n*100) * time.Nanosecond
}

func CarrayToString(c [MAX_PATH]byte) string {
	end := 0
	for {
		if c[end] == 0 {
			break
		}
		end++
	}
	return string(c[:end])
}

func (self *ProcState) Get(pid int) error {

	var err error

	self.Name, err = GetProcName(pid)
	if err != nil {
		return err
	}

	self.State, err = GetProcStatus(pid)
	if err != nil {
		return err
	}

	self.Ppid, err = GetParentPid(pid)
	if err != nil {
		return err
	}
	return nil
}

func GetProcName(pid int) (string, error) {

	handle, err := syscall.OpenProcess(syscall.PROCESS_QUERY_INFORMATION, false, uint32(pid))

	defer syscall.CloseHandle(handle)

	if err != nil {
		return "", fmt.Errorf("OpenProcess fails with %v", err)
	}

	var nameProc [MAX_PATH]byte

	ret, _, _ := procGetProcessImageFileName.Call(
		uintptr(handle),
		uintptr(unsafe.Pointer(&nameProc)),
		uintptr(MAX_PATH),
	)
	if ret == 0 {
		return "", syscall.GetLastError()
	}

	return filepath.Base(CarrayToString(nameProc)), nil

}

func GetProcStatus(pid int) (RunState, error) {

	handle, err := syscall.OpenProcess(syscall.PROCESS_QUERY_INFORMATION, false, uint32(pid))

	defer syscall.CloseHandle(handle)

	if err != nil {
		return RunStateUnknown, fmt.Errorf("OpenProcess fails with %v", err)
	}

	var ec uint32
	e := syscall.GetExitCodeProcess(syscall.Handle(handle), &ec)
	if e != nil {
		return RunStateUnknown, os.NewSyscallError("GetExitCodeProcess", e)
	}
	if ec == 259 { //still active
		return RunStateRun, nil
	}
	return RunStateSleep, nil
}

func GetParentPid(pid int) (int, error) {

	handle, _, _ := procCreateToolhelp32Snapshot.Call(
		uintptr(TH32CS_SNAPPROCESS),
		uintptr(uint32(pid)),
	)
	if handle < 0 {
		return 0, syscall.GetLastError()
	}
	defer syscall.CloseHandle(syscall.Handle(handle))

	var entry PROCESSENTRY32
	entry.Size = uint32(unsafe.Sizeof(entry))

	ret, _, _ := procProcess32First.Call(handle, uintptr(unsafe.Pointer(&entry)))
	if ret == 0 {
		return 0, fmt.Errorf("Error retrieving process info.")
	}
	return int(entry.ParentProcessID), nil

}

func (self *ProcMem) Get(pid int) error {
	handle, err := syscall.OpenProcess(PROCESS_ALL_ACCESS, false, uint32(pid))

	defer syscall.CloseHandle(handle)

	if err != nil {
		return fmt.Errorf("OpenProcess fails with %v", err)
	}

	var mem PROCESS_MEMORY_COUNTERS_EX
	mem.CB = uint32(unsafe.Sizeof(mem))

	r1, _, e1 := procGetProcessMemoryInfo.Call(
		uintptr(handle),
		uintptr(unsafe.Pointer(&mem)),
		uintptr(mem.CB),
	)
	if r1 == 0 {
		if e1 != nil {
			return error(e1)
		} else {
			return syscall.EINVAL
		}
	}

	self.Resident = uint64(mem.WorkingSetSize)
	self.Size = uint64(mem.PrivateUsage)
	// Size contains only to the Private Bytes
	// Virtual Bytes are the Working Set plus paged Private Bytes and standby list.
	return nil
}

func (self *ProcTime) Get(pid int) error {
	handle, err := syscall.OpenProcess(syscall.PROCESS_QUERY_INFORMATION, false, uint32(pid))

	defer syscall.CloseHandle(handle)

	if err != nil {
		return fmt.Errorf("OpenProcess fails with %v", err)

	}
	var CPU syscall.Rusage
	if err := syscall.GetProcessTimes(handle, &CPU.CreationTime, &CPU.ExitTime, &CPU.KernelTime, &CPU.UserTime); err != nil {
		return fmt.Errorf("GetProcessTimes fails with %v", err)
	}

	// convert to millis
	self.StartTime = uint64(FiletimeToDuration(&CPU.CreationTime).Nanoseconds() / 1e6)

	self.User = uint64(FiletimeToDuration(&CPU.UserTime).Nanoseconds() / 1e6)

	self.Sys = uint64(FiletimeToDuration(&CPU.KernelTime).Nanoseconds() / 1e6)

	self.Total = self.User + self.Sys

	return nil
}

func (self *ProcArgs) Get(pid int) error {
	return notImplemented()
}

func (self *ProcExe) Get(pid int) error {
	return notImplemented()
}

func (self *FileSystemUsage) Get(path string) error {

	/*
		Get free, available, total free bytes:
		fsutil volume diskfree C:
	*/
	var availableBytes C.ULARGE_INTEGER
	var totalBytes C.ULARGE_INTEGER
	var totalFreeBytes C.ULARGE_INTEGER

	pathChars := C.CString(path)
	defer C.free(unsafe.Pointer(pathChars))

	succeeded := C.GetDiskFreeSpaceEx((*C.CHAR)(pathChars), &availableBytes, &totalBytes, &totalFreeBytes)
	if succeeded == C.FALSE {
		return syscall.GetLastError()
	}

	self.Total = *(*uint64)(unsafe.Pointer(&totalBytes))
	self.Free = *(*uint64)(unsafe.Pointer(&totalFreeBytes))
	self.Used = self.Total - self.Free
	self.Avail = *(*uint64)(unsafe.Pointer(&availableBytes))

	return nil
}

func notImplemented() error {
	panic("Not Implemented")
	return nil
}
<|endoftext|>


<commit_before>package driver

import (
	"fmt"
	"io"
	"regexp"
	"strconv"
	"strings"

	"github.com/docker/distribution/context"
)

// Version is a string representing the storage driver version, of the form
// Major.Minor.
// The registry must accept storage drivers with equal major version and greater
// minor version, but may not be compatible with older storage driver versions.
type Version string

// Major returns the major (primary) component of a version.
func (version Version) Major() uint {
	majorPart := strings.Split(string(version), ".")[0]
	major, _ := strconv.ParseUint(majorPart, 10, 0)
	return uint(major)
}

// Minor returns the minor (secondary) component of a version.
func (version Version) Minor() uint {
	minorPart := strings.Split(string(version), ".")[1]
	minor, _ := strconv.ParseUint(minorPart, 10, 0)
	return uint(minor)
}

// CurrentVersion is the current storage driver Version.
const CurrentVersion Version = "0.1"

// StorageDriver defines methods that a Storage Driver must implement for a
// filesystem-like key/value object storage. Storage Drivers are automatically
// registered via an internal registration mechanism, and generally created
// via the StorageDriverFactory interface (https://godoc.org/github.com/docker/distribution/registry/storage/driver/factory).
// See below for an example of how to get a StorageDriver for S3:
//
//	import _ "github.com/docker/distribution/registry/storage/driver/s3-aws"
//	s3Driver, err = factory.Create("s3", storageParams)
//	// assuming no error, s3Driver is the StorageDriver that communicates with S3 according to storageParams
type StorageDriver interface {
	// Name returns the human-readable "name" of the driver, useful in error
	// messages and logging. By convention, this will just be the registration
	// name, but drivers may provide other information here.
	Name() string

	// GetContent retrieves the content stored at "path" as a []byte.
	// This should primarily be used for small objects.
	GetContent(ctx context.Context, path string) ([]byte, error)

	// PutContent stores the []byte content at a location designated by "path".
	// This should primarily be used for small objects.
	PutContent(ctx context.Context, path string, content []byte) error

	// Reader retrieves an io.ReadCloser for the content stored at "path"
	// with a given byte offset.
	// May be used to resume reading a stream by providing a nonzero offset.
	Reader(ctx context.Context, path string, offset int64) (io.ReadCloser, error)

	// Writer returns a FileWriter which will store the content written to it
	// at the location designated by "path" after the call to Commit.
	Writer(ctx context.Context, path string, append bool) (FileWriter, error)

	// Stat retrieves the FileInfo for the given path, including the current
	// size in bytes and the creation time.
	Stat(ctx context.Context, path string) (FileInfo, error)

	// List returns a list of the objects that are direct descendants of the
	//given path.
	List(ctx context.Context, path string) ([]string, error)

	// Move moves an object stored at sourcePath to destPath, removing the
	// original object.
	// Note: This may be no more efficient than a copy followed by a delete for
	// many implementations.
	Move(ctx context.Context, sourcePath string, destPath string) error

	// Delete recursively deletes all objects stored at "path" and its subpaths.
	Delete(ctx context.Context, path string) error

	// URLFor returns a URL which may be used to retrieve the content stored at
	// the given path, possibly using the given options.
	// May return an ErrUnsupportedMethod in certain StorageDriver
	// implementations.
	URLFor(ctx context.Context, path string, options map[string]interface{}) (string, error)
}

// FileWriter provides an abstraction for an opened writable file-like object in
// the storage backend. The FileWriter must flush all content written to it on
// the call to Close, but is only required to make its content readable on a
// call to Commit.
type FileWriter interface {
	io.WriteCloser

	// Size returns the number of bytes written to this FileWriter.
	Size() int64

	// Cancel removes any written content from this FileWriter.
	Cancel() error

	// Commit flushes all content written to this FileWriter and makes it
	// available for future calls to StorageDriver.GetContent and
	// StorageDriver.Reader.
	Commit() error
}

// PathRegexp is the regular expression which each file path must match. A
// file path is absolute, beginning with a slash and containing a positive
// number of path components separated by slashes, where each component is
// restricted to alphanumeric characters or a period, underscore, or
// hyphen.
var PathRegexp = regexp.MustCompile(`^(/[A-Za-z0-9._-]+)+$`)

// ErrUnsupportedMethod may be returned in the case where a StorageDriver implementation does not support an optional method.
type ErrUnsupportedMethod struct {
	DriverName string
}

func (err ErrUnsupportedMethod) Error() string {
	return fmt.Sprintf("%s: unsupported method", err.DriverName)
}

// PathNotFoundError is returned when operating on a nonexistent path.
type PathNotFoundError struct {
	Path       string
	DriverName string
}

func (err PathNotFoundError) Error() string {
	return fmt.Sprintf("%s: Path not found: %s", err.DriverName, err.Path)
}

// InvalidPathError is returned when the provided path is malformed.
type InvalidPathError struct {
	Path       string
	DriverName string
}

func (err InvalidPathError) Error() string {
	return fmt.Sprintf("%s: invalid path: %s", err.DriverName, err.Path)
}

// InvalidOffsetError is returned when attempting to read or write from an
// invalid offset.
type InvalidOffsetError struct {
	Path       string
	Offset     int64
	DriverName string
}

func (err InvalidOffsetError) Error() string {
	return fmt.Sprintf("%s: invalid offset: %d for path: %s", err.DriverName, err.Offset, err.Path)
}

// Error is a catch-all error type which captures an error string and
// the driver type on which it occurred.
type Error struct {
	DriverName string
	Enclosed   error
}

func (err Error) Error() string {
	return fmt.Sprintf("%s: %s", err.DriverName, err.Enclosed)
}
<commit_msg>Remove the example<commit_after>package driver

import (
	"fmt"
	"io"
	"regexp"
	"strconv"
	"strings"

	"github.com/docker/distribution/context"
)

// Version is a string representing the storage driver version, of the form
// Major.Minor.
// The registry must accept storage drivers with equal major version and greater
// minor version, but may not be compatible with older storage driver versions.
type Version string

// Major returns the major (primary) component of a version.
func (version Version) Major() uint {
	majorPart := strings.Split(string(version), ".")[0]
	major, _ := strconv.ParseUint(majorPart, 10, 0)
	return uint(major)
}

// Minor returns the minor (secondary) component of a version.
func (version Version) Minor() uint {
	minorPart := strings.Split(string(version), ".")[1]
	minor, _ := strconv.ParseUint(minorPart, 10, 0)
	return uint(minor)
}

// CurrentVersion is the current storage driver Version.
const CurrentVersion Version = "0.1"

// StorageDriver defines methods that a Storage Driver must implement for a
// filesystem-like key/value object storage. Storage Drivers are automatically
// registered via an internal registration mechanism, and generally created
// via the StorageDriverFactory interface (https://godoc.org/github.com/docker/distribution/registry/storage/driver/factory).
// Please see the aforementioned factory package for example code showing how to get an instance
// of a StorageDriver
type StorageDriver interface {
	// Name returns the human-readable "name" of the driver, useful in error
	// messages and logging. By convention, this will just be the registration
	// name, but drivers may provide other information here.
	Name() string

	// GetContent retrieves the content stored at "path" as a []byte.
	// This should primarily be used for small objects.
	GetContent(ctx context.Context, path string) ([]byte, error)

	// PutContent stores the []byte content at a location designated by "path".
	// This should primarily be used for small objects.
	PutContent(ctx context.Context, path string, content []byte) error

	// Reader retrieves an io.ReadCloser for the content stored at "path"
	// with a given byte offset.
	// May be used to resume reading a stream by providing a nonzero offset.
	Reader(ctx context.Context, path string, offset int64) (io.ReadCloser, error)

	// Writer returns a FileWriter which will store the content written to it
	// at the location designated by "path" after the call to Commit.
	Writer(ctx context.Context, path string, append bool) (FileWriter, error)

	// Stat retrieves the FileInfo for the given path, including the current
	// size in bytes and the creation time.
	Stat(ctx context.Context, path string) (FileInfo, error)

	// List returns a list of the objects that are direct descendants of the
	//given path.
	List(ctx context.Context, path string) ([]string, error)

	// Move moves an object stored at sourcePath to destPath, removing the
	// original object.
	// Note: This may be no more efficient than a copy followed by a delete for
	// many implementations.
	Move(ctx context.Context, sourcePath string, destPath string) error

	// Delete recursively deletes all objects stored at "path" and its subpaths.
	Delete(ctx context.Context, path string) error

	// URLFor returns a URL which may be used to retrieve the content stored at
	// the given path, possibly using the given options.
	// May return an ErrUnsupportedMethod in certain StorageDriver
	// implementations.
	URLFor(ctx context.Context, path string, options map[string]interface{}) (string, error)
}

// FileWriter provides an abstraction for an opened writable file-like object in
// the storage backend. The FileWriter must flush all content written to it on
// the call to Close, but is only required to make its content readable on a
// call to Commit.
type FileWriter interface {
	io.WriteCloser

	// Size returns the number of bytes written to this FileWriter.
	Size() int64

	// Cancel removes any written content from this FileWriter.
	Cancel() error

	// Commit flushes all content written to this FileWriter and makes it
	// available for future calls to StorageDriver.GetContent and
	// StorageDriver.Reader.
	Commit() error
}

// PathRegexp is the regular expression which each file path must match. A
// file path is absolute, beginning with a slash and containing a positive
// number of path components separated by slashes, where each component is
// restricted to alphanumeric characters or a period, underscore, or
// hyphen.
var PathRegexp = regexp.MustCompile(`^(/[A-Za-z0-9._-]+)+$`)

// ErrUnsupportedMethod may be returned in the case where a StorageDriver implementation does not support an optional method.
type ErrUnsupportedMethod struct {
	DriverName string
}

func (err ErrUnsupportedMethod) Error() string {
	return fmt.Sprintf("%s: unsupported method", err.DriverName)
}

// PathNotFoundError is returned when operating on a nonexistent path.
type PathNotFoundError struct {
	Path       string
	DriverName string
}

func (err PathNotFoundError) Error() string {
	return fmt.Sprintf("%s: Path not found: %s", err.DriverName, err.Path)
}

// InvalidPathError is returned when the provided path is malformed.
type InvalidPathError struct {
	Path       string
	DriverName string
}

func (err InvalidPathError) Error() string {
	return fmt.Sprintf("%s: invalid path: %s", err.DriverName, err.Path)
}

// InvalidOffsetError is returned when attempting to read or write from an
// invalid offset.
type InvalidOffsetError struct {
	Path       string
	Offset     int64
	DriverName string
}

func (err InvalidOffsetError) Error() string {
	return fmt.Sprintf("%s: invalid offset: %d for path: %s", err.DriverName, err.Offset, err.Path)
}

// Error is a catch-all error type which captures an error string and
// the driver type on which it occurred.
type Error struct {
	DriverName string
	Enclosed   error
}

func (err Error) Error() string {
	return fmt.Sprintf("%s: %s", err.DriverName, err.Enclosed)
}
<|endoftext|>