<commit_before>package commands

import (
	"os"
	"path/filepath"
	"strings"

	"github.com/github/git-lfs/api"
	"github.com/github/git-lfs/config"
	"github.com/github/git-lfs/git"
	"github.com/spf13/cobra"
)

var (
	lockRemote     string
	lockRemoteHelp = "specify which remote to use when interacting with locks"

	// TODO(taylor): consider making this (and the above flag) a property of
	// some parent-command, or another similarly less ugly way of handling
	// this
	setLockRemoteFor = func(c *config.Configuration) {
		c.CurrentRemote = lockRemote
	}

	lockCmd = &cobra.Command{
		Use: "lock",
		Run: lockCommand,
	}
)

func lockCommand(cmd *cobra.Command, args []string) {
	setLockRemoteFor(config.Config)

	if len(args) == 0 {
		Print("Usage: git lfs lock <path>")
		return
	}

	latest, err := git.CurrentRemoteRef()
	if err != nil {
		Error(err.Error())
		Exit("Unable to determine lastest remote ref for branch.")
	}

	path, err := lockPath(args[0])
	if err != nil {
		Error(err.Error())
	}

	s, resp := API.Locks.Lock(&api.LockRequest{
		Path:               path,
		Committer:          api.CurrentCommitter(),
		LatestRemoteCommit: latest.Sha,
	})

	if _, err := API.Do(s); err != nil {
		Error(err.Error())
		Exit("Error communicating with LFS API.")
	}

	if len(resp.Err) > 0 {
		Error(resp.Err)
		Exit("Server unable to create lock.")
	}

	Print("\n'%s' was locked (%s)", args[0], resp.Lock.Id)
}

func lockPath(file string) (string, error) {
	repo, err := git.RootDir()
	if err != nil {
		return "", err
	}

	wd, err := os.Getwd()
	if err != nil {
		return "", err
	}

	abs := filepath.Join(wd, file)

	return strings.TrimPrefix(repo, abs), nil
}

func init() {
	lockCmd.Flags().StringVarP(&lockRemote, "remote", "r", config.Config.CurrentRemote, lockRemoteHelp)

	RootCmd.AddCommand(lockCmd)
}
<commit_msg>commands/lock: properly trim prefix from lock path<commit_after>package commands

import (
	"os"
	"path/filepath"
	"strings"

	"github.com/github/git-lfs/api"
	"github.com/github/git-lfs/config"
	"github.com/github/git-lfs/git"
	"github.com/spf13/cobra"
)

var (
	lockRemote     string
	lockRemoteHelp = "specify which remote to use when interacting with locks"

	// TODO(taylor): consider making this (and the above flag) a property of
	// some parent-command, or another similarly less ugly way of handling
	// this
	setLockRemoteFor = func(c *config.Configuration) {
		c.CurrentRemote = lockRemote
	}

	lockCmd = &cobra.Command{
		Use: "lock",
		Run: lockCommand,
	}
)

func lockCommand(cmd *cobra.Command, args []string) {
	setLockRemoteFor(config.Config)

	if len(args) == 0 {
		Print("Usage: git lfs lock <path>")
		return
	}

	latest, err := git.CurrentRemoteRef()
	if err != nil {
		Error(err.Error())
		Exit("Unable to determine lastest remote ref for branch.")
	}

	path, err := lockPath(args[0])
	if err != nil {
		Error(err.Error())
	}

	s, resp := API.Locks.Lock(&api.LockRequest{
		Path:               path,
		Committer:          api.CurrentCommitter(),
		LatestRemoteCommit: latest.Sha,
	})

	if _, err := API.Do(s); err != nil {
		Error(err.Error())
		Exit("Error communicating with LFS API.")
	}

	if len(resp.Err) > 0 {
		Error(resp.Err)
		Exit("Server unable to create lock.")
	}

	Print("\n'%s' was locked (%s)", args[0], resp.Lock.Id)
}

func lockPath(file string) (string, error) {
	repo, err := git.RootDir()
	if err != nil {
		return "", err
	}

	wd, err := os.Getwd()
	if err != nil {
		return "", err
	}

	abs := filepath.Join(wd, file)

	return strings.TrimPrefix(abs, repo), nil
}

func init() {
	lockCmd.Flags().StringVarP(&lockRemote, "remote", "r", config.Config.CurrentRemote, lockRemoteHelp)

	RootCmd.AddCommand(lockCmd)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"

	"github.com/conformal/btcec"
	"github.com/conformal/btcnet"
	"github.com/conformal/btcutil"
)

type rData struct {
	Sig       *btcec.Signature
	H         int64
	SigScript []byte
}

func popData(SignatureScript []byte) ([]byte, []byte, error) {
	if len(SignatureScript) < 1 {
		return nil, nil, fmt.Errorf("empty SignatureScript")
	}
	opcode := SignatureScript[0]

	if opcode >= 1 && opcode <= 75 {
		if len(SignatureScript) < int(opcode+1) {
			return nil, nil, fmt.Errorf("SignatureScript too short")
		}
		sigStr := SignatureScript[1 : opcode+1]
		remaining := SignatureScript[opcode+1:]
		return sigStr, remaining, nil
	}

	// TODO: OP_PUSHDATA1 OP_PUSHDATA2 OP_PUSHDATA3
	if opcode >= 76 && opcode <= 78 {
		return nil, nil, fmt.Errorf("FIXME: OP_PUSHDATA %v", opcode)
	}

	return nil, nil, fmt.Errorf("the first opcode (%x) is not a data push", opcode)
}

func main() {
	var jsonFile = flag.String("json", "blockchainr.json", "blockchainr output")
	flag.Parse()

	blockchainrFile, err := ioutil.ReadFile(*jsonFile)
	if err != nil {
		log.Fatalln("failed to read blockchainr.json:", err)
	}

	results := make(map[string][]*rData)
	err = json.Unmarshal(blockchainrFile, &results)
	if err != nil {
		log.Fatalln("Unmarshal error:", err)
	}

	balanceCache := make(map[string][]byte)

	for r, result := range results {
		log.Println(r)
		for _, rd := range result {
			_, remaining, err := popData(rd.SigScript)
			if err != nil {
				log.Fatalln("PopData failed:", err)
			}
			pkStr, _, err := popData(remaining)
			if err != nil {
				log.Printf("The second PopData failed - probably a pay-to-PubKey:", err)
				// FIX: use recoverKeyFromSignature?
				continue
			}

			aPubKey, err := btcutil.NewAddressPubKey(pkStr, &btcnet.MainNetParams)
			if err != nil {
				log.Printf("Pubkey parse error:", err)
				continue
			}
			address := aPubKey.EncodeAddress()

			balance, ok := balanceCache[address]
			if !ok {
				response, err := http.Get("https://blockchain.info/q/addressbalance/" + address)
				if err != nil {
					log.Fatalln("Get failed:", err)
				}
				defer response.Body.Close()
				balance, err = ioutil.ReadAll(response.Body)
				if err != nil {
					log.Fatalln("ReadAll failed:", err)
				}
				balanceCache[address] = balance
			}

			log.Println(address, string(balance))
		}
	}
}
<commit_msg>Skip false positives in analyzr<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"

	"github.com/conformal/btcec"
	"github.com/conformal/btcnet"
	"github.com/conformal/btcutil"
)

type rData struct {
	Sig       *btcec.Signature
	H         int64
	SigScript []byte
}

func popData(SignatureScript []byte) ([]byte, []byte, error) {
	if len(SignatureScript) < 1 {
		return nil, nil, fmt.Errorf("empty SignatureScript")
	}
	opcode := SignatureScript[0]

	if opcode >= 1 && opcode <= 75 {
		if len(SignatureScript) < int(opcode+1) {
			return nil, nil, fmt.Errorf("SignatureScript too short")
		}
		sigStr := SignatureScript[1 : opcode+1]
		remaining := SignatureScript[opcode+1:]
		return sigStr, remaining, nil
	}

	// TODO: OP_PUSHDATA1 OP_PUSHDATA2 OP_PUSHDATA3
	if opcode >= 76 && opcode <= 78 {
		return nil, nil, fmt.Errorf("FIXME: OP_PUSHDATA %v", opcode)
	}

	return nil, nil, fmt.Errorf("the first opcode (%x) is not a data push", opcode)
}

func main() {
	var jsonFile = flag.String("json", "blockchainr.json", "blockchainr output")
	flag.Parse()

	blockchainrFile, err := ioutil.ReadFile(*jsonFile)
	if err != nil {
		log.Fatalln("failed to read blockchainr.json:", err)
	}

	results := make(map[string][]*rData)
	err = json.Unmarshal(blockchainrFile, &results)
	if err != nil {
		log.Fatalln("Unmarshal error:", err)
	}

	balanceCache := make(map[string][]byte)

	for r, result := range results {
		if len(result) < 2 {
			continue
		}

		log.Println(r)

		for _, rd := range result {
			_, remaining, err := popData(rd.SigScript)
			if err != nil {
				log.Fatalln("PopData failed:", err)
			}
			pkStr, _, err := popData(remaining)
			if err != nil {
				log.Printf("The second PopData failed - probably a pay-to-PubKey:", err)
				// FIX: use recoverKeyFromSignature?
				continue
			}

			aPubKey, err := btcutil.NewAddressPubKey(pkStr, &btcnet.MainNetParams)
			if err != nil {
				log.Printf("Pubkey parse error:", err)
				continue
			}
			address := aPubKey.EncodeAddress()

			balance, ok := balanceCache[address]
			if !ok {
				response, err := http.Get("https://blockchain.info/q/addressbalance/" + address)
				if err != nil {
					log.Fatalln("Get failed:", err)
				}
				defer response.Body.Close()
				balance, err = ioutil.ReadAll(response.Body)
				if err != nil {
					log.Fatalln("ReadAll failed:", err)
				}
				balanceCache[address] = balance
			}

			log.Println(address, string(balance))
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	log "github.com/GameGophers/libs/nsq-logger"
	"golang.org/x/net/context"
	"sync"
)

import (
	. "proto"
	"pubsub"
)

const (
	SERVICE = "[CHAT]"
)

const (
	BOLTDB_FILE    = "/data/CHAT.DAT"
	BOLTDB_BUCKET  = "EPS"
	MAX_QUEUE_SIZE = 128 // num of message kept
)

var (
	OK                   = &Chat_Nil{}
	ERROR_ALREADY_EXISTS = errors.New("id already exists")
	ERROR_NOT_EXISTS     = errors.New("id not exists")
)

type EndPoint struct {
	Inbox []Chat_Message
	ps    *pubsub.PubSub
	sync.Mutex
}

func (ep *EndPoint) Push(msg *Chat_Message) {
	ep.Lock()
	defer ep.Unlock()
	if len(ep.Inbox) > MAX_QUEUE_SIZE {
		ep.Inbox = append(ep.Inbox[1:], *msg)
	} else {
		ep.Inbox = append(ep.Inbox, *msg)
	}
}

func (ep *EndPoint) Read() []Chat_Message {
	ep.Lock()
	defer ep.Unlock()
	return append([]Chat_Message(nil), ep.Inbox...)
}

func NewEndPoint() *EndPoint {
	u := &EndPoint{}
	u.ps = pubsub.New()
	return u
}

type server struct {
	eps map[uint64]*EndPoint
	sync.RWMutex
}

func (s *server) read_ep(id uint64) *EndPoint {
	s.RLock()
	defer s.RUnlock()
	return s.eps[id]
}

func (s *server) init() {
	s.eps = make(map[uint64]*EndPoint)
}

func (s *server) Subscribe(p *Chat_Id, stream ChatService_SubscribeServer) error {
	die := make(chan bool)
	f := func(msg *Chat_Message) {
		if err := stream.Send(msg); err != nil {
			close(die)
		}
	}

	ep := s.read_ep(p.Id)
	if ep == nil {
		log.Errorf("cannot find endpoint %v", p)
		return ERROR_NOT_EXISTS
	}

	ep.ps.Sub(f)
	defer func() {
		ep.ps.Leave(f)
	}()

	<-die
	return nil
}

func (s *server) Read(p *Chat_Id, stream ChatService_ReadServer) error {
	ep := s.read_ep(p.Id)
	if ep == nil {
		log.Errorf("cannot find endpoint %v", p)
		return ERROR_NOT_EXISTS
	}

	msgs := ep.Read()
	for k := range msgs {
		if err := stream.Send(&msgs[k]); err != nil {
			return err
		}
	}
	return nil
}

func (s *server) Send(ctx context.Context, msg *Chat_Message) (*Chat_Nil, error) {
	ep := s.read_ep(msg.Dst)
	if ep == nil {
		return nil, ERROR_NOT_EXISTS
	}

	ep.ps.Pub(msg)
	ep.Push(msg)
	return OK, nil
}

func (s *server) Reg(ctx context.Context, p *Chat_Id) (*Chat_Nil, error) {
	s.Lock()
	defer s.Unlock()
	ep := s.eps[p.Id]
	if ep != nil {
		log.Errorf("id already exists:%v", p.Id)
		return nil, ERROR_ALREADY_EXISTS
	}

	s.eps[p.Id] = NewEndPoint()
	return OK, nil
}
<commit_msg>update<commit_after>package main

import (
	"errors"
	log "github.com/GameGophers/libs/nsq-logger"
	"golang.org/x/net/context"
	"sync"
)

import (
	. "proto"
	"pubsub"
)

const (
	SERVICE = "[CHAT]"
)

const (
	BOLTDB_FILE    = "/data/CHAT.DAT"
	BOLTDB_BUCKET  = "EPS"
	MAX_QUEUE_SIZE = 128 // num of message kept
)

var (
	OK                   = &Chat_Nil{}
	ERROR_ALREADY_EXISTS = errors.New("id already exists")
	ERROR_NOT_EXISTS     = errors.New("id not exists")
)

type EndPoint struct {
	inbox []Chat_Message
	ps    *pubsub.PubSub
	sync.Mutex
}

func (ep *EndPoint) Push(msg *Chat_Message) {
	ep.Lock()
	defer ep.Unlock()
	if len(ep.inbox) > MAX_QUEUE_SIZE {
		ep.inbox = append(ep.inbox[1:], *msg)
	} else {
		ep.inbox = append(ep.inbox, *msg)
	}
}

func (ep *EndPoint) Read() []Chat_Message {
	ep.Lock()
	defer ep.Unlock()
	return append([]Chat_Message(nil), ep.inbox...)
}

func NewEndPoint() *EndPoint {
	u := &EndPoint{}
	u.ps = pubsub.New()
	return u
}

type server struct {
	eps map[uint64]*EndPoint
	sync.RWMutex
}

func (s *server) read_ep(id uint64) *EndPoint {
	s.RLock()
	defer s.RUnlock()
	return s.eps[id]
}

func (s *server) init() {
	s.eps = make(map[uint64]*EndPoint)
}

func (s *server) Subscribe(p *Chat_Id, stream ChatService_SubscribeServer) error {
	die := make(chan bool)
	f := func(msg *Chat_Message) {
		if err := stream.Send(msg); err != nil {
			close(die)
		}
	}

	ep := s.read_ep(p.Id)
	if ep == nil {
		log.Errorf("cannot find endpoint %v", p)
		return ERROR_NOT_EXISTS
	}

	ep.ps.Sub(f)
	defer func() {
		ep.ps.Leave(f)
	}()

	<-die
	return nil
}

func (s *server) Read(p *Chat_Id, stream ChatService_ReadServer) error {
	ep := s.read_ep(p.Id)
	if ep == nil {
		log.Errorf("cannot find endpoint %v", p)
		return ERROR_NOT_EXISTS
	}

	msgs := ep.Read()
	for k := range msgs {
		if err := stream.Send(&msgs[k]); err != nil {
			return err
		}
	}
	return nil
}

func (s *server) Send(ctx context.Context, msg *Chat_Message) (*Chat_Nil, error) {
	ep := s.read_ep(msg.Dst)
	if ep == nil {
		return nil, ERROR_NOT_EXISTS
	}

	ep.ps.Pub(msg)
	ep.Push(msg)
	return OK, nil
}

func (s *server) Reg(ctx context.Context, p *Chat_Id) (*Chat_Nil, error) {
	s.Lock()
	defer s.Unlock()
	ep := s.eps[p.Id]
	if ep != nil {
		log.Errorf("id already exists:%v", p.Id)
		return nil, ERROR_ALREADY_EXISTS
	}

	s.eps[p.Id] = NewEndPoint()
	return OK, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Doc (usually run as go doc) accepts zero, one or two arguments.
//
// Zero arguments:
//	go doc
// Show the documentation for the package in the current directory.
//
// One argument:
//	go doc <pkg>
//	go doc <sym>[.<method>]
//	go doc [<pkg>].<sym>[.<method>]
// The first item in this list that succeeds is the one whose documentation
// is printed. If there is a symbol but no package, the package in the current
// directory is chosen.
//
// Two arguments:
//	go doc <pkg> <sym>[.<method>]
//
// Show the documentation for the package, symbol, and method. The
// first argument must be a full package path. This is similar to the
// command-line usage for the godoc command.
//
// For complete documentation, run "go help doc".
package main

import (
	"flag"
	"fmt"
	"go/build"
	"log"
	"os"
	"path"
	"path/filepath"
	"strings"
	"unicode"
	"unicode/utf8"
)

var (
	unexported = flag.Bool("u", false, "show unexported symbols as well as exported")
	matchCase  = flag.Bool("c", false, "symbol matching honors case (paths not affected)")
)

// usage is a replacement usage function for the flags package.
func usage() {
	fmt.Fprintf(os.Stderr, "Usage of [go] doc:\n")
	fmt.Fprintf(os.Stderr, "\tgo doc\n")
	fmt.Fprintf(os.Stderr, "\tgo doc <pkg>\n")
	fmt.Fprintf(os.Stderr, "\tgo doc <sym>[.<method>]\n")
	fmt.Fprintf(os.Stderr, "\tgo doc [<pkg>].<sym>[.<method>]\n")
	fmt.Fprintf(os.Stderr, "\tgo doc <pkg> <sym>[.<method>]\n")
	fmt.Fprintf(os.Stderr, "For more information run\n")
	fmt.Fprintf(os.Stderr, "\tgo help doc\n\n")
	fmt.Fprintf(os.Stderr, "Flags:\n")
	flag.PrintDefaults()
	os.Exit(2)
}

func main() {
	log.SetFlags(0)
	log.SetPrefix("doc: ")
	flag.Usage = usage
	flag.Parse()
	buildPackage, userPath, symbol := parseArgs()
	symbol, method := parseSymbol(symbol)
	pkg := parsePackage(buildPackage, userPath)
	switch {
	case symbol == "":
		pkg.packageDoc()
		return
	case method == "":
		pkg.symbolDoc(symbol)
	default:
		pkg.methodDoc(symbol, method)
	}
}

// parseArgs analyzes the arguments (if any) and returns the package
// it represents, the part of the argument the user used to identify
// the path (or "" if it's the current package) and the symbol
// (possibly with a .method) within that package.
// parseSymbol is used to analyze the symbol itself.
func parseArgs() (*build.Package, string, string) {
	switch flag.NArg() {
	default:
		usage()
	case 0:
		// Easy: current directory.
		return importDir("."), "", ""
	case 1:
		// Done below.
	case 2:
		// Package must be importable.
		pkg, err := build.Import(flag.Arg(0), "", build.ImportComment)
		if err != nil {
			log.Fatal(err)
		}
		return pkg, flag.Arg(0), flag.Arg(1)
	}
	// Usual case: one argument.
	arg := flag.Arg(0)
	// If it contains slashes, it begins with a package path.
	// First, is it a complete package path as it is? If so, we are done.
	// This avoids confusion over package paths that have other
	// package paths as their prefix.
	pkg, err := build.Import(arg, "", build.ImportComment)
	if err == nil {
		return pkg, arg, ""
	}
	// Another disambiguator: If the symbol starts with an upper
	// case letter, it can only be a symbol in the current directory.
	// Kills the problem caused by case-insensitive file systems
	// matching an upper case name as a package name.
	if isUpper(arg) {
		pkg, err := build.ImportDir(".", build.ImportComment)
		if err == nil {
			return pkg, "", arg
		}
	}
	// If it has a slash, it must be a package path but there is a symbol.
	// It's the last package path we care about.
	slash := strings.LastIndex(arg, "/")
	// There may be periods in the package path before or after the slash
	// and between a symbol and method.
	// Split the string at various periods to see what we find.
	// In general there may be ambiguities but this should almost always
	// work.
	var period int
	// slash+1: if there's no slash, the value is -1 and start is 0; otherwise
	// start is the byte after the slash.
	for start := slash + 1; start < len(arg); start = period + 1 {
		period = start + strings.Index(arg[start:], ".")
		symbol := ""
		if period < 0 {
			period = len(arg)
		} else {
			symbol = arg[period+1:]
		}
		// Have we identified a package already?
		pkg, err := build.Import(arg[0:period], "", build.ImportComment)
		if err == nil {
			return pkg, arg[0:period], symbol
		}
		// See if we have the basename or tail of a package, as in json for encoding/json
		// or ivy/value for robpike.io/ivy/value.
		path := findPackage(arg[0:period])
		if path != "" {
			return importDir(path), arg[0:period], symbol
		}
	}
	// If it has a slash, we've failed.
	if slash >= 0 {
		log.Fatalf("no such package %s", arg[0:period])
	}
	// Guess it's a symbol in the current directory.
	return importDir("."), "", arg
}

// importDir is just an error-catching wrapper for build.ImportDir.
func importDir(dir string) *build.Package {
	pkg, err := build.ImportDir(dir, build.ImportComment)
	if err != nil {
		log.Fatal(err)
	}
	return pkg
}

// parseSymbol breaks str apart into a symbol and method.
// Both may be missing or the method may be missing.
// If present, each must be a valid Go identifier.
func parseSymbol(str string) (symbol, method string) {
	if str == "" {
		return
	}
	elem := strings.Split(str, ".")
	switch len(elem) {
	case 1:
	case 2:
		method = elem[1]
		isIdentifier(method)
	default:
		log.Printf("too many periods in symbol specification")
		usage()
	}
	symbol = elem[0]
	isIdentifier(symbol)
	return
}

// isIdentifier checks that the name is valid Go identifier, and
// logs and exits if it is not.
func isIdentifier(name string) {
	if len(name) == 0 {
		log.Fatal("empty symbol")
	}
	for i, ch := range name {
		if unicode.IsLetter(ch) || ch == '_' || i > 0 && unicode.IsDigit(ch) {
			continue
		}
		log.Fatalf("invalid identifier %q", name)
	}
}

// isExported reports whether the name is an exported identifier.
// If the unexported flag (-u) is true, isExported returns true because
// it means that we treat the name as if it is exported.
func isExported(name string) bool {
	return *unexported || isUpper(name)
}

// isUpper reports whether the name starts with an upper case letter.
func isUpper(name string) bool {
	ch, _ := utf8.DecodeRuneInString(name)
	return unicode.IsUpper(ch)
}

// findPackage returns the full file name path specified by the
// (perhaps partial) package path pkg.
func findPackage(pkg string) string {
	if pkg == "" {
		return ""
	}
	if isUpper(pkg) {
		return "" // Upper case symbol cannot be a package name.
	}
	path := pathFor(build.Default.GOROOT, pkg)
	if path != "" {
		return path
	}
	for _, root := range splitGopath() {
		path = pathFor(root, pkg)
		if path != "" {
			return path
		}
	}
	return ""
}

// splitGopath splits $GOPATH into a list of roots.
func splitGopath() []string {
	return filepath.SplitList(build.Default.GOPATH)
}

// pathsFor recursively walks the tree at root looking for possible directories for the package:
// those whose package path is pkg or which have a proper suffix pkg.
func pathFor(root, pkg string) (result string) {
	root = path.Join(root, "src")
	slashDot := string(filepath.Separator) + "."
	// We put a slash on the pkg so can use simple string comparison below
	// yet avoid inadvertent matches, like /foobar matching bar.
	pkgString := filepath.Clean(string(filepath.Separator) + pkg)

	// We use panic/defer to short-circuit processing at the first match.
	// A nil panic reports that the path has been found.
	defer func() {
		err := recover()
		if err != nil {
			panic(err)
		}
	}()

	visit := func(pathName string, f os.FileInfo, err error) error {
		if err != nil {
			return nil
		}
		// One package per directory. Ignore the files themselves.
		if !f.IsDir() {
			return nil
		}
		// No .git or other dot nonsense please.
		if strings.Contains(pathName, slashDot) {
			return filepath.SkipDir
		}
		// Is the tail of the path correct?
		if strings.HasSuffix(pathName, pkgString) {
			result = pathName
			panic(nil)
		}
		return nil
	}

	filepath.Walk(root, visit)
	return "" // Call to panic above sets the real value.
}
<commit_msg>cmd/doc: show the true import path rather than "."<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Doc (usually run as go doc) accepts zero, one or two arguments.
//
// Zero arguments:
//	go doc
// Show the documentation for the package in the current directory.
//
// One argument:
//	go doc <pkg>
//	go doc <sym>[.<method>]
//	go doc [<pkg>].<sym>[.<method>]
// The first item in this list that succeeds is the one whose documentation
// is printed. If there is a symbol but no package, the package in the current
// directory is chosen.
//
// Two arguments:
//	go doc <pkg> <sym>[.<method>]
//
// Show the documentation for the package, symbol, and method. The
// first argument must be a full package path. This is similar to the
// command-line usage for the godoc command.
//
// For complete documentation, run "go help doc".
package main

import (
	"flag"
	"fmt"
	"go/build"
	"log"
	"os"
	"path"
	"path/filepath"
	"strings"
	"unicode"
	"unicode/utf8"
)

var (
	unexported = flag.Bool("u", false, "show unexported symbols as well as exported")
	matchCase  = flag.Bool("c", false, "symbol matching honors case (paths not affected)")
)

// usage is a replacement usage function for the flags package.
func usage() {
	fmt.Fprintf(os.Stderr, "Usage of [go] doc:\n")
	fmt.Fprintf(os.Stderr, "\tgo doc\n")
	fmt.Fprintf(os.Stderr, "\tgo doc <pkg>\n")
	fmt.Fprintf(os.Stderr, "\tgo doc <sym>[.<method>]\n")
	fmt.Fprintf(os.Stderr, "\tgo doc [<pkg>].<sym>[.<method>]\n")
	fmt.Fprintf(os.Stderr, "\tgo doc <pkg> <sym>[.<method>]\n")
	fmt.Fprintf(os.Stderr, "For more information run\n")
	fmt.Fprintf(os.Stderr, "\tgo help doc\n\n")
	fmt.Fprintf(os.Stderr, "Flags:\n")
	flag.PrintDefaults()
	os.Exit(2)
}

func main() {
	log.SetFlags(0)
	log.SetPrefix("doc: ")
	flag.Usage = usage
	flag.Parse()
	buildPackage, userPath, symbol := parseArgs()
	symbol, method := parseSymbol(symbol)
	pkg := parsePackage(buildPackage, userPath)
	switch {
	case symbol == "":
		pkg.packageDoc()
		return
	case method == "":
		pkg.symbolDoc(symbol)
	default:
		pkg.methodDoc(symbol, method)
	}
}

// parseArgs analyzes the arguments (if any) and returns the package
// it represents, the part of the argument the user used to identify
// the path (or "" if it's the current package) and the symbol
// (possibly with a .method) within that package.
// parseSymbol is used to analyze the symbol itself.
func parseArgs() (*build.Package, string, string) {
	switch flag.NArg() {
	default:
		usage()
	case 0:
		// Easy: current directory.
		return importDir(pwd()), "", ""
	case 1:
		// Done below.
	case 2:
		// Package must be importable.
		pkg, err := build.Import(flag.Arg(0), "", build.ImportComment)
		if err != nil {
			log.Fatal(err)
		}
		return pkg, flag.Arg(0), flag.Arg(1)
	}
	// Usual case: one argument.
	arg := flag.Arg(0)
	// If it contains slashes, it begins with a package path.
	// First, is it a complete package path as it is? If so, we are done.
	// This avoids confusion over package paths that have other
	// package paths as their prefix.
	pkg, err := build.Import(arg, "", build.ImportComment)
	if err == nil {
		return pkg, arg, ""
	}
	// Another disambiguator: If the symbol starts with an upper
	// case letter, it can only be a symbol in the current directory.
	// Kills the problem caused by case-insensitive file systems
	// matching an upper case name as a package name.
	if isUpper(arg) {
		pkg, err := build.ImportDir(".", build.ImportComment)
		if err == nil {
			return pkg, "", arg
		}
	}
	// If it has a slash, it must be a package path but there is a symbol.
	// It's the last package path we care about.
	slash := strings.LastIndex(arg, "/")
	// There may be periods in the package path before or after the slash
	// and between a symbol and method.
	// Split the string at various periods to see what we find.
	// In general there may be ambiguities but this should almost always
	// work.
	var period int
	// slash+1: if there's no slash, the value is -1 and start is 0; otherwise
	// start is the byte after the slash.
	for start := slash + 1; start < len(arg); start = period + 1 {
		period = start + strings.Index(arg[start:], ".")
		symbol := ""
		if period < 0 {
			period = len(arg)
		} else {
			symbol = arg[period+1:]
		}
		// Have we identified a package already?
		pkg, err := build.Import(arg[0:period], "", build.ImportComment)
		if err == nil {
			return pkg, arg[0:period], symbol
		}
		// See if we have the basename or tail of a package, as in json for encoding/json
		// or ivy/value for robpike.io/ivy/value.
		path := findPackage(arg[0:period])
		if path != "" {
			return importDir(path), arg[0:period], symbol
		}
	}
	// If it has a slash, we've failed.
	if slash >= 0 {
		log.Fatalf("no such package %s", arg[0:period])
	}
	// Guess it's a symbol in the current directory.
	return importDir(pwd()), "", arg
}

// importDir is just an error-catching wrapper for build.ImportDir.
func importDir(dir string) *build.Package {
	pkg, err := build.ImportDir(dir, build.ImportComment)
	if err != nil {
		log.Fatal(err)
	}
	return pkg
}

// parseSymbol breaks str apart into a symbol and method.
// Both may be missing or the method may be missing.
// If present, each must be a valid Go identifier.
func parseSymbol(str string) (symbol, method string) {
	if str == "" {
		return
	}
	elem := strings.Split(str, ".")
	switch len(elem) {
	case 1:
	case 2:
		method = elem[1]
		isIdentifier(method)
	default:
		log.Printf("too many periods in symbol specification")
		usage()
	}
	symbol = elem[0]
	isIdentifier(symbol)
	return
}

// isIdentifier checks that the name is valid Go identifier, and
// logs and exits if it is not.
func isIdentifier(name string) {
	if len(name) == 0 {
		log.Fatal("empty symbol")
	}
	for i, ch := range name {
		if unicode.IsLetter(ch) || ch == '_' || i > 0 && unicode.IsDigit(ch) {
			continue
		}
		log.Fatalf("invalid identifier %q", name)
	}
}

// isExported reports whether the name is an exported identifier.
// If the unexported flag (-u) is true, isExported returns true because
// it means that we treat the name as if it is exported.
func isExported(name string) bool {
	return *unexported || isUpper(name)
}

// isUpper reports whether the name starts with an upper case letter.
func isUpper(name string) bool {
	ch, _ := utf8.DecodeRuneInString(name)
	return unicode.IsUpper(ch)
}

// findPackage returns the full file name path specified by the
// (perhaps partial) package path pkg.
func findPackage(pkg string) string {
	if pkg == "" {
		return ""
	}
	if isUpper(pkg) {
		return "" // Upper case symbol cannot be a package name.
	}
	path := pathFor(build.Default.GOROOT, pkg)
	if path != "" {
		return path
	}
	for _, root := range splitGopath() {
		path = pathFor(root, pkg)
		if path != "" {
			return path
		}
	}
	return ""
}

// splitGopath splits $GOPATH into a list of roots.
func splitGopath() []string {
	return filepath.SplitList(build.Default.GOPATH)
}

// pathsFor recursively walks the tree at root looking for possible directories for the package:
// those whose package path is pkg or which have a proper suffix pkg.
func pathFor(root, pkg string) (result string) {
	root = path.Join(root, "src")
	slashDot := string(filepath.Separator) + "."
	// We put a slash on the pkg so can use simple string comparison below
	// yet avoid inadvertent matches, like /foobar matching bar.
	pkgString := filepath.Clean(string(filepath.Separator) + pkg)

	// We use panic/defer to short-circuit processing at the first match.
	// A nil panic reports that the path has been found.
	defer func() {
		err := recover()
		if err != nil {
			panic(err)
		}
	}()

	visit := func(pathName string, f os.FileInfo, err error) error {
		if err != nil {
			return nil
		}
		// One package per directory. Ignore the files themselves.
		if !f.IsDir() {
			return nil
		}
		// No .git or other dot nonsense please.
		if strings.Contains(pathName, slashDot) {
			return filepath.SkipDir
		}
		// Is the tail of the path correct?
		if strings.HasSuffix(pathName, pkgString) {
			result = pathName
			panic(nil)
		}
		return nil
	}

	filepath.Walk(root, visit)
	return "" // Call to panic above sets the real value.
}

// pwd returns the current directory.
func pwd() string {
	wd, err := os.Getwd()
	if err != nil {
		log.Fatal(err)
	}
	return wd
}
<|endoftext|>


<commit_before>package vm

import "fmt"

type Error string

func NewError(s string, xs ...interface{}) *Thunk {
	return Normal(Error(fmt.Sprintf(s, xs...)))
}

func isError(o Object) bool {
	_, ok := o.(Error)
	return ok
}
<commit_msg>Add TODO of error package<commit_after>package vm

import "fmt"

type Error string

func NewError(s string, xs ...interface{}) *Thunk {
	return Normal(Error(fmt.Sprintf(s, xs...)))
}

func ChainError(e *Thunk, s string, xs ...interface{}) *Thunk {
	// TODO: Error { name string, messsage string, child *Thunk }
	return nil
}

func isError(o Object) bool {
	_, ok := o.(Error)
	return ok
}
<|endoftext|>


<commit_before>package vm

import (
	"sync"
	"sync/atomic"
)

type thunkState uint32

const (
	illegal thunkState = iota
	normal
	locked
	app
)

type Thunk struct {
	Result    Object
	function  *Thunk
	args      []*Thunk
	state     thunkState
	blackHole sync.WaitGroup
}

func Normal(o Object) *Thunk {
	return &Thunk{Result: o, state: normal}
}

func App(f *Thunk, args ...*Thunk) *Thunk {
	t := &Thunk{function: f, args: args, state: app}
	t.blackHole.Add(1)
	return t
}

func (t *Thunk) Eval() Object { // into WHNF
	if t.compareAndSwapState(app, locked) {
		go t.function.Eval()

		f, ok := t.function.Eval().(Callable)

		if !ok {
			panic("Something not callable was called.")
		}

		t.Result = f.Call(t.args...).Eval()

		t.function = nil
		t.args = nil

		t.storeState(normal)

		t.blackHole.Done()
	} else {
		t.blackHole.Wait()
	}

	return t.Result
}

func (t *Thunk) compareAndSwapState(old, new thunkState) bool {
	return atomic.CompareAndSwapUint32(
		(*uint32)(&t.state),
		uint32(old),
		uint32(new))
}

func (t *Thunk) storeState(new thunkState) {
	atomic.StoreUint32((*uint32)(&t.state), uint32(new))
}
<commit_msg>Return not callable error<commit_after>package vm

import (
	"sync"
	"sync/atomic"
)

type thunkState uint32

const (
	illegal thunkState = iota
	normal
	locked
	app
)

type Thunk struct {
	Result    Object
	function  *Thunk
	args      []*Thunk
	state     thunkState
	blackHole sync.WaitGroup
}

func Normal(o Object) *Thunk {
	return &Thunk{Result: o, state: normal}
}

func App(f *Thunk, args ...*Thunk) *Thunk {
	t := &Thunk{function: f, args: args, state: app}
	t.blackHole.Add(1)
	return t
}

func (t *Thunk) Eval() Object { // into WHNF
	if t.compareAndSwapState(app, locked) {
		go t.function.Eval()

		f, ok := t.function.Eval().(Callable)

		if !ok {
			return NewError("Something not callable was called.").Eval()
		}

		t.Result = f.Call(t.args...).Eval()

		t.function = nil
		t.args = nil

		t.storeState(normal)

		t.blackHole.Done()
	} else {
		t.blackHole.Wait()
	}

	return t.Result
}

func (t *Thunk) compareAndSwapState(old, new thunkState) bool {
	return atomic.CompareAndSwapUint32(
		(*uint32)(&t.state),
		uint32(old),
		uint32(new))
}

func (t *Thunk) storeState(new thunkState) {
	atomic.StoreUint32((*uint32)(&t.state), uint32(new))
}
<|endoftext|>


<commit_before>package connection

import (
	"bufio"
	"errors"
	"fmt"
	"math"
	"net"
	"strconv"
	"sync"

	"code.google.com/p/goprotobuf/proto"

	"github.com/vito/gordon/warden"
)

var DisconnectedError = errors.New("disconnected")

type Connection struct {
	Disconnected chan bool

	messages chan *warden.Message

	conn      net.Conn
	read      *bufio.Reader
	writeLock sync.Mutex
	readLock  sync.Mutex
}

type WardenError struct {
	Message   string
	Data      string
	Backtrace []string
}

func (e *WardenError) Error() string {
	return e.Message
}

func Connect(socketPath string) (*Connection, error) {
	conn, err := net.Dial("unix", socketPath)
	if err != nil {
		return nil, err
	}

	return New(conn), nil
}

func New(conn net.Conn) *Connection {
	messages := make(chan *warden.Message)

	connection := &Connection{
		// buffered so that read and write errors
		// can both send without blocking
		Disconnected: make(chan bool, 2),

		messages: messages,

		conn: conn,
		read: bufio.NewReader(conn),
	}

	go connection.readMessages()

	return connection
}

func (c *Connection) Close() {
	c.conn.Close()
}

func (c *Connection) Create() (*warden.CreateResponse, error) {
	res, err := c.roundTrip(&warden.CreateRequest{}, &warden.CreateResponse{})
	if err != nil {
		return nil, err
	}

	return res.(*warden.CreateResponse), nil
}

func (c *Connection) Stop(handle string, background, kill bool) (*warden.StopResponse, error) {
	res, err := c.roundTrip(
		&warden.StopRequest{
			Handle:     proto.String(handle),
			Background: proto.Bool(background),
			Kill:       proto.Bool(kill),
		},
		&warden.StopResponse{},
	)

	if err != nil {
		return nil, err
	}

	return res.(*warden.StopResponse), nil
}

func (c *Connection) Destroy(handle string) (*warden.DestroyResponse, error) {
	res, err := c.roundTrip(
		&warden.DestroyRequest{Handle: proto.String(handle)},
		&warden.DestroyResponse{},
	)

	if err != nil {
		return nil, err
	}

	return res.(*warden.DestroyResponse), nil
}

func (c *Connection) Spawn(handle, script string, discardOutput bool) (*warden.SpawnResponse, error) {
	res, err := c.roundTrip(
		&warden.SpawnRequest{
			Handle:        proto.String(handle),
			Script:        proto.String(script),
			DiscardOutput: proto.Bool(discardOutput),
		},
		&warden.SpawnResponse{},
	)

	if err != nil {
		return nil, err
	}

	return res.(*warden.SpawnResponse), nil
}

func (c *Connection) Run(handle, script string) (*warden.RunResponse, error) {
	res, err := c.roundTrip(
		&warden.RunRequest{
			Handle: proto.String(handle),
			Script: proto.String(script),
		},
		&warden.RunResponse{},
	)

	if err != nil {
		return nil, err
	}

	return res.(*warden.RunResponse), nil
}

func (c *Connection) Link(handle string, jobID uint32) (*warden.LinkResponse, error) {
	res, err := c.roundTrip(
		&warden.LinkRequest{
			Handle: proto.String(handle),
			JobId:  proto.Uint32(jobID),
		},
		&warden.LinkResponse{},
	)

	if err != nil {
		return nil, err
	}

	return res.(*warden.LinkResponse), nil
}

func (c *Connection) Stream(handle string, jobId uint32) (chan *warden.StreamResponse, chan bool, error) {
	err := c.sendMessage(
		&warden.StreamRequest{
			Handle: proto.String(handle),
			JobId:  proto.Uint32(jobId),
		},
	)

	if err != nil {
		return nil, nil, err
	}

	responses := make(chan *warden.StreamResponse)

	streamDone := make(chan bool)

	go func() {
		for {
			resMsg, err := c.readResponse(&warden.StreamResponse{})
			if err != nil {
				close(responses)
				close(streamDone)
				break
			}

			response := resMsg.(*warden.StreamResponse)

			responses <- response

			if response.ExitStatus != nil {
				close(responses)
				close(streamDone)
				break
			}
		}
	}()

	return responses, streamDone, nil
}

func (c *Connection) NetIn(handle string) (*warden.NetInResponse, error) {
	res, err := c.roundTrip(
		&warden.NetInRequest{Handle: proto.String(handle)},
		&warden.NetInResponse{},
	)

	if err != nil {
		return nil, err
	}

	return res.(*warden.NetInResponse), nil
}

func (c *Connection) LimitMemory(handle string, limit uint64) (*warden.LimitMemoryResponse, error) {
	res, err := c.roundTrip(
		&warden.LimitMemoryRequest{
			Handle:       proto.String(handle),
			LimitInBytes: proto.Uint64(limit),
		},
		&warden.LimitMemoryResponse{},
	)

	if err != nil {
		return nil, err
	}

	return res.(*warden.LimitMemoryResponse), nil
}

func (c *Connection) GetMemoryLimit(handle string) (uint64, error) {
	res, err := c.roundTrip(
		&warden.LimitMemoryRequest{
			Handle: proto.String(handle),
		},
		&warden.LimitMemoryResponse{},
	)

	if err != nil {
		return 0, err
	}

	limit := res.(*warden.LimitMemoryResponse).GetLimitInBytes()
	if limit == math.MaxInt64 { // PROBABLY NOT A LIMIT
		return 0, nil
	}

	return limit, nil
}

func (c *Connection) LimitDisk(handle string, limit uint64) (*warden.LimitDiskResponse, error) {
	res, err := c.roundTrip(
		&warden.LimitDiskRequest{
			Handle:    proto.String(handle),
			ByteLimit: proto.Uint64(limit),
		},
		&warden.LimitDiskResponse{},
	)

	if err != nil {
		return nil, err
	}

	return res.(*warden.LimitDiskResponse), nil
}

func (c *Connection) GetDiskLimit(handle string) (uint64, error) {
	res, err := c.roundTrip(
		&warden.LimitDiskRequest{
			Handle: proto.String(handle),
		},
		&warden.LimitDiskResponse{},
	)

	if err != nil {
		return 0, err
	}

	return res.(*warden.LimitDiskResponse).GetByteLimit(), nil
}

func (c *Connection) CopyIn(handle, src, dst string) (*warden.CopyInResponse, error) {
	res, err := c.roundTrip(
		&warden.CopyInRequest{
			Handle:  proto.String(handle),
			SrcPath: proto.String(src),
			DstPath: proto.String(dst),
		},
		&warden.CopyInResponse{},
	)

	if err != nil {
		return nil, err
	}

	return res.(*warden.CopyInResponse), nil
}

func (c *Connection) List() (*warden.ListResponse, error) {
	res, err := c.roundTrip(&warden.ListRequest{}, &warden.ListResponse{})
	if err != nil {
		return nil, err
	}

	return res.(*warden.ListResponse), nil
}

func (c *Connection) Info(handle string) (*warden.InfoResponse, error) {
	res, err := c.roundTrip(&warden.InfoRequest{
		Handle: proto.String(handle),
	}, &warden.InfoResponse{})
	if err != nil {
		return nil, err
	}

	return res.(*warden.InfoResponse), nil
}

func (c *Connection) roundTrip(request proto.Message, response proto.Message) (proto.Message, error) {
	err := c.sendMessage(request)
	if err != nil {
		return nil, err
	}

	resp, err := c.readResponse(response)
	if err != nil {
		return nil, err
	}

	return resp, nil
}

func (c *Connection) sendMessage(req proto.Message) error {
	c.writeLock.Lock()
	defer c.writeLock.Unlock()

	request, err := proto.Marshal(req)
	if err != nil {
		return err
	}

	msg := &warden.Message{
		Type:    warden.TypeForMessage(req).Enum(),
		Payload: request,
	}

	data, err := proto.Marshal(msg)
	if err != nil {
		return err
	}

	_, err = c.conn.Write(
		[]byte(
			fmt.Sprintf(
				"%d\r\n%s\r\n",
				len(data),
				data,
			),
		),
	)

	if err != nil {
		c.disconnected()
		return err
	}

	return nil
}

func (c *Connection) readMessages() {
	for {
		payload, err := c.readPayload()
		if err != nil {
			c.disconnected()
			break
		}

		message := &warden.Message{}
		err = proto.Unmarshal(payload, message)
		if err != nil {
			continue
		}

		c.messages <- message
	}
}

func (c *Connection) disconnected() {
	c.Disconnected <- true
	close(c.messages)
}

func (c *Connection) readResponse(response proto.Message) (proto.Message, error) {
	message, ok := <-c.messages
	if !ok {
		return nil, DisconnectedError
	}

	if message.GetType() == warden.Message_Error {
		errorResponse := &warden.ErrorResponse{}
		err := proto.Unmarshal(message.Payload, errorResponse)
		if err != nil {
			return nil, errors.New("error unmarshalling error!")
		}

		return nil, &WardenError{
			Message:   errorResponse.GetMessage(),
			Data:      errorResponse.GetData(),
			Backtrace: errorResponse.GetBacktrace(),
		}
	}

	responseType := warden.TypeForMessage(response)
	if message.GetType() != responseType {
		return nil, errors.New(
			fmt.Sprintf(
				"expected message type %s, got %s\n",
				responseType.String(),
				message.GetType().String(),
			),
		)
	}

	err := proto.Unmarshal(message.GetPayload(), response)

	return response, err
}

func (c *Connection) readPayload() ([]byte, error) {
	c.readLock.Lock()
	defer c.readLock.Unlock()

	msgHeader, err := c.read.ReadBytes('\n')
	if err != nil {
		return nil, err
	}

	msgLen, err := strconv.ParseUint(string(msgHeader[0:len(msgHeader)-2]), 10, 0)
	if err != nil {
		return nil, err
	}

	payload, err := readNBytes(int(msgLen), c.read)
	if err != nil {
		return nil, err
	}

	_, err = readNBytes(2, c.read) // CRLN
	if err != nil {
		return nil, err
	}

	return payload, err
}

func readNBytes(payloadLen int, io *bufio.Reader) ([]byte, error) {
	payload := make([]byte, payloadLen)

	for readCount := 0; readCount < payloadLen; {
		n, err := io.Read(payload[readCount:])
		if err != nil {
			return nil, err
		}

		readCount += n
	}

	return payload, nil
}
<commit_msg>prevent double-closing messages channel<commit_after>package connection

import (
	"bufio"
	"errors"
	"fmt"
	"math"
	"net"
	"strconv"
	"sync"

	"code.google.com/p/goprotobuf/proto"

	"github.com/vito/gordon/warden"
)

var DisconnectedError = errors.New("disconnected")

type Connection struct {
	Disconnected chan bool

	messages chan *warden.Message

	conn      net.Conn
	read      *bufio.Reader
	writeLock sync.Mutex
	readLock  sync.Mutex
}

type WardenError struct {
	Message   string
	Data      string
	Backtrace []string
}

func (e *WardenError) Error() string {
	return e.Message
}

func Connect(socketPath string) (*Connection, error) {
	conn, err := net.Dial("unix", socketPath)
	if err != nil {
		return nil, err
	}

	return New(conn), nil
}

func New(conn net.Conn) *Connection {
	messages := make(chan *warden.Message)

	connection := &Connection{
		// buffered so that read and write errors
		// can both send without blocking
		Disconnected: make(chan bool, 2),

		messages: messages,

		conn: conn,
		read: bufio.NewReader(conn),
	}

	go connection.readMessages()

	return connection
}

func (c *Connection) Close() {
	c.conn.Close()
}

func (c *Connection) Create() (*warden.CreateResponse, error) {
	res, err := c.roundTrip(&warden.CreateRequest{}, &warden.CreateResponse{})
	if err != nil {
		return nil, err
	}

	return res.(*warden.CreateResponse), nil
}

func (c *Connection) Stop(handle string, background, kill bool) (*warden.StopResponse, error) {
	res, err := c.roundTrip(
		&warden.StopRequest{
			Handle:     proto.String(handle),
			Background: proto.Bool(background),
			Kill:       proto.Bool(kill),
		},
		&warden.StopResponse{},
	)

	if err != nil {
		return nil, err
	}

	return res.(*warden.StopResponse), nil
}

func (c *Connection) Destroy(handle string) (*warden.DestroyResponse, error) {
	res, err := c.roundTrip(
		&warden.DestroyRequest{Handle: proto.String(handle)},
		&warden.DestroyResponse{},
	)

	if err != nil {
		return nil, err
	}

	return res.(*warden.DestroyResponse), nil
}

func (c *Connection) Spawn(handle, script string, discardOutput bool) (*warden.SpawnResponse, error) {
	res, err := c.roundTrip(
		&warden.SpawnRequest{
			Handle:        proto.String(handle),
			Script:        proto.String(script),
			DiscardOutput: proto.Bool(discardOutput),
		},
		&warden.SpawnResponse{},
	)

	if err != nil {
		return nil, err
	}

	return res.(*warden.SpawnResponse), nil
}

func (c *Connection) Run(handle, script string) (*warden.RunResponse, error) {
	res, err := c.roundTrip(
		&warden.RunRequest{
			Handle: proto.String(handle),
			Script: proto.String(script),
		},
		&warden.RunResponse{},
	)

	if err != nil {
		return nil, err
	}

	return res.(*warden.RunResponse), nil
}

func (c *Connection) Link(handle string, jobID uint32) (*warden.LinkResponse, error) {
	res, err := c.roundTrip(
		&warden.LinkRequest{
			Handle: proto.String(handle),
			JobId:  proto.Uint32(jobID),
		},
		&warden.LinkResponse{},
	)

	if err != nil {
		return nil, err
	}

	return res.(*warden.LinkResponse), nil
}

func (c *Connection) Stream(handle string, jobId uint32) (chan *warden.StreamResponse, chan bool, error) {
	err := c.sendMessage(
		&warden.StreamRequest{
			Handle: proto.String(handle),
			JobId:  proto.Uint32(jobId),
		},
	)

	if err != nil {
		return nil, nil, err
	}

	responses := make(chan *warden.StreamResponse)

	streamDone := make(chan bool)

	go func() {
		for {
			resMsg, err := c.readResponse(&warden.StreamResponse{})
			if err != nil {
				close(responses)
				close(streamDone)
				break
			}

			response := resMsg.(*warden.StreamResponse)

			responses <- response

			if response.ExitStatus != nil {
				close(responses)
				close(streamDone)
				break
			}
		}
	}()

	return responses, streamDone, nil
}

func (c *Connection) NetIn(handle string) (*warden.NetInResponse, error) {
	res, err := c.roundTrip(
		&warden.NetInRequest{Handle: proto.String(handle)},
		&warden.NetInResponse{},
	)

	if err != nil {
		return nil, err
	}

	return res.(*warden.NetInResponse), nil
}

func (c *Connection) LimitMemory(handle string, limit uint64) (*warden.LimitMemoryResponse, error) {
	res, err := c.roundTrip(
		&warden.LimitMemoryRequest{
			Handle:       proto.String(handle),
			LimitInBytes: proto.Uint64(limit),
		},
		&warden.LimitMemoryResponse{},
	)

	if err != nil {
		return nil, err
	}

	return res.(*warden.LimitMemoryResponse), nil
}

func (c *Connection) GetMemoryLimit(handle string) (uint64, error) {
	res, err := c.roundTrip(
		&warden.LimitMemoryRequest{
			Handle: proto.String(handle),
		},
		&warden.LimitMemoryResponse{},
	)

	if err != nil {
		return 0, err
	}

	limit := res.(*warden.LimitMemoryResponse).GetLimitInBytes()
	if limit == math.MaxInt64 { // PROBABLY NOT A LIMIT
		return 0, nil
	}

	return limit, nil
}

func (c *Connection) LimitDisk(handle string, limit uint64) (*warden.LimitDiskResponse, error) {
	res, err := c.roundTrip(
		&warden.LimitDiskRequest{
			Handle:    proto.String(handle),
			ByteLimit: proto.Uint64(limit),
		},
		&warden.LimitDiskResponse{},
	)

	if err != nil {
		return nil, err
	}

	return res.(*warden.LimitDiskResponse), nil
}

func (c *Connection) GetDiskLimit(handle string) (uint64, error) {
	res, err := c.roundTrip(
		&warden.LimitDiskRequest{
			Handle: proto.String(handle),
		},
		&warden.LimitDiskResponse{},
	)

	if err != nil {
		return 0, err
	}

	return res.(*warden.LimitDiskResponse).GetByteLimit(), nil
}

func (c *Connection) CopyIn(handle, src, dst string) (*warden.CopyInResponse, error) {
	res, err := c.roundTrip(
		&warden.CopyInRequest{
			Handle:  proto.String(handle),
			SrcPath: proto.String(src),
			DstPath: proto.String(dst),
		},
		&warden.CopyInResponse{},
	)

	if err != nil {
		return nil, err
	}

	return res.(*warden.CopyInResponse), nil
}

func (c *Connection) List() (*warden.ListResponse, error) {
	res, err := c.roundTrip(&warden.ListRequest{}, &warden.ListResponse{})
	if err != nil {
		return nil, err
	}

	return res.(*warden.ListResponse), nil
}

func (c *Connection) Info(handle string) (*warden.InfoResponse, error) {
	res, err := c.roundTrip(&warden.InfoRequest{
		Handle: proto.String(handle),
	}, &warden.InfoResponse{})
	if err != nil {
		return nil, err
	}

	return res.(*warden.InfoResponse), nil
}

func (c *Connection) roundTrip(request proto.Message, response proto.Message) (proto.Message, error) {
	err := c.sendMessage(request)
	if err != nil {
		return nil, err
	}

	resp, err := c.readResponse(response)
	if err != nil {
		return nil, err
	}

	return resp, nil
}

func (c *Connection) sendMessage(req proto.Message) error {
	c.writeLock.Lock()
	defer c.writeLock.Unlock()

	request, err := proto.Marshal(req)
	if err != nil {
		return err
	}

	msg := &warden.Message{
		Type:    warden.TypeForMessage(req).Enum(),
		Payload: request,
	}

	data, err := proto.Marshal(msg)
	if err != nil {
		return err
	}

	_, err = c.conn.Write(
		[]byte(
			fmt.Sprintf(
				"%d\r\n%s\r\n",
				len(data),
				data,
			),
		),
	)

	if err != nil {
		c.disconnected()
		return err
	}

	return nil
}

func (c *Connection) readMessages() {
	for {
		payload, err := c.readPayload()
		if err != nil {
			c.disconnected()
			close(c.messages)
			break
		}

		message := &warden.Message{}
		err = proto.Unmarshal(payload, message)
		if err != nil {
			continue
		}

		c.messages <- message
	}
}

func (c *Connection) disconnected() {
	c.Disconnected <- true
}

func (c *Connection) readResponse(response proto.Message) (proto.Message, error) {
	message, ok := <-c.messages
	if !ok {
		return nil, DisconnectedError
	}

	if message.GetType() == warden.Message_Error {
		errorResponse := &warden.ErrorResponse{}
		err := proto.Unmarshal(message.Payload, errorResponse)
		if err != nil {
			return nil, errors.New("error unmarshalling error!")
		}

		return nil, &WardenError{
			Message:   errorResponse.GetMessage(),
			Data:      errorResponse.GetData(),
			Backtrace: errorResponse.GetBacktrace(),
		}
	}

	responseType := warden.TypeForMessage(response)
	if message.GetType() != responseType {
		return nil, errors.New(
			fmt.Sprintf(
				"expected message type %s, got %s\n",
				responseType.String(),
				message.GetType().String(),
			),
		)
	}

	err := proto.Unmarshal(message.GetPayload(), response)

	return response, err
}

func (c *Connection) readPayload() ([]byte, error) {
	c.readLock.Lock()
	defer c.readLock.Unlock()

	msgHeader, err := c.read.ReadBytes('\n')
	if err != nil {
		return nil, err
	}

	msgLen, err := strconv.ParseUint(string(msgHeader[0:len(msgHeader)-2]), 10, 0)
	if err != nil {
		return nil, err
	}

	payload, err := readNBytes(int(msgLen), c.read)
	if err != nil {
		return nil, err
	}

	_, err = readNBytes(2, c.read) // CRLN
	if err != nil {
		return nil, err
	}

	return payload, err
}

func readNBytes(payloadLen int, io *bufio.Reader) ([]byte, error) {
	payload := make([]byte, payloadLen)

	for readCount := 0; readCount < payloadLen; {
		n, err := io.Read(payload[readCount:])
		if err != nil {
			return nil, err
		}

		readCount += n
	}

	return payload, nil
}
<|endoftext|>


<commit_before>package consul

import (
	"math/rand"
	"sync/atomic"
	"time"

	"github.com/hashicorp/consul/lib"
)

type consulServerEventTypes int

const (
	// consulServersNodeJoin is used to notify of a new consulServer.
	// The primary effect of this is a reshuffling of consulServers and
	// finding a new preferredServer.
	consulServersNodeJoin = iota

	// consulServersRebalance is used to signal we should rebalance our
	// connection load across servers
	consulServersRebalance

	// consulServersRPCError is used to signal when a server has either
	// timed out or returned an error and we would like to have the
	// server manager find a new preferredServer.
	consulServersRPCError
)

// serverCfg is the thread-safe configuration structure that is used to
// maintain the list of consul servers in Client.
//
// NOTE(sean@): We are explicitly relying on the fact that this is copied.
// Please keep this structure light.
type serverConfig struct {
	// servers tracks the locally known servers
	servers []*serverParts

	// Timer used to control rebalancing of servers
	rebalanceTimer *time.Timer
}

// consulServersManager is used to automatically shuffle and rebalance the
// list of consulServers.  This maintenance happens either when a new server
// is added or when a duration has been exceed.
func (c *Client) consulServersManager() {
	defaultTimeout := 5 * time.Second // FIXME(sean@): This is a bullshit value
	var rebalanceTimer *time.Timer
	func(c *Client) {
		c.serverConfigLock.Lock()
		defer c.serverConfigLock.Unlock()

		serverCfgPtr := c.serverConfigValue.Load()
		if serverCfgPtr == nil {
			panic("server config has not been initialized")
		}
		var serverCfg serverConfig
		serverCfg = serverCfgPtr.(serverConfig)
		rebalanceTimer = time.NewTimer(defaultTimeout)
		serverCfg.rebalanceTimer = rebalanceTimer
	}(c)

	for {
		select {
		case e := <-c.consulServersCh:
			switch e {
			case consulServersNodeJoin:
				c.logger.Printf("[INFO] consul: new node joined cluster")
				c.RebalanceServers()
			case consulServersRebalance:
				c.logger.Printf("[INFO] consul: rebalancing servers by request")
				c.RebalanceServers()
			case consulServersRPCError:
				c.logger.Printf("[INFO] consul: need to find a new server to talk with")
				c.CycleFailedServers()
				// FIXME(sean@): wtb preemptive Status.Ping
				// of servers, ideally parallel fan-out of N
				// nodes, then settle on the first node which
				// responds successfully.
				//
				// Is there a distinction between slow and
				// offline?  Do we run the Status.Ping with a
				// fixed timeout (say 30s) that way we can
				// alert administrators that they've set
				// their RPC time too low even though the
				// Ping did return successfully?
			default:
				c.logger.Printf("[WARN] consul: unhandled LAN Serf Event: %#v", e)
			}
		case <-rebalanceTimer.C:
			c.logger.Printf("[INFO] consul: server rebalance timeout")
			c.RebalanceServers()

		case <-c.shutdownCh:
			return
		}
	}
}

func (c *Client) AddServer(server *serverParts) {
	c.serverConfigLock.Lock()
	defer c.serverConfigLock.Unlock()
	serverCfg := c.serverConfigValue.Load().(serverConfig)

	// Check if this server is known
	found := false
	for idx, existing := range serverCfg.servers {
		if existing.Name == server.Name {
			// Overwrite the existing server parts in order to
			// possibly update metadata (i.e. server version)
			serverCfg.servers[idx] = server
			found = true
			break
		}
	}

	// Add to the list if not known
	if !found {
		serverCfg.servers = append(serverCfg.servers, server)

		// Notify the server maintenance task of a new server
		c.consulServersCh <- consulServersNodeJoin
	}

	c.serverConfigValue.Store(serverCfg)

}

func (c *Client) CycleFailedServers() {
	c.serverConfigLock.Lock()
	defer c.serverConfigLock.Unlock()
	serverCfg := c.serverConfigValue.Load().(serverConfig)

	for i := range serverCfg.servers {
		failCount := atomic.LoadUint64(&(serverCfg.servers[i].Disabled))
		if failCount == 0 {
			break
		} else if failCount > 0 {
			serverCfg.servers = serverCfg.cycleServer()
		}
	}

	serverCfg.resetRebalanceTimer(c)
	c.serverConfigValue.Store(serverCfg)
}

func (sc *serverConfig) cycleServer() (servers []*serverParts) {
	numServers := len(servers)
	if numServers < 2 {
		// No action required for zero or one server situations
		return servers
	}

	var failedNode *serverParts
	failedNode, servers = servers[0], servers[1:]
	servers = append(servers, failedNode)
	return servers
}

func (c *Client) RebalanceServers() {
	c.serverConfigLock.Lock()
	defer c.serverConfigLock.Unlock()
	serverCfg := c.serverConfigValue.Load().(serverConfig)

	// Shuffle the server list on server join.  Servers are selected from
	// the head of the list and are moved to the end of the list on
	// failure.
	for i := len(serverCfg.servers) - 1; i > 0; i-- {
		j := rand.Int31n(int32(i + 1))
		serverCfg.servers[i], serverCfg.servers[j] = serverCfg.servers[j], serverCfg.servers[i]
	}

	serverCfg.resetRebalanceTimer(c)
	c.serverConfigValue.Store(serverCfg)
}

func (c *Client) RemoveServer(server *serverParts) {
	c.serverConfigLock.Lock()
	defer c.serverConfigLock.Unlock()
	serverCfg := c.serverConfigValue.Load().(serverConfig)

	// Remove the server if known
	n := len(serverCfg.servers)
	for i := 0; i < n; i++ {
		if serverCfg.servers[i].Name == server.Name {
			serverCfg.servers[i], serverCfg.servers[n-1] = serverCfg.servers[n-1], nil
			serverCfg.servers = serverCfg.servers[:n-1]
			break
		}
	}

	c.serverConfigValue.Store(serverCfg)

}

// resetRebalanceTimer assumes:
//
// 1) the serverConfigLock is already held by the caller.
// 2) the caller will call serverConfigValue.Store()
func (sc *serverConfig) resetRebalanceTimer(c *Client) {
	numConsulServers := len(sc.servers)
	// Limit this connection's life based on the size (and health) of the
	// cluster.  Never rebalance a connection more frequently than
	// connReuseLowWatermarkDuration, and make sure we never exceed
	// clusterWideRebalanceConnsPerSec operations/s across numLANMembers.
	clusterWideRebalanceConnsPerSec := float64(numConsulServers * newRebalanceConnsPerSecPerServer)
	connReuseLowWatermarkDuration := clientRPCMinReuseDuration + lib.RandomStagger(clientRPCMinReuseDuration/clientRPCJitterFraction)
	numLANMembers := len(c.LANMembers())
	connRebalanceTimeout := lib.RateScaledInterval(clusterWideRebalanceConnsPerSec, connReuseLowWatermarkDuration, numLANMembers)
	c.logger.Printf("[DEBUG] consul: connection will be rebalanced in %v", connRebalanceTimeout)

	sc.rebalanceTimer.Reset(connRebalanceTimeout)
}
<commit_msg>Move consul.serverConfig out of the consul package<commit_after><|endoftext|>


<commit_before>package commands

import (
	"errors"
	"fmt"
	"io"
	"strings"
	"time"

	cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv"
	e "github.com/ipfs/go-ipfs/core/commands/e"
	ncmd "github.com/ipfs/go-ipfs/core/commands/name"
	coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface"
	options "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
	ns "github.com/ipfs/go-ipfs/namesys"
	nsopts "github.com/ipfs/go-ipfs/namesys/opts"
	path "gx/ipfs/QmX7uSbkNz76yNwBhuwYwRbhihLnJqM73VTCjS3UMJud9A/go-path"

	"gx/ipfs/QmPXR4tNdLbp8HsZiPMjpsgqphX9Vhw2J6Jh5MKH2ovW3D/go-ipfs-cmds"
	"gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit"
)

var ResolveCmd = &cmds.Command{
	Helptext: cmdkit.HelpText{
		Tagline: "Resolve the value of names to IPFS.",
		ShortDescription: `
There are a number of mutable name protocols that can link among
themselves and into IPNS. This command accepts any of these
identifiers and resolves them to the referenced item.
`,
		LongDescription: `
There are a number of mutable name protocols that can link among
themselves and into IPNS. For example IPNS references can (currently)
point at an IPFS object, and DNS links can point at other DNS links, IPNS
entries, or IPFS objects. This command accepts any of these
identifiers and resolves them to the referenced item.

EXAMPLES

Resolve the value of your identity:

  $ ipfs resolve /ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy
  /ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj

Resolve the value of another name:

  $ ipfs resolve /ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n
  /ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy

Resolve the value of another name recursively:

  $ ipfs resolve -r /ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n
  /ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj

Resolve the value of an IPFS DAG path:

  $ ipfs resolve /ipfs/QmeZy1fGbwgVSrqbfh9fKQrAWgeyRnj7h8fsHS1oy3k99x/beep/boop
  /ipfs/QmYRMjyvAiHKN9UTi8Bzt1HUspmSRD8T8DwxfSMzLgBon1

`,
	},

	Arguments: []cmdkit.Argument{
		cmdkit.StringArg("name", true, false, "The name to resolve.").EnableStdin(),
	},
	Options: []cmdkit.Option{
		cmdkit.BoolOption("recursive", "r", "Resolve until the result is an IPFS name."),
		cmdkit.IntOption("dht-record-count", "dhtrc", "Number of records to request for DHT resolution."),
		cmdkit.StringOption("dht-timeout", "dhtt", "Max time to collect values during DHT resolution eg \"30s\". Pass 0 for no timeout."),
	},
	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) {
		api, err := cmdenv.GetApi(env)
		if err != nil {
			res.SetError(err, cmdkit.ErrNormal)
			return
		}

		n, err := cmdenv.GetNode(env)
		if err != nil {
			res.SetError(err, cmdkit.ErrNormal)
			return
		}

		if !n.OnlineMode() {
			err := n.SetupOfflineRouting()
			if err != nil {
				res.SetError(err, cmdkit.ErrNormal)
				return
			}
		}

		name := req.Arguments[0]
		recursive, _ := req.Options["recursive"].(bool)

		// the case when ipns is resolved step by step
		if strings.HasPrefix(name, "/ipns/") && !recursive {
			rc, rcok := req.Options["dht-record-count"].(uint)
			dhtt, dhttok := req.Options["dht-timeout"].(string)
			ropts := []options.NameResolveOption{
				options.Name.ResolveOption(nsopts.Depth(1)),
			}

			if rcok {
				ropts = append(ropts, options.Name.ResolveOption(nsopts.DhtRecordCount(rc)))
			}
			if dhttok {
				d, err := time.ParseDuration(dhtt)
				if err != nil {
					res.SetError(err, cmdkit.ErrNormal)
					return
				}
				if d < 0 {
					res.SetError(errors.New("DHT timeout value must be >= 0"), cmdkit.ErrNormal)
					return
				}
				ropts = append(ropts, options.Name.ResolveOption(nsopts.DhtTimeout(d)))
			}
			p, err := api.Name().Resolve(req.Context, name, ropts...)
			// ErrResolveRecursion is fine
			if err != nil && err != ns.ErrResolveRecursion {
				res.SetError(err, cmdkit.ErrNormal)
				return
			}
			cmds.EmitOnce(res, &ncmd.ResolvedPath{Path: path.Path(p.String())})
			return
		}

		// else, ipfs path or ipns with recursive flag
		p, err := coreiface.ParsePath(name)
		if err != nil {
			res.SetError(err, cmdkit.ErrNormal)
			return
		}

		rp, err := api.ResolvePath(req.Context, p)
		if err != nil {
			res.SetError(err, cmdkit.ErrNormal)
			return
		}

		c := rp.Cid()

		cmds.EmitOnce(res, &ncmd.ResolvedPath{Path: path.FromCid(c)})
	},
	Encoders: cmds.EncoderMap{
		cmds.Text: cmds.MakeEncoder(func(req *cmds.Request, w io.Writer, v interface{}) error {
			output, ok := v.(*ncmd.ResolvedPath)
			if !ok {
				return e.TypeErr(output, v)
			}

			fmt.Fprintln(w, output.Path.String())
			return nil
		}),
	},
	Type: ncmd.ResolvedPath{},
}
<commit_msg>fix resolve command for the new cmds refactor<commit_after>package commands

import (
	"errors"
	"fmt"
	"io"
	"strings"
	"time"

	cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv"
	e "github.com/ipfs/go-ipfs/core/commands/e"
	ncmd "github.com/ipfs/go-ipfs/core/commands/name"
	coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface"
	options "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
	ns "github.com/ipfs/go-ipfs/namesys"
	nsopts "github.com/ipfs/go-ipfs/namesys/opts"
	path "gx/ipfs/QmX7uSbkNz76yNwBhuwYwRbhihLnJqM73VTCjS3UMJud9A/go-path"

	"gx/ipfs/QmPXR4tNdLbp8HsZiPMjpsgqphX9Vhw2J6Jh5MKH2ovW3D/go-ipfs-cmds"
	"gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit"
)

var ResolveCmd = &cmds.Command{
	Helptext: cmdkit.HelpText{
		Tagline: "Resolve the value of names to IPFS.",
		ShortDescription: `
There are a number of mutable name protocols that can link among
themselves and into IPNS. This command accepts any of these
identifiers and resolves them to the referenced item.
`,
		LongDescription: `
There are a number of mutable name protocols that can link among
themselves and into IPNS. For example IPNS references can (currently)
point at an IPFS object, and DNS links can point at other DNS links, IPNS
entries, or IPFS objects. This command accepts any of these
identifiers and resolves them to the referenced item.

EXAMPLES

Resolve the value of your identity:

  $ ipfs resolve /ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy
  /ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj

Resolve the value of another name:

  $ ipfs resolve /ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n
  /ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy

Resolve the value of another name recursively:

  $ ipfs resolve -r /ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n
  /ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj

Resolve the value of an IPFS DAG path:

  $ ipfs resolve /ipfs/QmeZy1fGbwgVSrqbfh9fKQrAWgeyRnj7h8fsHS1oy3k99x/beep/boop
  /ipfs/QmYRMjyvAiHKN9UTi8Bzt1HUspmSRD8T8DwxfSMzLgBon1

`,
	},

	Arguments: []cmdkit.Argument{
		cmdkit.StringArg("name", true, false, "The name to resolve.").EnableStdin(),
	},
	Options: []cmdkit.Option{
		cmdkit.BoolOption("recursive", "r", "Resolve until the result is an IPFS name."),
		cmdkit.IntOption("dht-record-count", "dhtrc", "Number of records to request for DHT resolution."),
		cmdkit.StringOption("dht-timeout", "dhtt", "Max time to collect values during DHT resolution eg \"30s\". Pass 0 for no timeout."),
	},
	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
		api, err := cmdenv.GetApi(env)
		if err != nil {
			return err
		}

		n, err := cmdenv.GetNode(env)
		if err != nil {
			return err
		}

		if !n.OnlineMode() {
			err := n.SetupOfflineRouting()
			if err != nil {
				return err
			}
		}

		name := req.Arguments[0]
		recursive, _ := req.Options["recursive"].(bool)

		// the case when ipns is resolved step by step
		if strings.HasPrefix(name, "/ipns/") && !recursive {
			rc, rcok := req.Options["dht-record-count"].(uint)
			dhtt, dhttok := req.Options["dht-timeout"].(string)
			ropts := []options.NameResolveOption{
				options.Name.ResolveOption(nsopts.Depth(1)),
			}

			if rcok {
				ropts = append(ropts, options.Name.ResolveOption(nsopts.DhtRecordCount(rc)))
			}
			if dhttok {
				d, err := time.ParseDuration(dhtt)
				if err != nil {
					return err
				}
				if d < 0 {
					return errors.New("DHT timeout value must be >= 0")
				}
				ropts = append(ropts, options.Name.ResolveOption(nsopts.DhtTimeout(d)))
			}
			p, err := api.Name().Resolve(req.Context, name, ropts...)
			// ErrResolveRecursion is fine
			if err != nil && err != ns.ErrResolveRecursion {
				return err
			}
			return cmds.EmitOnce(res, &ncmd.ResolvedPath{Path: path.Path(p.String())})
		}

		// else, ipfs path or ipns with recursive flag
		p, err := coreiface.ParsePath(name)
		if err != nil {
			return err
		}

		rp, err := api.ResolvePath(req.Context, p)
		if err != nil {
			return err
		}

		c := rp.Cid()

		return cmds.EmitOnce(res, &ncmd.ResolvedPath{Path: path.FromCid(c)})
	},
	Encoders: cmds.EncoderMap{
		cmds.Text: cmds.MakeEncoder(func(req *cmds.Request, w io.Writer, v interface{}) error {
			output, ok := v.(*ncmd.ResolvedPath)
			if !ok {
				return e.TypeErr(output, v)
			}

			fmt.Fprintln(w, output.Path.String())
			return nil
		}),
	},
	Type: ncmd.ResolvedPath{},
}
<|endoftext|>


<commit_before>// Copyright 2018 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sequencer

import (
	"context"
	"sync"

	"github.com/gogo/protobuf/proto"
	"github.com/golang/glog"
	"github.com/golang/protobuf/ptypes/empty"
	"github.com/google/keytransparency/core/client"
	"github.com/google/keytransparency/core/domain"
	"github.com/google/keytransparency/core/keyserver"
	"github.com/google/keytransparency/core/mutator"
	"github.com/google/keytransparency/core/mutator/entry"
	"github.com/google/trillian/monitoring"
	"github.com/google/trillian/types"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	ktpb "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto"
	spb "github.com/google/keytransparency/core/sequencer/sequencer_go_proto"
	tpb "github.com/google/trillian"
	tclient "github.com/google/trillian/client"
)

const (
	domainIDLabel = "domainid"
	reasonLabel   = "reason"
)

var (
	once             sync.Once
	knownDomains     monitoring.Gauge
	batchSize        monitoring.Gauge
	mutationCount    monitoring.Counter
	mutationFailures monitoring.Counter
)

func createMetrics(mf monitoring.MetricFactory) {
	knownDomains = mf.NewGauge(
		"known_domains",
		"Set to 1 for known domains (whether this instance is master or not)",
		domainIDLabel)
	mutationCount = mf.NewCounter(
		"mutation_count",
		"Number of mutations the signer has processed for domainid since process start",
		domainIDLabel)
	mutationFailures = mf.NewCounter(
		"mutation_failures",
		"Number of invalid mutations the signer has processed for domainid since process start",
		domainIDLabel, reasonLabel)
	batchSize = mf.NewGauge(
		"batch_size",
		"Number of mutations the signer is attempting to process for domainid",
		domainIDLabel)
}

// Server implements KeyTransparencySequencerServer.
type Server struct {
	ktServer  *keyserver.Server
	mutations mutator.MutationStorage
	tmap      tpb.TrillianMapClient
	tlog      tpb.TrillianLogClient
}

// NewServer creates a new KeyTransparencySequencerServer.
func NewServer(
	domains domain.Storage,
	logAdmin tpb.TrillianAdminClient,
	mapAdmin tpb.TrillianAdminClient,
	tlog tpb.TrillianLogClient,
	tmap tpb.TrillianMapClient,
	mutations mutator.MutationStorage,
	metricsFactory monitoring.MetricFactory,
) *Server {
	once.Do(func() { createMetrics(metricsFactory) })
	return &Server{
		ktServer:  keyserver.New(nil, nil, logAdmin, mapAdmin, nil, domains, nil, nil),
		tlog:      tlog,
		tmap:      tmap,
		mutations: mutations,
	}
}

// CreateEpoch applies the supplied mutations to the current map revision and creates a new epoch.
func (s *Server) CreateEpoch(ctx context.Context, in *spb.CreateEpochRequest) (*empty.Empty, error) {
	domainID := in.GetDomainId()
	msgs := in.GetMessages()
	glog.Infof("CreateEpoch: for %v with %d messages", domainID, len(msgs))
	// Fetch verification objects for domainID.
	config, err := s.ktServer.GetDomain(ctx, &ktpb.GetDomainRequest{DomainId: domainID})
	if err != nil {
		return nil, err
	}
	cli, err := client.NewVerifierFromDomain(config)
	if err != nil {
		return nil, err
	}

	// Parse mutations using the mutator for this domain.
	batchSize.Set(float64(len(msgs)), config.DomainId)
	indexes := make([][]byte, 0, len(msgs))
	for _, m := range msgs {
		indexes = append(indexes, m.GetMutation().GetIndex())
	}
	glog.V(2).Infof("CreateEpoch: %v mutations, %v indexes", len(msgs), len(indexes))

	mapRoot, err := s.getAndVerifyMapRoot(ctx, cli, config.Map.TreeId)
	if err != nil {
		return nil, err
	}
	glog.V(3).Infof("CreateEpoch: Previous SignedMapRoot: {Revision: %v}", mapRoot.Revision)

	leaves, err := s.getAndVerifyMapLeaves(ctx, cli.MapVerifier, mapRoot, config.Map.TreeId, indexes)
	if err != nil {
		return nil, err
	}

	// Apply mutations to values.
	newLeaves, err := s.applyMutations(domainID, entry.New(), in.GetMessages(), leaves)
	if err != nil {
		return nil, err
	}

	// Set new leaf values.
	setResp, err := s.tmap.SetLeaves(ctx, &tpb.SetMapLeavesRequest{
		MapId:  config.Map.TreeId,
		Leaves: newLeaves,
	})
	if err != nil {
		return nil, status.Errorf(codes.Internal, "tmap.SetLeaves(): %v", err)
	}
	mapRoot, err = cli.VerifySignedMapRoot(setResp.GetMapRoot())
	if err != nil {
		return nil, status.Errorf(codes.Internal, "VerifySignedMapRoot(): %v", err)
	}
	glog.V(2).Infof("CreateEpoch: SetLeaves:{Revision: %v}", mapRoot.Revision)

	// Write mutations associated with this epoch.
	mutations := make([]*ktpb.Entry, 0, len(msgs))
	for _, msg := range msgs {
		mutations = append(mutations, msg.Mutation)
	}
	if err := s.mutations.WriteBatch(ctx, domainID, int64(mapRoot.Revision), mutations); err != nil {
		glog.Errorf("Could not write mutations for revision %v: %v", mapRoot.Revision, err)
		return nil, status.Errorf(codes.Internal, "mutations.WriteBatch(): %v", err)
	}

	// TODO(gbelvin): Store and track trustedRoot.
	trustedRoot := types.LogRootV1{} // Automatically trust the first observed log root.

	// Put SignedMapHead in the append only log.
	logClient, err := tclient.NewFromTree(s.tlog, config.Log, trustedRoot)
	if err != nil {
		return nil, err
	}
	if err := logClient.AddSequencedLeafAndWait(ctx, setResp.GetMapRoot().GetMapRoot(), int64(mapRoot.Revision)); err != nil {
		glog.Fatalf("AddSequencedLeaf(logID: %v, rev: %v): %v", config.Log.TreeId, mapRoot.Revision, err)
		// TODO(gdbelvin): Implement retries.
		return nil, err
	}

	mutationCount.Add(float64(len(msgs)), domainID)
	glog.Infof("CreatedEpoch: rev: %v with %v mutations, root: %x", mapRoot.Revision, len(msgs), mapRoot.RootHash)
	return &empty.Empty{}, nil
}

// applyMutations takes the set of mutations and applies them to given leafs.
// Multiple mutations for the same leaf will be applied to provided leaf.
// The last valid mutation for each leaf is included in the output.
// Returns a list of map leaves that should be updated.
func (s *Server) applyMutations(domainID string, mutatorFunc mutator.Func,
	msgs []*ktpb.EntryUpdate, leaves []*tpb.MapLeaf) ([]*tpb.MapLeaf, error) {
	// Put leaves in a map from index to leaf value.
	leafMap := make(map[string]*tpb.MapLeaf)
	for _, l := range leaves {
		leafMap[string(l.Index)] = l
	}

	retMap := make(map[string]*tpb.MapLeaf)
	for _, msg := range msgs {
		index := msg.Mutation.GetIndex()
		var oldValue *ktpb.Entry // If no map leaf was found, oldValue will be nil.
		if leaf, ok := leafMap[string(index)]; ok {
			var err error
			oldValue, err = entry.FromLeafValue(leaf.GetLeafValue())
			if err != nil {
				glog.Warningf("entry.FromLeafValue(%v): %v", leaf.GetLeafValue(), err)
				mutationFailures.Inc(domainID, "Unmarshal")
				continue
			}
		}

		newValue, err := mutatorFunc.Mutate(oldValue, msg.Mutation)
		if err != nil {
			glog.Warningf("Mutate(): %v", err)
			mutationFailures.Inc(domainID, "Mutate")
			continue // A bad mutation should not make the whole batch fail.
		}
		leafValue, err := entry.ToLeafValue(newValue)
		if err != nil {
			glog.Warningf("ToLeafValue(): %v", err)
			mutationFailures.Inc(domainID, "Marshal")
			continue
		}
		extraData, err := proto.Marshal(msg.Committed)
		if err != nil {
			glog.Warningf("proto.Marshal(): %v", err)
			mutationFailures.Inc(domainID, "Marshal")
			continue
		}

		// Make sure that only ONE MapLeaf is output per index.
		retMap[string(index)] = &tpb.MapLeaf{
			Index:     index,
			LeafValue: leafValue,
			ExtraData: extraData,
		}
	}
	// Convert return map back into a list.
	ret := make([]*tpb.MapLeaf, 0, len(retMap))
	for _, v := range retMap {
		ret = append(ret, v)
	}
	glog.V(2).Infof("applyMutations applied %v mutations to %v leaves", len(msgs), len(leaves))
	return ret, nil
}

func (s *Server) getAndVerifyMapRoot(ctx context.Context, cli client.Verifier, mapID int64) (*types.MapRootV1, error) {
	rootResp, err := s.tmap.GetSignedMapRoot(ctx, &tpb.GetSignedMapRootRequest{MapId: mapID})
	if err != nil {
		return nil, status.Errorf(codes.Internal, "GetSignedMapRoot(%v): %v", mapID, err)
	}
	mapRoot, err := cli.VerifySignedMapRoot(rootResp.GetMapRoot())
	if err != nil {
		return nil, status.Errorf(codes.Internal, "VerifySignedMapRoot(): %v", err)
	}
	return mapRoot, nil
}

func (s *Server) getAndVerifyMapLeaves(ctx context.Context, verifier *tclient.MapVerifier, mapRoot *types.MapRootV1, mapID int64, indexes [][]byte) ([]*tpb.MapLeaf, error) {
	getResp, err := s.tmap.GetLeaves(ctx, &tpb.GetMapLeavesRequest{
		MapId: mapID,
		Index: indexes,
	})
	if err != nil {
		return nil, status.Errorf(codes.Internal, "tmap.GetLeaves(): %v", err)
	}
	if got, want := len(getResp.MapLeafInclusion), len(indexes); got != want {
		return nil, status.Errorf(codes.Internal, "got %v leaves, want %v", got, want)
	}
	leaves := make([]*tpb.MapLeaf, 0, len(getResp.MapLeafInclusion))
	for _, m := range getResp.MapLeafInclusion {
		if err := verifier.VerifyMapLeafInclusionHash(mapRoot.RootHash, m); err != nil {
			return nil, status.Errorf(codes.Internal, "map: VerifyMapLeafInclusion(): %v", err)
		}
		leaves = append(leaves, m.Leaf)
	}
	return leaves, nil
}
<commit_msg>Use MapClient's convenience methods. (#1043)<commit_after>// Copyright 2018 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sequencer

import (
	"context"
	"sync"

	"github.com/gogo/protobuf/proto"
	"github.com/golang/glog"
	"github.com/golang/protobuf/ptypes/empty"
	"github.com/google/keytransparency/core/domain"
	"github.com/google/keytransparency/core/keyserver"
	"github.com/google/keytransparency/core/mutator"
	"github.com/google/keytransparency/core/mutator/entry"
	"github.com/google/trillian/monitoring"
	"github.com/google/trillian/types"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	ktpb "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto"
	spb "github.com/google/keytransparency/core/sequencer/sequencer_go_proto"
	tpb "github.com/google/trillian"
	tclient "github.com/google/trillian/client"
)

const (
	domainIDLabel = "domainid"
	reasonLabel   = "reason"
)

var (
	once             sync.Once
	knownDomains     monitoring.Gauge
	batchSize        monitoring.Gauge
	mutationCount    monitoring.Counter
	mutationFailures monitoring.Counter
)

func createMetrics(mf monitoring.MetricFactory) {
	knownDomains = mf.NewGauge(
		"known_domains",
		"Set to 1 for known domains (whether this instance is master or not)",
		domainIDLabel)
	mutationCount = mf.NewCounter(
		"mutation_count",
		"Number of mutations the signer has processed for domainid since process start",
		domainIDLabel)
	mutationFailures = mf.NewCounter(
		"mutation_failures",
		"Number of invalid mutations the signer has processed for domainid since process start",
		domainIDLabel, reasonLabel)
	batchSize = mf.NewGauge(
		"batch_size",
		"Number of mutations the signer is attempting to process for domainid",
		domainIDLabel)
}

// Server implements KeyTransparencySequencerServer.
type Server struct {
	ktServer  *keyserver.Server
	mutations mutator.MutationStorage
	tmap      tpb.TrillianMapClient
	tlog      tpb.TrillianLogClient
}

// NewServer creates a new KeyTransparencySequencerServer.
func NewServer(
	domains domain.Storage,
	logAdmin tpb.TrillianAdminClient,
	mapAdmin tpb.TrillianAdminClient,
	tlog tpb.TrillianLogClient,
	tmap tpb.TrillianMapClient,
	mutations mutator.MutationStorage,
	metricsFactory monitoring.MetricFactory,
) *Server {
	once.Do(func() { createMetrics(metricsFactory) })
	return &Server{
		ktServer:  keyserver.New(nil, nil, logAdmin, mapAdmin, nil, domains, nil, nil),
		tlog:      tlog,
		tmap:      tmap,
		mutations: mutations,
	}
}

// CreateEpoch applies the supplied mutations to the current map revision and creates a new epoch.
func (s *Server) CreateEpoch(ctx context.Context, in *spb.CreateEpochRequest) (*empty.Empty, error) {
	domainID := in.GetDomainId()
	msgs := in.GetMessages()
	glog.Infof("CreateEpoch: for %v with %d messages", domainID, len(msgs))
	// Fetch verification objects for domainID.
	config, err := s.ktServer.GetDomain(ctx, &ktpb.GetDomainRequest{DomainId: domainID})
	if err != nil {
		return nil, err
	}
	mapClient, err := tclient.NewMapClientFromTree(s.tmap, config.Map)
	if err != nil {
		return nil, err
	}

	// Parse mutations using the mutator for this domain.
	batchSize.Set(float64(len(msgs)), config.DomainId)
	indexes := make([][]byte, 0, len(msgs))
	for _, m := range msgs {
		indexes = append(indexes, m.GetMutation().GetIndex())
	}
	glog.V(2).Infof("CreateEpoch: %v mutations, %v indexes", len(msgs), len(indexes))

	leaves, err := mapClient.GetAndVerifyMapLeaves(ctx, indexes)
	if err != nil {
		return nil, err
	}

	// Apply mutations to values.
	newLeaves, err := s.applyMutations(domainID, entry.New(), in.GetMessages(), leaves)
	if err != nil {
		return nil, err
	}

	// Set new leaf values.
	setResp, err := s.tmap.SetLeaves(ctx, &tpb.SetMapLeavesRequest{
		MapId:  config.Map.TreeId,
		Leaves: newLeaves,
	})
	if err != nil {
		return nil, status.Errorf(codes.Internal, "tmap.SetLeaves(): %v", err)
	}
	mapRoot, err := mapClient.VerifySignedMapRoot(setResp.GetMapRoot())
	if err != nil {
		return nil, status.Errorf(codes.Internal, "VerifySignedMapRoot(): %v", err)
	}
	glog.V(2).Infof("CreateEpoch: SetLeaves:{Revision: %v}", mapRoot.Revision)

	// Write mutations associated with this epoch.
	mutations := make([]*ktpb.Entry, 0, len(msgs))
	for _, msg := range msgs {
		mutations = append(mutations, msg.Mutation)
	}
	if err := s.mutations.WriteBatch(ctx, domainID, int64(mapRoot.Revision), mutations); err != nil {
		glog.Errorf("Could not write mutations for revision %v: %v", mapRoot.Revision, err)
		return nil, status.Errorf(codes.Internal, "mutations.WriteBatch(): %v", err)
	}

	// TODO(gbelvin): Store and track trustedRoot.
	trustedRoot := types.LogRootV1{} // Automatically trust the first observed log root.

	// Put SignedMapHead in the append only log.
	logClient, err := tclient.NewFromTree(s.tlog, config.Log, trustedRoot)
	if err != nil {
		return nil, err
	}
	if err := logClient.AddSequencedLeafAndWait(ctx, setResp.GetMapRoot().GetMapRoot(), int64(mapRoot.Revision)); err != nil {
		glog.Fatalf("AddSequencedLeaf(logID: %v, rev: %v): %v", config.Log.TreeId, mapRoot.Revision, err)
		// TODO(gdbelvin): Implement retries.
		return nil, err
	}

	mutationCount.Add(float64(len(msgs)), domainID)
	glog.Infof("CreatedEpoch: rev: %v with %v mutations, root: %x", mapRoot.Revision, len(msgs), mapRoot.RootHash)
	return &empty.Empty{}, nil
}

// applyMutations takes the set of mutations and applies them to given leafs.
// Multiple mutations for the same leaf will be applied to provided leaf.
// The last valid mutation for each leaf is included in the output.
// Returns a list of map leaves that should be updated.
func (s *Server) applyMutations(domainID string, mutatorFunc mutator.Func,
	msgs []*ktpb.EntryUpdate, leaves []*tpb.MapLeaf) ([]*tpb.MapLeaf, error) {
	// Put leaves in a map from index to leaf value.
	leafMap := make(map[string]*tpb.MapLeaf)
	for _, l := range leaves {
		leafMap[string(l.Index)] = l
	}

	retMap := make(map[string]*tpb.MapLeaf)
	for _, msg := range msgs {
		index := msg.Mutation.GetIndex()
		var oldValue *ktpb.Entry // If no map leaf was found, oldValue will be nil.
		if leaf, ok := leafMap[string(index)]; ok {
			var err error
			oldValue, err = entry.FromLeafValue(leaf.GetLeafValue())
			if err != nil {
				glog.Warningf("entry.FromLeafValue(%v): %v", leaf.GetLeafValue(), err)
				mutationFailures.Inc(domainID, "Unmarshal")
				continue
			}
		}

		newValue, err := mutatorFunc.Mutate(oldValue, msg.Mutation)
		if err != nil {
			glog.Warningf("Mutate(): %v", err)
			mutationFailures.Inc(domainID, "Mutate")
			continue // A bad mutation should not make the whole batch fail.
		}
		leafValue, err := entry.ToLeafValue(newValue)
		if err != nil {
			glog.Warningf("ToLeafValue(): %v", err)
			mutationFailures.Inc(domainID, "Marshal")
			continue
		}
		extraData, err := proto.Marshal(msg.Committed)
		if err != nil {
			glog.Warningf("proto.Marshal(): %v", err)
			mutationFailures.Inc(domainID, "Marshal")
			continue
		}

		// Make sure that only ONE MapLeaf is output per index.
		retMap[string(index)] = &tpb.MapLeaf{
			Index:     index,
			LeafValue: leafValue,
			ExtraData: extraData,
		}
	}
	// Convert return map back into a list.
	ret := make([]*tpb.MapLeaf, 0, len(retMap))
	for _, v := range retMap {
		ret = append(ret, v)
	}
	glog.V(2).Infof("applyMutations applied %v mutations to %v leaves", len(msgs), len(leaves))
	return ret, nil
}
<|endoftext|>


<commit_before>package termite

import (
	"fmt"
	"github.com/hanwen/go-fuse/fuse"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
)

var _ = fmt.Println

type FsServer struct {
	contentServer *ContentServer
	contentCache  *ContentCache
	Root          string
	excluded      map[string]bool

	multiplyPaths func(string)[]string

	hashCacheMutex sync.RWMutex
	// TODO - should use string (immutable) throughout for storing MD5 signatures.
	hashCache map[string][]byte

	attrCacheMutex sync.RWMutex
	attrCache map[string]FileAttr
}

func NewFsServer(root string, cache *ContentCache, excluded []string) *FsServer {
	fs := &FsServer{
		contentCache:  cache,
		contentServer: &ContentServer{Cache: cache},
		Root:          root,
		hashCache:     make(map[string][]byte),
		attrCache:     make(map[string]FileAttr),
	}

	fs.excluded = make(map[string]bool)
	for _, e := range excluded {
		fs.excluded[e] = true
	}
	return fs
}

type AttrRequest struct {
	Name string
}

type FileAttr struct {
	Path string
	*os.FileInfo
	fuse.Status
	Hash    []byte
	Link    string
	Content []byte // optional.
}

type AttrResponse struct {
	Attrs []FileAttr
}

func (me FileAttr) String() string {
	id := ""
	if me.Hash != nil {
		id = fmt.Sprintf(" sz %d", me.FileInfo.Size)
	}
	if me.Link != "" {
		id = fmt.Sprintf(" -> %s", me.Link)
	}
	if me.Deletion() {
		id = " (del)"
	}
	return fmt.Sprintf("%s%s", me.Path, id)
}

func (me FileAttr) Deletion() bool {
	return me.Status == fuse.ENOENT
}

type DirRequest struct {
	Name string
}

type DirResponse struct {
	NameModeMap map[string]uint32
}

func (me *FsServer) path(n string) string {
	if me.Root == "" {
		return n
	}
	return filepath.Join(me.Root, strings.TrimLeft(n, "/"))
}

func (me *FsServer) FileContent(req *ContentRequest, rep *ContentResponse) os.Error {
	return me.contentServer.FileContent(req, rep)
}

func (me *FsServer) ReadDir(req *DirRequest, r *DirResponse) os.Error {
	d, e := ioutil.ReadDir(me.path(req.Name))
	log.Println("ReadDir", req)
	r.NameModeMap = make(map[string]uint32)
	for _, v := range d {
		r.NameModeMap[v.Name] = v.Mode
	}
	return e
}

func (me *FsServer) GetAttr(req *AttrRequest, rep *AttrResponse) os.Error {
	log.Println("GetAttr req", req.Name)
	names := []string{}
	if me.multiplyPaths != nil {
		names = me.multiplyPaths(req.Name)
	} else {
		names = append(names, req.Name)
	}
	for _, n := range names {
		a := FileAttr{}
		err := me.oneGetAttr(n, &a)
		if err != nil {
			return err
		}
		if a.Hash != nil {
			log.Printf("GetAttr %s %v %x", n, a, a.Hash)
		}
		rep.Attrs = append(rep.Attrs, a)
	}
	return nil
}

func (me *FsServer) oneGetAttr(name string, rep *FileAttr) os.Error {
	rep.Path = name
	// TODO - this is not a good security measure, as we are not
	// checking the prefix; someone might directly ask for
	// /forbidden/subdir/
	if me.excluded[name] {
		rep.Status = fuse.ENOENT
		return nil
	}

	me.attrCacheMutex.RLock()
	attr, ok := me.attrCache[name]
	me.attrCacheMutex.RUnlock()

	if ok {
		*rep = attr
		return nil
	}
	me.attrCacheMutex.Lock()
	defer me.attrCacheMutex.Unlock()
	attr, ok = me.attrCache[name]
	if ok {
		*rep = attr
		return nil
	}

	fi, err := os.Lstat(me.path(name))
	rep.FileInfo = fi
	rep.Status = fuse.OsErrorToErrno(err)
	rep.Path = name
	if fi != nil {
		me.fillContent(rep)
	}

	me.attrCache[name] = *rep
	return nil
}

func (me *FsServer) fillContent(rep *FileAttr) {
	if rep.FileInfo.IsSymlink() {
		rep.Link, _ = os.Readlink(rep.Path)
	}
	if rep.FileInfo.IsRegular() {
		rep.Hash, rep.Content = me.getHash(rep.Path)
	}
}

func (me *FsServer) updateFiles(infos []FileAttr) {
	me.updateHashes(infos)
	me.updateAttrs(infos)
}

func (me *FsServer) updateAttrs(infos []FileAttr) {
	me.attrCacheMutex.Lock()
	defer me.attrCacheMutex.Unlock()

	for _, r := range infos {
		name := r.Path
		me.attrCache[name] = r
	}
}

func (me *FsServer) updateHashes(infos []FileAttr) {
	me.hashCacheMutex.Lock()
	defer me.hashCacheMutex.Unlock()

	for _, r := range infos {
		name := r.Path
		if !r.Status.Ok() || r.Link != "" {
			me.hashCache[name] = nil, false
		}
		if r.Hash != nil {
			me.hashCache[name] = r.Hash
		}
	}
}

func (me *FsServer) getHash(name string) (hash []byte, content []byte) {
	fullPath := me.path(name)

	me.hashCacheMutex.RLock()
	hash = me.hashCache[name]
	me.hashCacheMutex.RUnlock()

	if hash != nil {
		return []byte(hash), nil
	}

	me.hashCacheMutex.Lock()
	defer me.hashCacheMutex.Unlock()
	hash = me.hashCache[name]
	if hash != nil {
		return []byte(hash), nil
	}

	// TODO - would it be better to not stop other hash lookups
	// from succeeding?

	// TODO - /usr should be configurable.
	if strings.HasPrefix(fullPath, "/usr") {
		hash, content = me.contentCache.SaveImmutablePath(fullPath)
	} else {
		hash, content = me.contentCache.SavePath(fullPath)
	}

	me.hashCache[name] = hash
	return hash, content
}

// TODO - decide between []FileAttr and []*FileAttr.
func (me *FsServer) refreshAttributeCache(prefix string) []FileAttr {
	me.attrCacheMutex.Lock()
	defer me.attrCacheMutex.Unlock()

	updated := []FileAttr{}
	entries := ListFilesRecursively(prefix)
	for key, attr := range me.attrCache {
		if HasDirPrefix(key, prefix) {
			_, ok := entries[key]
			if !ok && attr.Status.Ok() {
				del := FileAttr{
				Path: key,
				Status: fuse.ENOENT,
				}

				updated = append(updated, del)
			}
		}
	}

	for name, e := range entries {
		attr, ok := me.attrCache[name]
		newFi := e
		if ok && EncodeFileInfo(*attr.FileInfo) == EncodeFileInfo(e) {
			continue
		}
		newEnt := FileAttr{
		Path: name,
		Status: fuse.OK,
		FileInfo: &newFi,
		}
		me.fillContent(&newEnt)
		updated = append(updated, newEnt)
	}

	for _, u := range updated {
		me.attrCache[u.Path] = u
	}
	return updated
}
<commit_msg>Do not deref nil pointer.<commit_after>package termite

import (
	"fmt"
	"github.com/hanwen/go-fuse/fuse"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
)

var _ = fmt.Println

type FsServer struct {
	contentServer *ContentServer
	contentCache  *ContentCache
	Root          string
	excluded      map[string]bool

	multiplyPaths func(string)[]string

	hashCacheMutex sync.RWMutex
	// TODO - should use string (immutable) throughout for storing MD5 signatures.
	hashCache map[string][]byte

	attrCacheMutex sync.RWMutex
	attrCache map[string]FileAttr
}

func NewFsServer(root string, cache *ContentCache, excluded []string) *FsServer {
	fs := &FsServer{
		contentCache:  cache,
		contentServer: &ContentServer{Cache: cache},
		Root:          root,
		hashCache:     make(map[string][]byte),
		attrCache:     make(map[string]FileAttr),
	}

	fs.excluded = make(map[string]bool)
	for _, e := range excluded {
		fs.excluded[e] = true
	}
	return fs
}

type AttrRequest struct {
	Name string
}

type FileAttr struct {
	Path string
	*os.FileInfo
	fuse.Status
	Hash    []byte
	Link    string
	Content []byte // optional.
}

type AttrResponse struct {
	Attrs []FileAttr
}

func (me FileAttr) String() string {
	id := ""
	if me.Hash != nil {
		id = fmt.Sprintf(" sz %d", me.FileInfo.Size)
	}
	if me.Link != "" {
		id = fmt.Sprintf(" -> %s", me.Link)
	}
	if me.Deletion() {
		id = " (del)"
	}
	return fmt.Sprintf("%s%s", me.Path, id)
}

func (me FileAttr) Deletion() bool {
	return me.Status == fuse.ENOENT
}

type DirRequest struct {
	Name string
}

type DirResponse struct {
	NameModeMap map[string]uint32
}

func (me *FsServer) path(n string) string {
	if me.Root == "" {
		return n
	}
	return filepath.Join(me.Root, strings.TrimLeft(n, "/"))
}

func (me *FsServer) FileContent(req *ContentRequest, rep *ContentResponse) os.Error {
	return me.contentServer.FileContent(req, rep)
}

func (me *FsServer) ReadDir(req *DirRequest, r *DirResponse) os.Error {
	d, e := ioutil.ReadDir(me.path(req.Name))
	log.Println("ReadDir", req)
	r.NameModeMap = make(map[string]uint32)
	for _, v := range d {
		r.NameModeMap[v.Name] = v.Mode
	}
	return e
}

func (me *FsServer) GetAttr(req *AttrRequest, rep *AttrResponse) os.Error {
	log.Println("GetAttr req", req.Name)
	names := []string{}
	if me.multiplyPaths != nil {
		names = me.multiplyPaths(req.Name)
	} else {
		names = append(names, req.Name)
	}
	for _, n := range names {
		a := FileAttr{}
		err := me.oneGetAttr(n, &a)
		if err != nil {
			return err
		}
		if a.Hash != nil {
			log.Printf("GetAttr %s %v %x", n, a, a.Hash)
		}
		rep.Attrs = append(rep.Attrs, a)
	}
	return nil
}

func (me *FsServer) oneGetAttr(name string, rep *FileAttr) os.Error {
	rep.Path = name
	// TODO - this is not a good security measure, as we are not
	// checking the prefix; someone might directly ask for
	// /forbidden/subdir/
	if me.excluded[name] {
		rep.Status = fuse.ENOENT
		return nil
	}

	me.attrCacheMutex.RLock()
	attr, ok := me.attrCache[name]
	me.attrCacheMutex.RUnlock()

	if ok {
		*rep = attr
		return nil
	}
	me.attrCacheMutex.Lock()
	defer me.attrCacheMutex.Unlock()
	attr, ok = me.attrCache[name]
	if ok {
		*rep = attr
		return nil
	}

	fi, err := os.Lstat(me.path(name))
	rep.FileInfo = fi
	rep.Status = fuse.OsErrorToErrno(err)
	rep.Path = name
	if fi != nil {
		me.fillContent(rep)
	}

	me.attrCache[name] = *rep
	return nil
}

func (me *FsServer) fillContent(rep *FileAttr) {
	if rep.FileInfo.IsSymlink() {
		rep.Link, _ = os.Readlink(rep.Path)
	}
	if rep.FileInfo.IsRegular() {
		// TODO - saving the content easily overflows memory
		// on 32-bit.
		rep.Hash, _ = me.getHash(rep.Path)
	}
}

func (me *FsServer) updateFiles(infos []FileAttr) {
	me.updateHashes(infos)
	me.updateAttrs(infos)
}

func (me *FsServer) updateAttrs(infos []FileAttr) {
	me.attrCacheMutex.Lock()
	defer me.attrCacheMutex.Unlock()

	for _, r := range infos {
		name := r.Path
		me.attrCache[name] = r
	}
}

func (me *FsServer) updateHashes(infos []FileAttr) {
	me.hashCacheMutex.Lock()
	defer me.hashCacheMutex.Unlock()

	for _, r := range infos {
		name := r.Path
		if !r.Status.Ok() || r.Link != "" {
			me.hashCache[name] = nil, false
		}
		if r.Hash != nil {
			me.hashCache[name] = r.Hash
		}
	}
}

func (me *FsServer) getHash(name string) (hash []byte, content []byte) {
	fullPath := me.path(name)

	me.hashCacheMutex.RLock()
	hash = me.hashCache[name]
	me.hashCacheMutex.RUnlock()

	if hash != nil {
		return []byte(hash), nil
	}

	me.hashCacheMutex.Lock()
	defer me.hashCacheMutex.Unlock()
	hash = me.hashCache[name]
	if hash != nil {
		return []byte(hash), nil
	}

	// TODO - would it be better to not stop other hash lookups
	// from succeeding?

	// TODO - /usr should be configurable.
	if strings.HasPrefix(fullPath, "/usr") {
		hash, content = me.contentCache.SaveImmutablePath(fullPath)
	} else {
		hash, content = me.contentCache.SavePath(fullPath)
	}

	me.hashCache[name] = hash
	return hash, content
}

// TODO - decide between []FileAttr and []*FileAttr.
func (me *FsServer) refreshAttributeCache(prefix string) []FileAttr {
	me.attrCacheMutex.Lock()
	defer me.attrCacheMutex.Unlock()

	updated := []FileAttr{}
	entries := ListFilesRecursively(prefix)
	for key, attr := range me.attrCache {
		if HasDirPrefix(key, prefix) {
			_, ok := entries[key]
			if !ok && attr.Status.Ok() {
				del := FileAttr{
				Path: key,
				Status: fuse.ENOENT,
				}

				updated = append(updated, del)
			}
		}
	}

	for name, e := range entries {
		attr, ok := me.attrCache[name]
		newFi := e
		if ok && attr.FileInfo != nil && EncodeFileInfo(*attr.FileInfo) == EncodeFileInfo(e) {
			continue
		}
		newEnt := FileAttr{
		Path: name,
		Status: fuse.OK,
		FileInfo: &newFi,
		}
		me.fillContent(&newEnt)
		updated = append(updated, newEnt)
	}

	for _, u := range updated {
		me.attrCache[u.Path] = u
	}
	return updated
}
<|endoftext|>


<commit_before>package test

type Match struct {
	Input string
	InputMatch string
	Response string
	Score float64
}

type ByScore []Match

func (s ByScore) Len() int {
    return len(s)
}
func (s ByScore) Swap(i, j int) {
    s[i], s[j] = s[j], s[i]
}
func (s ByScore) Less(i, j int) bool {
    return s[i].Score < s[j].Score
}
<commit_msg>Reverse the sorting of the simish test output<commit_after>package test

type Match struct {
	Input string
	InputMatch string
	Response string
	Score float64
}

type ByScore []Match

func (s ByScore) Len() int {
    return len(s)
}
func (s ByScore) Swap(i, j int) {
    s[i], s[j] = s[j], s[i]
}
func (s ByScore) Less(i, j int) bool {
    return s[i].Score > s[j].Score
}
<|endoftext|>