<commit_before>package gateway

import (
	"errors"
	"sync"
	"time"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
)

type rpcID [8]byte

func (id rpcID) String() string {
	for i := range id {
		if id[i] == 0 {
			id[i] = ' '
		}
	}
	return string(id[:])
}

// handlerName truncates a string to 8 bytes. If len(name) < 8, the remaining
// bytes are 0. A handlerName is specified at the beginning of each network
// call, indicating which function should handle the connection.
func handlerName(name string) (id rpcID) {
	copy(id[:], name)
	return
}

// RPC calls an RPC on the given address. RPC cannot be called on an address
// that the Gateway is not connected to.
func (g *Gateway) RPC(addr modules.NetAddress, name string, fn modules.RPCFunc) error {
	id := g.mu.RLock()
	peer, ok := g.peers[addr]
	g.mu.RUnlock(id)
	if !ok {
		return errors.New("can't call RPC on unconnected peer " + string(addr))
	}

	conn, err := peer.open()
	if err != nil {
		return err
	}
	defer conn.Close()

	// write header
	if err := encoding.WriteObject(conn, handlerName(name)); err != nil {
		return err
	}
	// call fn
	return fn(conn)
}

// RegisterRPC registers an RPCFunc as a handler for a given identifier. To
// call an RPC, use gateway.RPC, supplying the same identifier given to
// RegisterRPC. Identifiers should always use PascalCase. The first 8
// characters of an identifier should be unique, as the identifier used
// internally is truncated to 8 bytes.
func (g *Gateway) RegisterRPC(name string, fn modules.RPCFunc) {
	id := g.mu.Lock()
	defer g.mu.Unlock(id)
	if build.DEBUG {
		if _, ok := g.handlers[handlerName(name)]; ok {
			panic("refusing to overwrite RPC " + name)
		}
	}
	g.handlers[handlerName(name)] = fn
}

// UnregisterRPC unregisters an RPC and removes the corresponding RPCFunc from
// g.handlers. Future calls to the RPC by peers will fail.
func (g *Gateway) UnregisterRPC(name string) {
	id := g.mu.Lock()
	defer g.mu.Unlock(id)
	if build.DEBUG {
		if _, ok := g.handlers[handlerName(name)]; !ok {
			panic("RPC is not registered: " + name)
		}
	}
	delete(g.handlers, handlerName(name))
}

// RegisterConnectCall registers a name and RPCFunc to be called on a peer
// upon connecting.
func (g *Gateway) RegisterConnectCall(name string, fn modules.RPCFunc) {
	id := g.mu.Lock()
	defer g.mu.Unlock(id)
	if build.DEBUG {
		if _, ok := g.initRPCs[name]; ok {
			panic("refusing to overwrite RPC " + name)
		}
	}
	g.initRPCs[name] = fn
}

// UnregisterConnectCall unregisters an on-connect call and removes the
// corresponding RPCFunc from g.initRPCs. Future connections to peers will not
// trigger the RPC to be called on them.
func (g *Gateway) UnregisterConnectCall(name string) {
	id := g.mu.Lock()
	defer g.mu.Unlock(id)
	if build.DEBUG {
		if _, ok := g.initRPCs[name]; !ok {
			panic("ConnectCall is not registered: " + name)
		}
	}
	delete(g.initRPCs, name)
}

// listenPeer listens for new streams on a peer connection and serves them via
// threadedHandleConn.
func (g *Gateway) listenPeer(p *peer) {
	for {
		conn, err := p.accept()
		if err != nil {
			g.log.Println("WARN: lost connection to peer", p.NetAddress)
			break
		}

		// it is the handler's responsibility to close the connection
		go g.threadedHandleConn(conn)
	}
	g.Disconnect(p.NetAddress)
}

// threadedHandleConn reads header data from a connection, then routes it to the
// appropriate handler for further processing.
func (g *Gateway) threadedHandleConn(conn modules.PeerConn) {
	defer conn.Close()
	var id rpcID
	if err := encoding.ReadObject(conn, &id, 8); err != nil {
		return
	}
	// call registered handler for this ID
	lockid := g.mu.RLock()
	fn, ok := g.handlers[id]
	g.mu.RUnlock(lockid)
	if !ok {
		g.log.Printf("WARN: incoming conn %v requested unknown RPC \"%v\"", conn.RemoteAddr(), id)
		return
	}
	if build.DEBUG {
		g.log.Printf("INFO: incoming conn %v requested RPC \"%v\"", conn.RemoteAddr(), id)
	}

	// call fn
	err := fn(conn)
	// don't log benign errors
	if err == modules.ErrDuplicateTransactionSet || err == modules.ErrBlockKnown {
		err = nil
	}
	if err != nil {
		g.log.Printf("WARN: incoming RPC \"%v\" from conn %v failed: %v", id, conn.RemoteAddr(), err)
	}
}

// Broadcast calls an RPC on all of the specified peers. The calls are run in
// parallel. Broadcasts are restricted to "one-way" RPCs, which simply write an
// object and disconnect. This is why Broadcast takes an interface{} instead of
// an RPCFunc.
func (g *Gateway) Broadcast(name string, obj interface{}, peers []modules.Peer) {
	g.log.Printf("INFO: broadcasting RPC \"%v\" to %v peers", name, len(peers))

	// only encode obj once, instead of using WriteObject
	enc := encoding.Marshal(obj)
	fn := func(conn modules.PeerConn) error {
		return encoding.WritePrefix(conn, enc)
	}

	var wg sync.WaitGroup
	wg.Add(len(peers))
	for _, p := range peers {
		go func(addr modules.NetAddress) {
			err := g.RPC(addr, name, fn)
			if err != nil {
				// try one more time before giving up
				time.Sleep(10 * time.Second)
				g.RPC(addr, name, fn)
			}
			wg.Done()
		}(p.NetAddress)
	}
	wg.Wait()
}
<commit_msg>Replace panics with build.Criticals in {Unr,R}egister{RPC,ConnectCall}<commit_after>package gateway

import (
	"errors"
	"sync"
	"time"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
)

type rpcID [8]byte

func (id rpcID) String() string {
	for i := range id {
		if id[i] == 0 {
			id[i] = ' '
		}
	}
	return string(id[:])
}

// handlerName truncates a string to 8 bytes. If len(name) < 8, the remaining
// bytes are 0. A handlerName is specified at the beginning of each network
// call, indicating which function should handle the connection.
func handlerName(name string) (id rpcID) {
	copy(id[:], name)
	return
}

// RPC calls an RPC on the given address. RPC cannot be called on an address
// that the Gateway is not connected to.
func (g *Gateway) RPC(addr modules.NetAddress, name string, fn modules.RPCFunc) error {
	id := g.mu.RLock()
	peer, ok := g.peers[addr]
	g.mu.RUnlock(id)
	if !ok {
		return errors.New("can't call RPC on unconnected peer " + string(addr))
	}

	conn, err := peer.open()
	if err != nil {
		return err
	}
	defer conn.Close()

	// write header
	if err := encoding.WriteObject(conn, handlerName(name)); err != nil {
		return err
	}
	// call fn
	return fn(conn)
}

// RegisterRPC registers an RPCFunc as a handler for a given identifier. To
// call an RPC, use gateway.RPC, supplying the same identifier given to
// RegisterRPC. Identifiers should always use PascalCase. The first 8
// characters of an identifier should be unique, as the identifier used
// internally is truncated to 8 bytes.
func (g *Gateway) RegisterRPC(name string, fn modules.RPCFunc) {
	id := g.mu.Lock()
	defer g.mu.Unlock(id)
	if _, ok := g.handlers[handlerName(name)]; ok {
		build.Critical("RPC already registered: " + name)
	}
	g.handlers[handlerName(name)] = fn
}

// UnregisterRPC unregisters an RPC and removes the corresponding RPCFunc from
// g.handlers. Future calls to the RPC by peers will fail.
func (g *Gateway) UnregisterRPC(name string) {
	id := g.mu.Lock()
	defer g.mu.Unlock(id)
	if _, ok := g.handlers[handlerName(name)]; !ok {
		build.Critical("RPC not registered: " + name)
	}
	delete(g.handlers, handlerName(name))
}

// RegisterConnectCall registers a name and RPCFunc to be called on a peer
// upon connecting.
func (g *Gateway) RegisterConnectCall(name string, fn modules.RPCFunc) {
	id := g.mu.Lock()
	defer g.mu.Unlock(id)
	if _, ok := g.initRPCs[name]; ok {
		build.Critical("ConnectCall already registered: " + name)
	}
	g.initRPCs[name] = fn
}

// UnregisterConnectCall unregisters an on-connect call and removes the
// corresponding RPCFunc from g.initRPCs. Future connections to peers will not
// trigger the RPC to be called on them.
func (g *Gateway) UnregisterConnectCall(name string) {
	id := g.mu.Lock()
	defer g.mu.Unlock(id)
	if _, ok := g.initRPCs[name]; !ok {
		build.Critical("ConnectCall not registered: " + name)
	}
	delete(g.initRPCs, name)
}

// listenPeer listens for new streams on a peer connection and serves them via
// threadedHandleConn.
func (g *Gateway) listenPeer(p *peer) {
	for {
		conn, err := p.accept()
		if err != nil {
			g.log.Println("WARN: lost connection to peer", p.NetAddress)
			break
		}

		// it is the handler's responsibility to close the connection
		go g.threadedHandleConn(conn)
	}
	g.Disconnect(p.NetAddress)
}

// threadedHandleConn reads header data from a connection, then routes it to the
// appropriate handler for further processing.
func (g *Gateway) threadedHandleConn(conn modules.PeerConn) {
	defer conn.Close()
	var id rpcID
	if err := encoding.ReadObject(conn, &id, 8); err != nil {
		return
	}
	// call registered handler for this ID
	lockid := g.mu.RLock()
	fn, ok := g.handlers[id]
	g.mu.RUnlock(lockid)
	if !ok {
		g.log.Printf("WARN: incoming conn %v requested unknown RPC \"%v\"", conn.RemoteAddr(), id)
		return
	}
	if build.DEBUG {
		g.log.Printf("INFO: incoming conn %v requested RPC \"%v\"", conn.RemoteAddr(), id)
	}

	// call fn
	err := fn(conn)
	// don't log benign errors
	if err == modules.ErrDuplicateTransactionSet || err == modules.ErrBlockKnown {
		err = nil
	}
	if err != nil {
		g.log.Printf("WARN: incoming RPC \"%v\" from conn %v failed: %v", id, conn.RemoteAddr(), err)
	}
}

// Broadcast calls an RPC on all of the specified peers. The calls are run in
// parallel. Broadcasts are restricted to "one-way" RPCs, which simply write an
// object and disconnect. This is why Broadcast takes an interface{} instead of
// an RPCFunc.
func (g *Gateway) Broadcast(name string, obj interface{}, peers []modules.Peer) {
	g.log.Printf("INFO: broadcasting RPC \"%v\" to %v peers", name, len(peers))

	// only encode obj once, instead of using WriteObject
	enc := encoding.Marshal(obj)
	fn := func(conn modules.PeerConn) error {
		return encoding.WritePrefix(conn, enc)
	}

	var wg sync.WaitGroup
	wg.Add(len(peers))
	for _, p := range peers {
		go func(addr modules.NetAddress) {
			err := g.RPC(addr, name, fn)
			if err != nil {
				// try one more time before giving up
				time.Sleep(10 * time.Second)
				g.RPC(addr, name, fn)
			}
			wg.Done()
		}(p.NetAddress)
	}
	wg.Wait()
}
<|endoftext|>


<commit_before>// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2016 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package git

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strings"
	"time"

	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/process"
)

var (
	// GlobalCommandArgs global command args for external package setting
	GlobalCommandArgs []string

	// defaultCommandExecutionTimeout default command execution timeout duration
	defaultCommandExecutionTimeout = 360 * time.Second
)

// DefaultLocale is the default LC_ALL to run git commands in.
const DefaultLocale = "C"

// Command represents a command with its subcommands or arguments.
type Command struct {
	name          string
	args          []string
	parentContext context.Context
	desc          string
}

func (c *Command) String() string {
	if len(c.args) == 0 {
		return c.name
	}
	return fmt.Sprintf("%s %s", c.name, strings.Join(c.args, " "))
}

// NewCommand creates and returns a new Git Command based on given command and arguments.
func NewCommand(args ...string) *Command {
	return NewCommandContext(DefaultContext, args...)
}

// NewCommandContext creates and returns a new Git Command based on given command and arguments.
func NewCommandContext(ctx context.Context, args ...string) *Command {
	// Make an explicit copy of GlobalCommandArgs, otherwise append might overwrite it
	cargs := make([]string, len(GlobalCommandArgs))
	copy(cargs, GlobalCommandArgs)
	return &Command{
		name:          GitExecutable,
		args:          append(cargs, args...),
		parentContext: ctx,
	}
}

// NewCommandNoGlobals creates and returns a new Git Command based on given command and arguments only with the specify args and don't care global command args
func NewCommandNoGlobals(args ...string) *Command {
	return NewCommandContextNoGlobals(DefaultContext, args...)
}

// NewCommandContextNoGlobals creates and returns a new Git Command based on given command and arguments only with the specify args and don't care global command args
func NewCommandContextNoGlobals(ctx context.Context, args ...string) *Command {
	return &Command{
		name:          GitExecutable,
		args:          args,
		parentContext: ctx,
	}
}

// SetParentContext sets the parent context for this command
func (c *Command) SetParentContext(ctx context.Context) *Command {
	c.parentContext = ctx
	return c
}

// SetDescription sets the description for this command which be returned on
// c.String()
func (c *Command) SetDescription(desc string) *Command {
	c.desc = desc
	return c
}

// AddArguments adds new argument(s) to the command.
func (c *Command) AddArguments(args ...string) *Command {
	c.args = append(c.args, args...)
	return c
}

// RunInDirTimeoutEnvPipeline executes the command in given directory with given timeout,
// it pipes stdout and stderr to given io.Writer.
func (c *Command) RunInDirTimeoutEnvPipeline(env []string, timeout time.Duration, dir string, stdout, stderr io.Writer) error {
	return c.RunInDirTimeoutEnvFullPipeline(env, timeout, dir, stdout, stderr, nil)
}

// RunInDirTimeoutEnvFullPipeline executes the command in given directory with given timeout,
// it pipes stdout and stderr to given io.Writer and passes in an io.Reader as stdin.
func (c *Command) RunInDirTimeoutEnvFullPipeline(env []string, timeout time.Duration, dir string, stdout, stderr io.Writer, stdin io.Reader) error {
	return c.RunInDirTimeoutEnvFullPipelineFunc(env, timeout, dir, stdout, stderr, stdin, nil)
}

// RunInDirTimeoutEnvFullPipelineFunc executes the command in given directory with given timeout,
// it pipes stdout and stderr to given io.Writer and passes in an io.Reader as stdin. Between cmd.Start and cmd.Wait the passed in function is run.
func (c *Command) RunInDirTimeoutEnvFullPipelineFunc(env []string, timeout time.Duration, dir string, stdout, stderr io.Writer, stdin io.Reader, fn func(context.Context, context.CancelFunc) error) error {
	if timeout == -1 {
		timeout = defaultCommandExecutionTimeout
	}

	if len(dir) == 0 {
		log.Debug("%s", c)
	} else {
		log.Debug("%s: %v", dir, c)
	}

	ctx, cancel := context.WithTimeout(c.parentContext, timeout)
	defer cancel()

	cmd := exec.CommandContext(ctx, c.name, c.args...)
	if env == nil {
		cmd.Env = os.Environ()
	} else {
		cmd.Env = env
	}

	cmd.Env = append(
		cmd.Env,
		fmt.Sprintf("LC_ALL=%s", DefaultLocale),
		// avoid prompting for credentials interactively, supported since git v2.3
		"GIT_TERMINAL_PROMPT=0",
	)

	// TODO: verify if this is still needed in golang 1.15
	if goVersionLessThan115 {
		cmd.Env = append(cmd.Env, "GODEBUG=asyncpreemptoff=1")
	}
	cmd.Dir = dir
	cmd.Stdout = stdout
	cmd.Stderr = stderr
	cmd.Stdin = stdin
	if err := cmd.Start(); err != nil {
		return err
	}

	desc := c.desc
	if desc == "" {
		desc = fmt.Sprintf("%s %s %s [repo_path: %s]", GitExecutable, c.name, strings.Join(c.args, " "), dir)
	}
	pid := process.GetManager().Add(desc, cancel)
	defer process.GetManager().Remove(pid)

	if fn != nil {
		err := fn(ctx, cancel)
		if err != nil {
			cancel()
			_ = cmd.Wait()
			return err
		}
	}

	if err := cmd.Wait(); err != nil && ctx.Err() != context.DeadlineExceeded {
		return err
	}

	return ctx.Err()
}

// RunInDirTimeoutPipeline executes the command in given directory with given timeout,
// it pipes stdout and stderr to given io.Writer.
func (c *Command) RunInDirTimeoutPipeline(timeout time.Duration, dir string, stdout, stderr io.Writer) error {
	return c.RunInDirTimeoutEnvPipeline(nil, timeout, dir, stdout, stderr)
}

// RunInDirTimeoutFullPipeline executes the command in given directory with given timeout,
// it pipes stdout and stderr to given io.Writer, and stdin from the given io.Reader
func (c *Command) RunInDirTimeoutFullPipeline(timeout time.Duration, dir string, stdout, stderr io.Writer, stdin io.Reader) error {
	return c.RunInDirTimeoutEnvFullPipeline(nil, timeout, dir, stdout, stderr, stdin)
}

// RunInDirTimeout executes the command in given directory with given timeout,
// and returns stdout in []byte and error (combined with stderr).
func (c *Command) RunInDirTimeout(timeout time.Duration, dir string) ([]byte, error) {
	return c.RunInDirTimeoutEnv(nil, timeout, dir)
}

// RunInDirTimeoutEnv executes the command in given directory with given timeout,
// and returns stdout in []byte and error (combined with stderr).
func (c *Command) RunInDirTimeoutEnv(env []string, timeout time.Duration, dir string) ([]byte, error) {
	stdout := new(bytes.Buffer)
	stderr := new(bytes.Buffer)
	if err := c.RunInDirTimeoutEnvPipeline(env, timeout, dir, stdout, stderr); err != nil {
		return nil, ConcatenateError(err, stderr.String())
	}
	if stdout.Len() > 0 && log.IsTrace() {
		log.Trace("Stdout:\n %s", stdout.Bytes()[:1024])
	}
	return stdout.Bytes(), nil
}

// RunInDirPipeline executes the command in given directory,
// it pipes stdout and stderr to given io.Writer.
func (c *Command) RunInDirPipeline(dir string, stdout, stderr io.Writer) error {
	return c.RunInDirFullPipeline(dir, stdout, stderr, nil)
}

// RunInDirFullPipeline executes the command in given directory,
// it pipes stdout and stderr to given io.Writer.
func (c *Command) RunInDirFullPipeline(dir string, stdout, stderr io.Writer, stdin io.Reader) error {
	return c.RunInDirTimeoutFullPipeline(-1, dir, stdout, stderr, stdin)
}

// RunInDirBytes executes the command in given directory
// and returns stdout in []byte and error (combined with stderr).
func (c *Command) RunInDirBytes(dir string) ([]byte, error) {
	return c.RunInDirTimeout(-1, dir)
}

// RunInDir executes the command in given directory
// and returns stdout in string and error (combined with stderr).
func (c *Command) RunInDir(dir string) (string, error) {
	return c.RunInDirWithEnv(dir, nil)
}

// RunInDirWithEnv executes the command in given directory
// and returns stdout in string and error (combined with stderr).
func (c *Command) RunInDirWithEnv(dir string, env []string) (string, error) {
	stdout, err := c.RunInDirTimeoutEnv(env, -1, dir)
	if err != nil {
		return "", err
	}
	return string(stdout), nil
}

// RunTimeout executes the command in default working directory with given timeout,
// and returns stdout in string and error (combined with stderr).
func (c *Command) RunTimeout(timeout time.Duration) (string, error) {
	stdout, err := c.RunInDirTimeout(timeout, "")
	if err != nil {
		return "", err
	}
	return string(stdout), nil
}

// Run executes the command in default working directory
// and returns stdout in string and error (combined with stderr).
func (c *Command) Run() (string, error) {
	return c.RunTimeout(-1)
}
<commit_msg>Limit stdout tracelog to actual stdout (#16258)<commit_after>// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2016 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package git

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strings"
	"time"

	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/process"
)

var (
	// GlobalCommandArgs global command args for external package setting
	GlobalCommandArgs []string

	// defaultCommandExecutionTimeout default command execution timeout duration
	defaultCommandExecutionTimeout = 360 * time.Second
)

// DefaultLocale is the default LC_ALL to run git commands in.
const DefaultLocale = "C"

// Command represents a command with its subcommands or arguments.
type Command struct {
	name          string
	args          []string
	parentContext context.Context
	desc          string
}

func (c *Command) String() string {
	if len(c.args) == 0 {
		return c.name
	}
	return fmt.Sprintf("%s %s", c.name, strings.Join(c.args, " "))
}

// NewCommand creates and returns a new Git Command based on given command and arguments.
func NewCommand(args ...string) *Command {
	return NewCommandContext(DefaultContext, args...)
}

// NewCommandContext creates and returns a new Git Command based on given command and arguments.
func NewCommandContext(ctx context.Context, args ...string) *Command {
	// Make an explicit copy of GlobalCommandArgs, otherwise append might overwrite it
	cargs := make([]string, len(GlobalCommandArgs))
	copy(cargs, GlobalCommandArgs)
	return &Command{
		name:          GitExecutable,
		args:          append(cargs, args...),
		parentContext: ctx,
	}
}

// NewCommandNoGlobals creates and returns a new Git Command based on given command and arguments only with the specify args and don't care global command args
func NewCommandNoGlobals(args ...string) *Command {
	return NewCommandContextNoGlobals(DefaultContext, args...)
}

// NewCommandContextNoGlobals creates and returns a new Git Command based on given command and arguments only with the specify args and don't care global command args
func NewCommandContextNoGlobals(ctx context.Context, args ...string) *Command {
	return &Command{
		name:          GitExecutable,
		args:          args,
		parentContext: ctx,
	}
}

// SetParentContext sets the parent context for this command
func (c *Command) SetParentContext(ctx context.Context) *Command {
	c.parentContext = ctx
	return c
}

// SetDescription sets the description for this command which be returned on
// c.String()
func (c *Command) SetDescription(desc string) *Command {
	c.desc = desc
	return c
}

// AddArguments adds new argument(s) to the command.
func (c *Command) AddArguments(args ...string) *Command {
	c.args = append(c.args, args...)
	return c
}

// RunInDirTimeoutEnvPipeline executes the command in given directory with given timeout,
// it pipes stdout and stderr to given io.Writer.
func (c *Command) RunInDirTimeoutEnvPipeline(env []string, timeout time.Duration, dir string, stdout, stderr io.Writer) error {
	return c.RunInDirTimeoutEnvFullPipeline(env, timeout, dir, stdout, stderr, nil)
}

// RunInDirTimeoutEnvFullPipeline executes the command in given directory with given timeout,
// it pipes stdout and stderr to given io.Writer and passes in an io.Reader as stdin.
func (c *Command) RunInDirTimeoutEnvFullPipeline(env []string, timeout time.Duration, dir string, stdout, stderr io.Writer, stdin io.Reader) error {
	return c.RunInDirTimeoutEnvFullPipelineFunc(env, timeout, dir, stdout, stderr, stdin, nil)
}

// RunInDirTimeoutEnvFullPipelineFunc executes the command in given directory with given timeout,
// it pipes stdout and stderr to given io.Writer and passes in an io.Reader as stdin. Between cmd.Start and cmd.Wait the passed in function is run.
func (c *Command) RunInDirTimeoutEnvFullPipelineFunc(env []string, timeout time.Duration, dir string, stdout, stderr io.Writer, stdin io.Reader, fn func(context.Context, context.CancelFunc) error) error {
	if timeout == -1 {
		timeout = defaultCommandExecutionTimeout
	}

	if len(dir) == 0 {
		log.Debug("%s", c)
	} else {
		log.Debug("%s: %v", dir, c)
	}

	ctx, cancel := context.WithTimeout(c.parentContext, timeout)
	defer cancel()

	cmd := exec.CommandContext(ctx, c.name, c.args...)
	if env == nil {
		cmd.Env = os.Environ()
	} else {
		cmd.Env = env
	}

	cmd.Env = append(
		cmd.Env,
		fmt.Sprintf("LC_ALL=%s", DefaultLocale),
		// avoid prompting for credentials interactively, supported since git v2.3
		"GIT_TERMINAL_PROMPT=0",
	)

	// TODO: verify if this is still needed in golang 1.15
	if goVersionLessThan115 {
		cmd.Env = append(cmd.Env, "GODEBUG=asyncpreemptoff=1")
	}
	cmd.Dir = dir
	cmd.Stdout = stdout
	cmd.Stderr = stderr
	cmd.Stdin = stdin
	if err := cmd.Start(); err != nil {
		return err
	}

	desc := c.desc
	if desc == "" {
		desc = fmt.Sprintf("%s %s %s [repo_path: %s]", GitExecutable, c.name, strings.Join(c.args, " "), dir)
	}
	pid := process.GetManager().Add(desc, cancel)
	defer process.GetManager().Remove(pid)

	if fn != nil {
		err := fn(ctx, cancel)
		if err != nil {
			cancel()
			_ = cmd.Wait()
			return err
		}
	}

	if err := cmd.Wait(); err != nil && ctx.Err() != context.DeadlineExceeded {
		return err
	}

	return ctx.Err()
}

// RunInDirTimeoutPipeline executes the command in given directory with given timeout,
// it pipes stdout and stderr to given io.Writer.
func (c *Command) RunInDirTimeoutPipeline(timeout time.Duration, dir string, stdout, stderr io.Writer) error {
	return c.RunInDirTimeoutEnvPipeline(nil, timeout, dir, stdout, stderr)
}

// RunInDirTimeoutFullPipeline executes the command in given directory with given timeout,
// it pipes stdout and stderr to given io.Writer, and stdin from the given io.Reader
func (c *Command) RunInDirTimeoutFullPipeline(timeout time.Duration, dir string, stdout, stderr io.Writer, stdin io.Reader) error {
	return c.RunInDirTimeoutEnvFullPipeline(nil, timeout, dir, stdout, stderr, stdin)
}

// RunInDirTimeout executes the command in given directory with given timeout,
// and returns stdout in []byte and error (combined with stderr).
func (c *Command) RunInDirTimeout(timeout time.Duration, dir string) ([]byte, error) {
	return c.RunInDirTimeoutEnv(nil, timeout, dir)
}

// RunInDirTimeoutEnv executes the command in given directory with given timeout,
// and returns stdout in []byte and error (combined with stderr).
func (c *Command) RunInDirTimeoutEnv(env []string, timeout time.Duration, dir string) ([]byte, error) {
	stdout := new(bytes.Buffer)
	stderr := new(bytes.Buffer)
	if err := c.RunInDirTimeoutEnvPipeline(env, timeout, dir, stdout, stderr); err != nil {
		return nil, ConcatenateError(err, stderr.String())
	}
	if stdout.Len() > 0 && log.IsTrace() {
		tracelen := stdout.Len()
		if tracelen > 1024 {
			tracelen = 1024
		}
		log.Trace("Stdout:\n %s", stdout.Bytes()[:tracelen])
	}
	return stdout.Bytes(), nil
}

// RunInDirPipeline executes the command in given directory,
// it pipes stdout and stderr to given io.Writer.
func (c *Command) RunInDirPipeline(dir string, stdout, stderr io.Writer) error {
	return c.RunInDirFullPipeline(dir, stdout, stderr, nil)
}

// RunInDirFullPipeline executes the command in given directory,
// it pipes stdout and stderr to given io.Writer.
func (c *Command) RunInDirFullPipeline(dir string, stdout, stderr io.Writer, stdin io.Reader) error {
	return c.RunInDirTimeoutFullPipeline(-1, dir, stdout, stderr, stdin)
}

// RunInDirBytes executes the command in given directory
// and returns stdout in []byte and error (combined with stderr).
func (c *Command) RunInDirBytes(dir string) ([]byte, error) {
	return c.RunInDirTimeout(-1, dir)
}

// RunInDir executes the command in given directory
// and returns stdout in string and error (combined with stderr).
func (c *Command) RunInDir(dir string) (string, error) {
	return c.RunInDirWithEnv(dir, nil)
}

// RunInDirWithEnv executes the command in given directory
// and returns stdout in string and error (combined with stderr).
func (c *Command) RunInDirWithEnv(dir string, env []string) (string, error) {
	stdout, err := c.RunInDirTimeoutEnv(env, -1, dir)
	if err != nil {
		return "", err
	}
	return string(stdout), nil
}

// RunTimeout executes the command in default working directory with given timeout,
// and returns stdout in string and error (combined with stderr).
func (c *Command) RunTimeout(timeout time.Duration) (string, error) {
	stdout, err := c.RunInDirTimeout(timeout, "")
	if err != nil {
		return "", err
	}
	return string(stdout), nil
}

// Run executes the command in default working directory
// and returns stdout in string and error (combined with stderr).
func (c *Command) Run() (string, error) {
	return c.RunTimeout(-1)
}
<|endoftext|>


<commit_before>/*
 * Minio Cloud Storage, (C) 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"encoding/json"
	"io"
	"path"
	"path/filepath"
	"strings"

	"github.com/minio/minio/pkg/mimedb"
)

const (
	multipartSuffix   = ".minio.multipart"
	multipartMetaFile = "00000" + multipartSuffix
)

// xlObjects - Implements fs object layer.
type xlObjects struct {
	storage StorageAPI
}

// newXLObjects - initialize new xl object layer.
func newXLObjects(exportPaths ...string) (ObjectLayer, error) {
	storage, err := newXL(exportPaths...)
	if err != nil {
		return nil, err
	}
	return xlObjects{storage}, nil
}

/// Bucket operations

// MakeBucket - make a bucket.
func (xl xlObjects) MakeBucket(bucket string) error {
	return makeBucket(xl.storage, bucket)
}

// GetBucketInfo - get bucket info.
func (xl xlObjects) GetBucketInfo(bucket string) (BucketInfo, error) {
	return getBucketInfo(xl.storage, bucket)
}

// ListBuckets - list buckets.
func (xl xlObjects) ListBuckets() ([]BucketInfo, error) {
	return listBuckets(xl.storage)
}

// DeleteBucket - delete a bucket.
func (xl xlObjects) DeleteBucket(bucket string) error {
	return deleteBucket(xl.storage, bucket)
}

/// Object Operations

// GetObject - get an object.
func (xl xlObjects) GetObject(bucket, object string, startOffset int64) (io.ReadCloser, error) {
	// Verify if bucket is valid.
	if !IsValidBucketName(bucket) {
		return nil, BucketNameInvalid{Bucket: bucket}
	}
	// Verify if object is valid.
	if !IsValidObjectName(object) {
		return nil, ObjectNameInvalid{Bucket: bucket, Object: object}
	}
	if _, err := xl.storage.StatFile(bucket, pathJoin(object, multipartMetaFile)); err != nil {
		if _, err = xl.storage.StatFile(bucket, object); err == nil {
			var reader io.ReadCloser
			reader, err = xl.storage.ReadFile(bucket, object, startOffset)
			if err != nil {
				return nil, toObjectErr(err, bucket, object)
			}
			return reader, nil
		}
		return nil, toObjectErr(err, bucket, object)
	}
	fileReader, fileWriter := io.Pipe()
	info, err := xl.getMultipartObjectInfo(bucket, object)
	if err != nil {
		return nil, toObjectErr(err, bucket, object)
	}
	partIndex, offset, err := info.GetPartNumberOffset(startOffset)
	if err != nil {
		return nil, toObjectErr(err, bucket, object)
	}
	go func() {
		for ; partIndex < len(info); partIndex++ {
			part := info[partIndex]
			r, err := xl.storage.ReadFile(bucket, pathJoin(object, partNumToPartFileName(part.PartNumber)), offset)
			if err != nil {
				fileWriter.CloseWithError(err)
				return
			}
			if _, err := io.Copy(fileWriter, r); err != nil {
				fileWriter.CloseWithError(err)
				return
			}
		}
		fileWriter.Close()
	}()
	return fileReader, nil
}

// Return the parts of a multipart upload.
func (xl xlObjects) getMultipartObjectInfo(bucket, object string) (info MultipartObjectInfo, err error) {
	offset := int64(0)
	r, err := xl.storage.ReadFile(bucket, pathJoin(object, multipartMetaFile), offset)
	if err != nil {
		return
	}
	decoder := json.NewDecoder(r)
	err = decoder.Decode(&info)
	return
}

// GetObjectInfo - get object info.
func (xl xlObjects) GetObjectInfo(bucket, object string) (ObjectInfo, error) {
	// Verify if bucket is valid.
	if !IsValidBucketName(bucket) {
		return ObjectInfo{}, BucketNameInvalid{Bucket: bucket}
	}
	// Verify if object is valid.
	if !IsValidObjectName(object) {
		return ObjectInfo{}, ObjectNameInvalid{Bucket: bucket, Object: object}
	}
	fi, err := xl.storage.StatFile(bucket, object)
	if err != nil {
		info, err := xl.getMultipartObjectInfo(bucket, object)
		if err != nil {
			return ObjectInfo{}, toObjectErr(err, bucket, object)
		}
		fi.Size = info.GetSize()
	}
	contentType := "application/octet-stream"
	if objectExt := filepath.Ext(object); objectExt != "" {
		content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]
		if ok {
			contentType = content.ContentType
		}
	}
	return ObjectInfo{
		Bucket:      bucket,
		Name:        object,
		ModTime:     fi.ModTime,
		Size:        fi.Size,
		IsDir:       fi.Mode.IsDir(),
		ContentType: contentType,
		MD5Sum:      "", // Read from metadata.
	}, nil
}

// PutObject - create an object.
func (xl xlObjects) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (string, error) {
	return putObjectCommon(xl.storage, bucket, object, size, data, metadata)
}

func (xl xlObjects) DeleteObject(bucket, object string) error {
	// Verify if bucket is valid.
	if !IsValidBucketName(bucket) {
		return BucketNameInvalid{Bucket: bucket}
	}
	if !IsValidObjectName(object) {
		return ObjectNameInvalid{Bucket: bucket, Object: object}
	}
	if err := xl.storage.DeleteFile(bucket, object); err != nil {
		return toObjectErr(err, bucket, object)
	}
	return nil
}

// TODO - support non-recursive case, figure out file size for files uploaded using multipart.

func (xl xlObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
	// Verify if bucket is valid.
	if !IsValidBucketName(bucket) {
		return ListObjectsInfo{}, BucketNameInvalid{Bucket: bucket}
	}
	if !IsValidObjectPrefix(prefix) {
		return ListObjectsInfo{}, ObjectNameInvalid{Bucket: bucket, Object: prefix}
	}
	// Verify if delimiter is anything other than '/', which we do not support.
	if delimiter != "" && delimiter != slashSeparator {
		return ListObjectsInfo{}, UnsupportedDelimiter{
			Delimiter: delimiter,
		}
	}
	// Verify if marker has prefix.
	if marker != "" {
		if !strings.HasPrefix(marker, prefix) {
			return ListObjectsInfo{}, InvalidMarkerPrefixCombination{
				Marker: marker,
				Prefix: prefix,
			}
		}
	}

	if maxKeys == 0 {
		return ListObjectsInfo{}, nil
	}

	// Default is recursive, if delimiter is set then list non recursive.
	recursive := true
	if delimiter == slashSeparator {
		recursive = false
	}
	var allFileInfos, fileInfos []FileInfo
	var eof bool
	var err error
	for {
		fileInfos, eof, err = xl.storage.ListFiles(bucket, prefix, marker, recursive, maxKeys)
		if err != nil {
			return ListObjectsInfo{}, toObjectErr(err, bucket)
		}
		for _, fileInfo := range fileInfos {
			// FIXME: use fileInfo.Mode.IsDir() instead after fixing the bug in
			// XL listing which is not reseting the Mode to 0 for leaf dirs.
			if strings.HasSuffix(fileInfo.Name, slashSeparator) && isLeafDirectory(xl.storage, bucket, fileInfo.Name) {
				// Set the Mode to a "regular" file.
				var info MultipartObjectInfo
				info, err = xl.getMultipartObjectInfo(bucket, fileInfo.Name)
				if err == nil {
					fileInfo.Mode = 0

					fileInfo.Name = strings.TrimSuffix(fileInfo.Name, slashSeparator)
					fileInfo.Size = info.GetSize()
				} else if err != errFileNotFound {
					return ListObjectsInfo{}, toObjectErr(err, bucket, fileInfo.Name)
				}
				allFileInfos = append(allFileInfos, fileInfo)
				maxKeys--
				continue
			} else if strings.HasSuffix(fileInfo.Name, multipartMetaFile) {
				fileInfo.Name = path.Dir(fileInfo.Name)
				var info MultipartObjectInfo
				info, err = xl.getMultipartObjectInfo(bucket, fileInfo.Name)
				if err != nil {
					return ListObjectsInfo{}, toObjectErr(err, bucket, fileInfo.Name)
				}
				fileInfo.Size = info.GetSize()
				allFileInfos = append(allFileInfos, fileInfo)
				maxKeys--
				continue
			} else if strings.HasSuffix(fileInfo.Name, multipartSuffix) {
				continue
			}
			allFileInfos = append(allFileInfos, fileInfo)
			maxKeys--
		}
		if maxKeys == 0 {
			break
		}
		if eof {
			break
		}
	}

	result := ListObjectsInfo{IsTruncated: !eof}

	for _, fileInfo := range allFileInfos {
		// With delimiter set we fill in NextMarker and Prefixes.
		if delimiter == slashSeparator {
			result.NextMarker = fileInfo.Name
			if fileInfo.Mode.IsDir() {
				result.Prefixes = append(result.Prefixes, fileInfo.Name)
				continue
			}
		}
		result.Objects = append(result.Objects, ObjectInfo{
			Name:    fileInfo.Name,
			ModTime: fileInfo.ModTime,
			Size:    fileInfo.Size,
			IsDir:   false,
		})
	}
	return result, nil
}
<commit_msg>xl/deleteObject: Support deleting special multipart object. (#1470)<commit_after>/*
 * Minio Cloud Storage, (C) 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"encoding/json"
	"io"
	"path"
	"path/filepath"
	"strings"

	"github.com/minio/minio/pkg/mimedb"
)

const (
	multipartSuffix   = ".minio.multipart"
	multipartMetaFile = "00000" + multipartSuffix
)

// xlObjects - Implements fs object layer.
type xlObjects struct {
	storage StorageAPI
}

// newXLObjects - initialize new xl object layer.
func newXLObjects(exportPaths ...string) (ObjectLayer, error) {
	storage, err := newXL(exportPaths...)
	if err != nil {
		return nil, err
	}
	return xlObjects{storage}, nil
}

/// Bucket operations

// MakeBucket - make a bucket.
func (xl xlObjects) MakeBucket(bucket string) error {
	return makeBucket(xl.storage, bucket)
}

// GetBucketInfo - get bucket info.
func (xl xlObjects) GetBucketInfo(bucket string) (BucketInfo, error) {
	return getBucketInfo(xl.storage, bucket)
}

// ListBuckets - list buckets.
func (xl xlObjects) ListBuckets() ([]BucketInfo, error) {
	return listBuckets(xl.storage)
}

// DeleteBucket - delete a bucket.
func (xl xlObjects) DeleteBucket(bucket string) error {
	return deleteBucket(xl.storage, bucket)
}

/// Object Operations

// GetObject - get an object.
func (xl xlObjects) GetObject(bucket, object string, startOffset int64) (io.ReadCloser, error) {
	// Verify if bucket is valid.
	if !IsValidBucketName(bucket) {
		return nil, BucketNameInvalid{Bucket: bucket}
	}
	// Verify if object is valid.
	if !IsValidObjectName(object) {
		return nil, ObjectNameInvalid{Bucket: bucket, Object: object}
	}
	if ok, err := isMultipartObject(xl.storage, bucket, object); err != nil {
		return nil, toObjectErr(err, bucket, object)
	} else if !ok {
		if _, err = xl.storage.StatFile(bucket, object); err == nil {
			var reader io.ReadCloser
			reader, err = xl.storage.ReadFile(bucket, object, startOffset)
			if err != nil {
				return nil, toObjectErr(err, bucket, object)
			}
			return reader, nil
		}
		return nil, toObjectErr(err, bucket, object)
	}
	fileReader, fileWriter := io.Pipe()
	info, err := getMultipartObjectInfo(xl.storage, bucket, object)
	if err != nil {
		return nil, toObjectErr(err, bucket, object)
	}
	partIndex, offset, err := info.GetPartNumberOffset(startOffset)
	if err != nil {
		return nil, toObjectErr(err, bucket, object)
	}
	go func() {
		for ; partIndex < len(info); partIndex++ {
			part := info[partIndex]
			r, err := xl.storage.ReadFile(bucket, pathJoin(object, partNumToPartFileName(part.PartNumber)), offset)
			if err != nil {
				fileWriter.CloseWithError(err)
				return
			}
			if _, err := io.Copy(fileWriter, r); err != nil {
				fileWriter.CloseWithError(err)
				return
			}
		}
		fileWriter.Close()
	}()
	return fileReader, nil
}

// Return the partsInfo of a special multipart object.
func getMultipartObjectInfo(storage StorageAPI, bucket, object string) (info MultipartObjectInfo, err error) {
	offset := int64(0)
	r, err := storage.ReadFile(bucket, pathJoin(object, multipartMetaFile), offset)
	if err != nil {
		return
	}
	decoder := json.NewDecoder(r)
	err = decoder.Decode(&info)
	return
}

// GetObjectInfo - get object info.
func (xl xlObjects) GetObjectInfo(bucket, object string) (ObjectInfo, error) {
	// Verify if bucket is valid.
	if !IsValidBucketName(bucket) {
		return ObjectInfo{}, BucketNameInvalid{Bucket: bucket}
	}
	// Verify if object is valid.
	if !IsValidObjectName(object) {
		return ObjectInfo{}, ObjectNameInvalid{Bucket: bucket, Object: object}
	}
	fi, err := xl.storage.StatFile(bucket, object)
	if err != nil {
		info, err := getMultipartObjectInfo(xl.storage, bucket, object)
		if err != nil {
			return ObjectInfo{}, toObjectErr(err, bucket, object)
		}
		fi.Size = info.GetSize()
	}
	contentType := "application/octet-stream"
	if objectExt := filepath.Ext(object); objectExt != "" {
		content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]
		if ok {
			contentType = content.ContentType
		}
	}
	return ObjectInfo{
		Bucket:      bucket,
		Name:        object,
		ModTime:     fi.ModTime,
		Size:        fi.Size,
		IsDir:       fi.Mode.IsDir(),
		ContentType: contentType,
		MD5Sum:      "", // Read from metadata.
	}, nil
}

// PutObject - create an object.
func (xl xlObjects) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (string, error) {
	return putObjectCommon(xl.storage, bucket, object, size, data, metadata)
}

// isMultipartObject - verifies if an object is special multipart file.
func isMultipartObject(storage StorageAPI, bucket, object string) (bool, error) {
	_, err := storage.StatFile(bucket, pathJoin(object, multipartMetaFile))
	if err != nil {
		if err == errFileNotFound {
			return false, nil
		}
		return false, err
	}
	return true, nil
}

func (xl xlObjects) DeleteObject(bucket, object string) error {
	// Verify if bucket is valid.
	if !IsValidBucketName(bucket) {
		return BucketNameInvalid{Bucket: bucket}
	}
	if !IsValidObjectName(object) {
		return ObjectNameInvalid{Bucket: bucket, Object: object}
	}
	// Verify if the object is a multipart object.
	if ok, err := isMultipartObject(xl.storage, bucket, object); err != nil {
		return toObjectErr(err, bucket, object)
	} else if !ok {
		if err := xl.storage.DeleteFile(bucket, object); err != nil {
			return toObjectErr(err, bucket, object)
		}
	}
	// Get parts info.
	info, err := getMultipartObjectInfo(xl.storage, bucket, object)
	if err != nil {
		return toObjectErr(err, bucket, object)
	}
	// Range through all files and delete it.
	for _, part := range info {
		err = xl.storage.DeleteFile(bucket, pathJoin(object, partNumToPartFileName(part.PartNumber)))
		if err != nil {
			return toObjectErr(err, bucket, object)
		}
	}
	err = xl.storage.DeleteFile(bucket, pathJoin(object, multipartMetaFile))
	if err != nil {
		return toObjectErr(err, bucket, object)
	}
	return nil
}

// TODO - support non-recursive case, figure out file size for files uploaded using multipart.

func (xl xlObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
	// Verify if bucket is valid.
	if !IsValidBucketName(bucket) {
		return ListObjectsInfo{}, BucketNameInvalid{Bucket: bucket}
	}
	if !IsValidObjectPrefix(prefix) {
		return ListObjectsInfo{}, ObjectNameInvalid{Bucket: bucket, Object: prefix}
	}
	// Verify if delimiter is anything other than '/', which we do not support.
	if delimiter != "" && delimiter != slashSeparator {
		return ListObjectsInfo{}, UnsupportedDelimiter{
			Delimiter: delimiter,
		}
	}
	// Verify if marker has prefix.
	if marker != "" {
		if !strings.HasPrefix(marker, prefix) {
			return ListObjectsInfo{}, InvalidMarkerPrefixCombination{
				Marker: marker,
				Prefix: prefix,
			}
		}
	}

	if maxKeys == 0 {
		return ListObjectsInfo{}, nil
	}

	// Default is recursive, if delimiter is set then list non recursive.
	recursive := true
	if delimiter == slashSeparator {
		recursive = false
	}
	var allFileInfos, fileInfos []FileInfo
	var eof bool
	var err error
	for {
		fileInfos, eof, err = xl.storage.ListFiles(bucket, prefix, marker, recursive, maxKeys)
		if err != nil {
			return ListObjectsInfo{}, toObjectErr(err, bucket)
		}
		for _, fileInfo := range fileInfos {
			// FIXME: use fileInfo.Mode.IsDir() instead after fixing the bug in
			// XL listing which is not reseting the Mode to 0 for leaf dirs.
			if strings.HasSuffix(fileInfo.Name, slashSeparator) && isLeafDirectory(xl.storage, bucket, fileInfo.Name) {
				// Set the Mode to a "regular" file.
				var info MultipartObjectInfo
				info, err = getMultipartObjectInfo(xl.storage, bucket, fileInfo.Name)
				if err == nil {
					fileInfo.Mode = 0

					fileInfo.Name = strings.TrimSuffix(fileInfo.Name, slashSeparator)
					fileInfo.Size = info.GetSize()
				} else if err != errFileNotFound {
					return ListObjectsInfo{}, toObjectErr(err, bucket, fileInfo.Name)
				}
				allFileInfos = append(allFileInfos, fileInfo)
				maxKeys--
				continue
			} else if strings.HasSuffix(fileInfo.Name, multipartMetaFile) {
				fileInfo.Name = path.Dir(fileInfo.Name)
				var info MultipartObjectInfo
				info, err = getMultipartObjectInfo(xl.storage, bucket, fileInfo.Name)
				if err != nil {
					return ListObjectsInfo{}, toObjectErr(err, bucket, fileInfo.Name)
				}
				fileInfo.Size = info.GetSize()
				allFileInfos = append(allFileInfos, fileInfo)
				maxKeys--
				continue
			} else if strings.HasSuffix(fileInfo.Name, multipartSuffix) {
				continue
			}
			allFileInfos = append(allFileInfos, fileInfo)
			maxKeys--
		}
		if maxKeys == 0 {
			break
		}
		if eof {
			break
		}
	}

	result := ListObjectsInfo{IsTruncated: !eof}

	for _, fileInfo := range allFileInfos {
		// With delimiter set we fill in NextMarker and Prefixes.
		if delimiter == slashSeparator {
			result.NextMarker = fileInfo.Name
			if fileInfo.Mode.IsDir() {
				result.Prefixes = append(result.Prefixes, fileInfo.Name)
				continue
			}
		}
		result.Objects = append(result.Objects, ObjectInfo{
			Name:    fileInfo.Name,
			ModTime: fileInfo.ModTime,
			Size:    fileInfo.Size,
			IsDir:   false,
		})
	}
	return result, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2015 Google Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package bigtable

import (
	"fmt"
	"regexp"
	"strings"

	btopt "cloud.google.com/go/bigtable/internal/option"
	"cloud.google.com/go/longrunning"
	lroauto "cloud.google.com/go/longrunning/autogen"
	"golang.org/x/net/context"
	"google.golang.org/api/option"
	gtransport "google.golang.org/api/transport/grpc"
	btapb "google.golang.org/genproto/googleapis/bigtable/admin/v2"
	"google.golang.org/grpc"
	"google.golang.org/grpc/metadata"
)

const adminAddr = "bigtableadmin.googleapis.com:443"

// AdminClient is a client type for performing admin operations within a specific instance.
type AdminClient struct {
	conn    *grpc.ClientConn
	tClient btapb.BigtableTableAdminClient

	project, instance string

	// Metadata to be sent with each request.
	md metadata.MD
}

// NewAdminClient creates a new AdminClient for a given project and instance.
func NewAdminClient(ctx context.Context, project, instance string, opts ...option.ClientOption) (*AdminClient, error) {
	o, err := btopt.DefaultClientOptions(adminAddr, AdminScope, clientUserAgent)
	if err != nil {
		return nil, err
	}
	o = append(o, opts...)
	conn, err := gtransport.Dial(ctx, o...)
	if err != nil {
		return nil, fmt.Errorf("dialing: %v", err)
	}
	return &AdminClient{
		conn:     conn,
		tClient:  btapb.NewBigtableTableAdminClient(conn),
		project:  project,
		instance: instance,
		md:       metadata.Pairs(resourcePrefixHeader, fmt.Sprintf("projects/%s/instances/%s", project, instance)),
	}, nil
}

// Close closes the AdminClient.
func (ac *AdminClient) Close() error {
	return ac.conn.Close()
}

func (ac *AdminClient) instancePrefix() string {
	return fmt.Sprintf("projects/%s/instances/%s", ac.project, ac.instance)
}

// Tables returns a list of the tables in the instance.
func (ac *AdminClient) Tables(ctx context.Context) ([]string, error) {
	ctx = mergeOutgoingMetadata(ctx, ac.md)
	prefix := ac.instancePrefix()
	req := &btapb.ListTablesRequest{
		Parent: prefix,
	}
	res, err := ac.tClient.ListTables(ctx, req)
	if err != nil {
		return nil, err
	}
	names := make([]string, 0, len(res.Tables))
	for _, tbl := range res.Tables {
		names = append(names, strings.TrimPrefix(tbl.Name, prefix+"/tables/"))
	}
	return names, nil
}

// TableConf contains all of the information necessary to create a table with column families.
type TableConf struct {
	TableID   string
	SplitKeys []string
	// Families is a map from family name to GCPolicy
	Families map[string]GCPolicy
}

// CreateTable creates a new table in the instance.
// This method may return before the table's creation is complete.
func (ac *AdminClient) CreateTable(ctx context.Context, table string) error {
	return ac.CreateTableFromConf(ctx, &TableConf{TableID: table})
}

// CreatePresplitTable creates a new table in the instance.
// The list of row keys will be used to initially split the table into multiple tablets.
// Given two split keys, "s1" and "s2", three tablets will be created,
// spanning the key ranges: [, s1), [s1, s2), [s2, ).
// This method may return before the table's creation is complete.
func (ac *AdminClient) CreatePresplitTable(ctx context.Context, table string, splitKeys []string) error {
	return ac.CreateTableFromConf(ctx, &TableConf{TableID: table, SplitKeys: splitKeys})
}

// CreateTableFromConf creates a new table in the instance from the given configuration.
func (ac *AdminClient) CreateTableFromConf(ctx context.Context, conf *TableConf) error {
	ctx = mergeOutgoingMetadata(ctx, ac.md)
	var req_splits []*btapb.CreateTableRequest_Split
	for _, split := range conf.SplitKeys {
		req_splits = append(req_splits, &btapb.CreateTableRequest_Split{[]byte(split)})
	}
	var tbl btapb.Table
	if conf.Families != nil {
		tbl.ColumnFamilies = make(map[string]*btapb.ColumnFamily)
		for fam, policy := range conf.Families {
			tbl.ColumnFamilies[fam] = &btapb.ColumnFamily{policy.proto()}
		}
	}
	prefix := ac.instancePrefix()
	req := &btapb.CreateTableRequest{
		Parent:        prefix,
		TableId:       conf.TableID,
		Table:         &tbl,
		InitialSplits: req_splits,
	}
	_, err := ac.tClient.CreateTable(ctx, req)
	return err
}

// CreateColumnFamily creates a new column family in a table.
func (ac *AdminClient) CreateColumnFamily(ctx context.Context, table, family string) error {
	// TODO(dsymonds): Permit specifying gcexpr and any other family settings.
	ctx = mergeOutgoingMetadata(ctx, ac.md)
	prefix := ac.instancePrefix()
	req := &btapb.ModifyColumnFamiliesRequest{
		Name: prefix + "/tables/" + table,
		Modifications: []*btapb.ModifyColumnFamiliesRequest_Modification{{
			Id:  family,
			Mod: &btapb.ModifyColumnFamiliesRequest_Modification_Create{&btapb.ColumnFamily{}},
		}},
	}
	_, err := ac.tClient.ModifyColumnFamilies(ctx, req)
	return err
}

// DeleteTable deletes a table and all of its data.
func (ac *AdminClient) DeleteTable(ctx context.Context, table string) error {
	ctx = mergeOutgoingMetadata(ctx, ac.md)
	prefix := ac.instancePrefix()
	req := &btapb.DeleteTableRequest{
		Name: prefix + "/tables/" + table,
	}
	_, err := ac.tClient.DeleteTable(ctx, req)
	return err
}

// DeleteColumnFamily deletes a column family in a table and all of its data.
func (ac *AdminClient) DeleteColumnFamily(ctx context.Context, table, family string) error {
	ctx = mergeOutgoingMetadata(ctx, ac.md)
	prefix := ac.instancePrefix()
	req := &btapb.ModifyColumnFamiliesRequest{
		Name: prefix + "/tables/" + table,
		Modifications: []*btapb.ModifyColumnFamiliesRequest_Modification{{
			Id:  family,
			Mod: &btapb.ModifyColumnFamiliesRequest_Modification_Drop{true},
		}},
	}
	_, err := ac.tClient.ModifyColumnFamilies(ctx, req)
	return err
}

// TableInfo represents information about a table.
type TableInfo struct {
	// DEPRECATED - This field is deprecated. Please use FamilyInfos instead.
	Families    []string
	FamilyInfos []FamilyInfo
}

// FamilyInfo represents information about a column family.
type FamilyInfo struct {
	Name     string
	GCPolicy string
}

// TableInfo retrieves information about a table.
func (ac *AdminClient) TableInfo(ctx context.Context, table string) (*TableInfo, error) {
	ctx = mergeOutgoingMetadata(ctx, ac.md)
	prefix := ac.instancePrefix()
	req := &btapb.GetTableRequest{
		Name: prefix + "/tables/" + table,
	}
	res, err := ac.tClient.GetTable(ctx, req)
	if err != nil {
		return nil, err
	}
	ti := &TableInfo{}
	for name, fam := range res.ColumnFamilies {
		ti.Families = append(ti.Families, name)
		ti.FamilyInfos = append(ti.FamilyInfos, FamilyInfo{Name: name, GCPolicy: GCRuleToString(fam.GcRule)})
	}
	return ti, nil
}

// SetGCPolicy specifies which cells in a column family should be garbage collected.
// GC executes opportunistically in the background; table reads may return data
// matching the GC policy.
func (ac *AdminClient) SetGCPolicy(ctx context.Context, table, family string, policy GCPolicy) error {
	ctx = mergeOutgoingMetadata(ctx, ac.md)
	prefix := ac.instancePrefix()
	req := &btapb.ModifyColumnFamiliesRequest{
		Name: prefix + "/tables/" + table,
		Modifications: []*btapb.ModifyColumnFamiliesRequest_Modification{{
			Id:  family,
			Mod: &btapb.ModifyColumnFamiliesRequest_Modification_Update{&btapb.ColumnFamily{GcRule: policy.proto()}},
		}},
	}
	_, err := ac.tClient.ModifyColumnFamilies(ctx, req)
	return err
}

// DropRowRange permanently deletes a row range from the specified table.
func (ac *AdminClient) DropRowRange(ctx context.Context, table, rowKeyPrefix string) error {
	ctx = mergeOutgoingMetadata(ctx, ac.md)
	prefix := ac.instancePrefix()
	req := &btapb.DropRowRangeRequest{
		Name:   prefix + "/tables/" + table,
		Target: &btapb.DropRowRangeRequest_RowKeyPrefix{[]byte(rowKeyPrefix)},
	}
	_, err := ac.tClient.DropRowRange(ctx, req)
	return err
}

const instanceAdminAddr = "bigtableadmin.googleapis.com:443"

// InstanceAdminClient is a client type for performing admin operations on instances.
// These operations can be substantially more dangerous than those provided by AdminClient.
type InstanceAdminClient struct {
	conn      *grpc.ClientConn
	iClient   btapb.BigtableInstanceAdminClient
	lroClient *lroauto.OperationsClient

	project string

	// Metadata to be sent with each request.
	md metadata.MD
}

// NewInstanceAdminClient creates a new InstanceAdminClient for a given project.
func NewInstanceAdminClient(ctx context.Context, project string, opts ...option.ClientOption) (*InstanceAdminClient, error) {
	o, err := btopt.DefaultClientOptions(instanceAdminAddr, InstanceAdminScope, clientUserAgent)
	if err != nil {
		return nil, err
	}
	o = append(o, opts...)
	conn, err := gtransport.Dial(ctx, o...)
	if err != nil {
		return nil, fmt.Errorf("dialing: %v", err)
	}

	lroClient, err := lroauto.NewOperationsClient(ctx, option.WithGRPCConn(conn))
	if err != nil {
		// This error "should not happen", since we are just reusing old connection
		// and never actually need to dial.
		// If this does happen, we could leak conn. However, we cannot close conn:
		// If the user invoked the function with option.WithGRPCConn,
		// we would close a connection that's still in use.
		// TODO(pongad): investigate error conditions.
		return nil, err
	}

	return &InstanceAdminClient{
		conn:      conn,
		iClient:   btapb.NewBigtableInstanceAdminClient(conn),
		lroClient: lroClient,

		project: project,
		md:      metadata.Pairs(resourcePrefixHeader, "projects/"+project),
	}, nil
}

// Close closes the InstanceAdminClient.
func (iac *InstanceAdminClient) Close() error {
	return iac.conn.Close()
}

// StorageType is the type of storage used for all tables in an instance
type StorageType int

const (
	SSD StorageType = iota
	HDD
)

func (st StorageType) proto() btapb.StorageType {
	if st == HDD {
		return btapb.StorageType_HDD
	}
	return btapb.StorageType_SSD
}

// InstanceInfo represents information about an instance
type InstanceInfo struct {
	Name        string // name of the instance
	DisplayName string // display name for UIs
}

// InstanceConf contains the information necessary to create an Instance
type InstanceConf struct {
	InstanceId, DisplayName, ClusterId, Zone string
	NumNodes                                 int32
	StorageType                              StorageType
}

var instanceNameRegexp = regexp.MustCompile(`^projects/([^/]+)/instances/([a-z][-a-z0-9]*)$`)

// CreateInstance creates a new instance in the project.
// This method will return when the instance has been created or when an error occurs.
func (iac *InstanceAdminClient) CreateInstance(ctx context.Context, conf *InstanceConf) error {
	ctx = mergeOutgoingMetadata(ctx, iac.md)
	req := &btapb.CreateInstanceRequest{
		Parent:     "projects/" + iac.project,
		InstanceId: conf.InstanceId,
		Instance:   &btapb.Instance{DisplayName: conf.DisplayName},
		Clusters: map[string]*btapb.Cluster{
			conf.ClusterId: {
				ServeNodes:         conf.NumNodes,
				DefaultStorageType: conf.StorageType.proto(),
				Location:           "projects/" + iac.project + "/locations/" + conf.Zone,
			},
		},
	}

	lro, err := iac.iClient.CreateInstance(ctx, req)
	if err != nil {
		return err
	}
	resp := btapb.Instance{}
	return longrunning.InternalNewOperation(iac.lroClient, lro).Wait(ctx, &resp)
}

// DeleteInstance deletes an instance from the project.
func (iac *InstanceAdminClient) DeleteInstance(ctx context.Context, instanceId string) error {
	ctx = mergeOutgoingMetadata(ctx, iac.md)
	req := &btapb.DeleteInstanceRequest{"projects/" + iac.project + "/instances/" + instanceId}
	_, err := iac.iClient.DeleteInstance(ctx, req)
	return err
}

// Instances returns a list of instances in the project.
func (iac *InstanceAdminClient) Instances(ctx context.Context) ([]*InstanceInfo, error) {
	ctx = mergeOutgoingMetadata(ctx, iac.md)
	req := &btapb.ListInstancesRequest{
		Parent: "projects/" + iac.project,
	}
	res, err := iac.iClient.ListInstances(ctx, req)
	if err != nil {
		return nil, err
	}

	var is []*InstanceInfo
	for _, i := range res.Instances {
		m := instanceNameRegexp.FindStringSubmatch(i.Name)
		if m == nil {
			return nil, fmt.Errorf("malformed instance name %q", i.Name)
		}
		is = append(is, &InstanceInfo{
			Name:        m[2],
			DisplayName: i.DisplayName,
		})
	}
	return is, nil
}

// InstanceInfo returns information about an instance.
func (iac *InstanceAdminClient) InstanceInfo(ctx context.Context, instanceId string) (*InstanceInfo, error) {
	ctx = mergeOutgoingMetadata(ctx, iac.md)
	req := &btapb.GetInstanceRequest{
		Name: "projects/" + iac.project + "/instances/" + instanceId,
	}
	res, err := iac.iClient.GetInstance(ctx, req)
	if err != nil {
		return nil, err
	}

	m := instanceNameRegexp.FindStringSubmatch(res.Name)
	if m == nil {
		return nil, fmt.Errorf("malformed instance name %q", res.Name)
	}
	return &InstanceInfo{
		Name:        m[2],
		DisplayName: res.DisplayName,
	}, nil
}
<commit_msg>bigtable: Add InstanceType to CreateInstance<commit_after>/*
Copyright 2015 Google Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package bigtable

import (
	"fmt"
	"regexp"
	"strings"

	btopt "cloud.google.com/go/bigtable/internal/option"
	"cloud.google.com/go/longrunning"
	lroauto "cloud.google.com/go/longrunning/autogen"
	"golang.org/x/net/context"
	"google.golang.org/api/option"
	gtransport "google.golang.org/api/transport/grpc"
	btapb "google.golang.org/genproto/googleapis/bigtable/admin/v2"
	"google.golang.org/grpc"
	"google.golang.org/grpc/metadata"
)

const adminAddr = "bigtableadmin.googleapis.com:443"

// AdminClient is a client type for performing admin operations within a specific instance.
type AdminClient struct {
	conn    *grpc.ClientConn
	tClient btapb.BigtableTableAdminClient

	project, instance string

	// Metadata to be sent with each request.
	md metadata.MD
}

// NewAdminClient creates a new AdminClient for a given project and instance.
func NewAdminClient(ctx context.Context, project, instance string, opts ...option.ClientOption) (*AdminClient, error) {
	o, err := btopt.DefaultClientOptions(adminAddr, AdminScope, clientUserAgent)
	if err != nil {
		return nil, err
	}
	o = append(o, opts...)
	conn, err := gtransport.Dial(ctx, o...)
	if err != nil {
		return nil, fmt.Errorf("dialing: %v", err)
	}
	return &AdminClient{
		conn:     conn,
		tClient:  btapb.NewBigtableTableAdminClient(conn),
		project:  project,
		instance: instance,
		md:       metadata.Pairs(resourcePrefixHeader, fmt.Sprintf("projects/%s/instances/%s", project, instance)),
	}, nil
}

// Close closes the AdminClient.
func (ac *AdminClient) Close() error {
	return ac.conn.Close()
}

func (ac *AdminClient) instancePrefix() string {
	return fmt.Sprintf("projects/%s/instances/%s", ac.project, ac.instance)
}

// Tables returns a list of the tables in the instance.
func (ac *AdminClient) Tables(ctx context.Context) ([]string, error) {
	ctx = mergeOutgoingMetadata(ctx, ac.md)
	prefix := ac.instancePrefix()
	req := &btapb.ListTablesRequest{
		Parent: prefix,
	}
	res, err := ac.tClient.ListTables(ctx, req)
	if err != nil {
		return nil, err
	}
	names := make([]string, 0, len(res.Tables))
	for _, tbl := range res.Tables {
		names = append(names, strings.TrimPrefix(tbl.Name, prefix+"/tables/"))
	}
	return names, nil
}

// TableConf contains all of the information necessary to create a table with column families.
type TableConf struct {
	TableID   string
	SplitKeys []string
	// Families is a map from family name to GCPolicy
	Families map[string]GCPolicy
}

// CreateTable creates a new table in the instance.
// This method may return before the table's creation is complete.
func (ac *AdminClient) CreateTable(ctx context.Context, table string) error {
	return ac.CreateTableFromConf(ctx, &TableConf{TableID: table})
}

// CreatePresplitTable creates a new table in the instance.
// The list of row keys will be used to initially split the table into multiple tablets.
// Given two split keys, "s1" and "s2", three tablets will be created,
// spanning the key ranges: [, s1), [s1, s2), [s2, ).
// This method may return before the table's creation is complete.
func (ac *AdminClient) CreatePresplitTable(ctx context.Context, table string, splitKeys []string) error {
	return ac.CreateTableFromConf(ctx, &TableConf{TableID: table, SplitKeys: splitKeys})
}

// CreateTableFromConf creates a new table in the instance from the given configuration.
func (ac *AdminClient) CreateTableFromConf(ctx context.Context, conf *TableConf) error {
	ctx = mergeOutgoingMetadata(ctx, ac.md)
	var req_splits []*btapb.CreateTableRequest_Split
	for _, split := range conf.SplitKeys {
		req_splits = append(req_splits, &btapb.CreateTableRequest_Split{[]byte(split)})
	}
	var tbl btapb.Table
	if conf.Families != nil {
		tbl.ColumnFamilies = make(map[string]*btapb.ColumnFamily)
		for fam, policy := range conf.Families {
			tbl.ColumnFamilies[fam] = &btapb.ColumnFamily{policy.proto()}
		}
	}
	prefix := ac.instancePrefix()
	req := &btapb.CreateTableRequest{
		Parent:        prefix,
		TableId:       conf.TableID,
		Table:         &tbl,
		InitialSplits: req_splits,
	}
	_, err := ac.tClient.CreateTable(ctx, req)
	return err
}

// CreateColumnFamily creates a new column family in a table.
func (ac *AdminClient) CreateColumnFamily(ctx context.Context, table, family string) error {
	// TODO(dsymonds): Permit specifying gcexpr and any other family settings.
	ctx = mergeOutgoingMetadata(ctx, ac.md)
	prefix := ac.instancePrefix()
	req := &btapb.ModifyColumnFamiliesRequest{
		Name: prefix + "/tables/" + table,
		Modifications: []*btapb.ModifyColumnFamiliesRequest_Modification{{
			Id:  family,
			Mod: &btapb.ModifyColumnFamiliesRequest_Modification_Create{&btapb.ColumnFamily{}},
		}},
	}
	_, err := ac.tClient.ModifyColumnFamilies(ctx, req)
	return err
}

// DeleteTable deletes a table and all of its data.
func (ac *AdminClient) DeleteTable(ctx context.Context, table string) error {
	ctx = mergeOutgoingMetadata(ctx, ac.md)
	prefix := ac.instancePrefix()
	req := &btapb.DeleteTableRequest{
		Name: prefix + "/tables/" + table,
	}
	_, err := ac.tClient.DeleteTable(ctx, req)
	return err
}

// DeleteColumnFamily deletes a column family in a table and all of its data.
func (ac *AdminClient) DeleteColumnFamily(ctx context.Context, table, family string) error {
	ctx = mergeOutgoingMetadata(ctx, ac.md)
	prefix := ac.instancePrefix()
	req := &btapb.ModifyColumnFamiliesRequest{
		Name: prefix + "/tables/" + table,
		Modifications: []*btapb.ModifyColumnFamiliesRequest_Modification{{
			Id:  family,
			Mod: &btapb.ModifyColumnFamiliesRequest_Modification_Drop{true},
		}},
	}
	_, err := ac.tClient.ModifyColumnFamilies(ctx, req)
	return err
}

// TableInfo represents information about a table.
type TableInfo struct {
	// DEPRECATED - This field is deprecated. Please use FamilyInfos instead.
	Families    []string
	FamilyInfos []FamilyInfo
}

// FamilyInfo represents information about a column family.
type FamilyInfo struct {
	Name     string
	GCPolicy string
}

// TableInfo retrieves information about a table.
func (ac *AdminClient) TableInfo(ctx context.Context, table string) (*TableInfo, error) {
	ctx = mergeOutgoingMetadata(ctx, ac.md)
	prefix := ac.instancePrefix()
	req := &btapb.GetTableRequest{
		Name: prefix + "/tables/" + table,
	}
	res, err := ac.tClient.GetTable(ctx, req)
	if err != nil {
		return nil, err
	}
	ti := &TableInfo{}
	for name, fam := range res.ColumnFamilies {
		ti.Families = append(ti.Families, name)
		ti.FamilyInfos = append(ti.FamilyInfos, FamilyInfo{Name: name, GCPolicy: GCRuleToString(fam.GcRule)})
	}
	return ti, nil
}

// SetGCPolicy specifies which cells in a column family should be garbage collected.
// GC executes opportunistically in the background; table reads may return data
// matching the GC policy.
func (ac *AdminClient) SetGCPolicy(ctx context.Context, table, family string, policy GCPolicy) error {
	ctx = mergeOutgoingMetadata(ctx, ac.md)
	prefix := ac.instancePrefix()
	req := &btapb.ModifyColumnFamiliesRequest{
		Name: prefix + "/tables/" + table,
		Modifications: []*btapb.ModifyColumnFamiliesRequest_Modification{{
			Id:  family,
			Mod: &btapb.ModifyColumnFamiliesRequest_Modification_Update{&btapb.ColumnFamily{GcRule: policy.proto()}},
		}},
	}
	_, err := ac.tClient.ModifyColumnFamilies(ctx, req)
	return err
}

// DropRowRange permanently deletes a row range from the specified table.
func (ac *AdminClient) DropRowRange(ctx context.Context, table, rowKeyPrefix string) error {
	ctx = mergeOutgoingMetadata(ctx, ac.md)
	prefix := ac.instancePrefix()
	req := &btapb.DropRowRangeRequest{
		Name:   prefix + "/tables/" + table,
		Target: &btapb.DropRowRangeRequest_RowKeyPrefix{[]byte(rowKeyPrefix)},
	}
	_, err := ac.tClient.DropRowRange(ctx, req)
	return err
}

const instanceAdminAddr = "bigtableadmin.googleapis.com:443"

// InstanceAdminClient is a client type for performing admin operations on instances.
// These operations can be substantially more dangerous than those provided by AdminClient.
type InstanceAdminClient struct {
	conn      *grpc.ClientConn
	iClient   btapb.BigtableInstanceAdminClient
	lroClient *lroauto.OperationsClient

	project string

	// Metadata to be sent with each request.
	md metadata.MD
}

// NewInstanceAdminClient creates a new InstanceAdminClient for a given project.
func NewInstanceAdminClient(ctx context.Context, project string, opts ...option.ClientOption) (*InstanceAdminClient, error) {
	o, err := btopt.DefaultClientOptions(instanceAdminAddr, InstanceAdminScope, clientUserAgent)
	if err != nil {
		return nil, err
	}
	o = append(o, opts...)
	conn, err := gtransport.Dial(ctx, o...)
	if err != nil {
		return nil, fmt.Errorf("dialing: %v", err)
	}

	lroClient, err := lroauto.NewOperationsClient(ctx, option.WithGRPCConn(conn))
	if err != nil {
		// This error "should not happen", since we are just reusing old connection
		// and never actually need to dial.
		// If this does happen, we could leak conn. However, we cannot close conn:
		// If the user invoked the function with option.WithGRPCConn,
		// we would close a connection that's still in use.
		// TODO(pongad): investigate error conditions.
		return nil, err
	}

	return &InstanceAdminClient{
		conn:      conn,
		iClient:   btapb.NewBigtableInstanceAdminClient(conn),
		lroClient: lroClient,

		project: project,
		md:      metadata.Pairs(resourcePrefixHeader, "projects/"+project),
	}, nil
}

// Close closes the InstanceAdminClient.
func (iac *InstanceAdminClient) Close() error {
	return iac.conn.Close()
}

// StorageType is the type of storage used for all tables in an instance
type StorageType int

const (
	SSD StorageType = iota
	HDD
)

func (st StorageType) proto() btapb.StorageType {
	if st == HDD {
		return btapb.StorageType_HDD
	}
	return btapb.StorageType_SSD
}

// InstanceType is the type of the instance
type InstanceType int32

const (
	PRODUCTION  InstanceType = InstanceType(btapb.Instance_PRODUCTION)
	DEVELOPMENT              = InstanceType(btapb.Instance_DEVELOPMENT)
)

// InstanceInfo represents information about an instance
type InstanceInfo struct {
	Name        string // name of the instance
	DisplayName string // display name for UIs
}

// InstanceConf contains the information necessary to create an Instance
type InstanceConf struct {
	InstanceId, DisplayName, ClusterId, Zone string
	// NumNodes must not be specified for DEVELOPMENT instance types
	NumNodes     int32
	StorageType  StorageType
	InstanceType InstanceType
}

var instanceNameRegexp = regexp.MustCompile(`^projects/([^/]+)/instances/([a-z][-a-z0-9]*)$`)

// CreateInstance creates a new instance in the project.
// This method will return when the instance has been created or when an error occurs.
func (iac *InstanceAdminClient) CreateInstance(ctx context.Context, conf *InstanceConf) error {
	ctx = mergeOutgoingMetadata(ctx, iac.md)
	req := &btapb.CreateInstanceRequest{
		Parent:     "projects/" + iac.project,
		InstanceId: conf.InstanceId,
		Instance:   &btapb.Instance{DisplayName: conf.DisplayName, Type: btapb.Instance_Type(conf.InstanceType)},
		Clusters: map[string]*btapb.Cluster{
			conf.ClusterId: {
				ServeNodes:         conf.NumNodes,
				DefaultStorageType: conf.StorageType.proto(),
				Location:           "projects/" + iac.project + "/locations/" + conf.Zone,
			},
		},
	}

	lro, err := iac.iClient.CreateInstance(ctx, req)
	if err != nil {
		return err
	}
	resp := btapb.Instance{}
	return longrunning.InternalNewOperation(iac.lroClient, lro).Wait(ctx, &resp)
}

// DeleteInstance deletes an instance from the project.
func (iac *InstanceAdminClient) DeleteInstance(ctx context.Context, instanceId string) error {
	ctx = mergeOutgoingMetadata(ctx, iac.md)
	req := &btapb.DeleteInstanceRequest{"projects/" + iac.project + "/instances/" + instanceId}
	_, err := iac.iClient.DeleteInstance(ctx, req)
	return err
}

// Instances returns a list of instances in the project.
func (iac *InstanceAdminClient) Instances(ctx context.Context) ([]*InstanceInfo, error) {
	ctx = mergeOutgoingMetadata(ctx, iac.md)
	req := &btapb.ListInstancesRequest{
		Parent: "projects/" + iac.project,
	}
	res, err := iac.iClient.ListInstances(ctx, req)
	if err != nil {
		return nil, err
	}

	var is []*InstanceInfo
	for _, i := range res.Instances {
		m := instanceNameRegexp.FindStringSubmatch(i.Name)
		if m == nil {
			return nil, fmt.Errorf("malformed instance name %q", i.Name)
		}
		is = append(is, &InstanceInfo{
			Name:        m[2],
			DisplayName: i.DisplayName,
		})
	}
	return is, nil
}

// InstanceInfo returns information about an instance.
func (iac *InstanceAdminClient) InstanceInfo(ctx context.Context, instanceId string) (*InstanceInfo, error) {
	ctx = mergeOutgoingMetadata(ctx, iac.md)
	req := &btapb.GetInstanceRequest{
		Name: "projects/" + iac.project + "/instances/" + instanceId,
	}
	res, err := iac.iClient.GetInstance(ctx, req)
	if err != nil {
		return nil, err
	}

	m := instanceNameRegexp.FindStringSubmatch(res.Name)
	if m == nil {
		return nil, fmt.Errorf("malformed instance name %q", res.Name)
	}
	return &InstanceInfo{
		Name:        m[2],
		DisplayName: res.DisplayName,
	}, nil
}
<|endoftext|>


<commit_before>package remote

import (
	"bytes"
	"testing"
)

// testClient is a generic function to test any client.
func testClient(t *testing.T, c Client) {
	data := []byte("foo")

	if err := c.Put(data); err != nil {
		t.Fatalf("put: %s", err)
	}

	p, err := c.Get()
	if err != nil {
		t.Fatalf("get: %s", err)
	}
	if !bytes.Equal(p.Data, data) {
		t.Fatalf("bad: %#v", p)
	}

	if err := c.Delete(); err != nil {
		t.Fatalf("delete: %s", err)
	}

	p, err = c.Get()
	if err != nil {
		t.Fatalf("get: %s", err)
	}
	if p != nil {
		t.Fatalf("bad: %#v", p)
	}
}
<commit_msg>state/remote: passing Atlas state test<commit_after>package remote

import (
	"bytes"
	"testing"

	"github.com/hashicorp/terraform/state"
	"github.com/hashicorp/terraform/terraform"
)

// testClient is a generic function to test any client.
func testClient(t *testing.T, c Client) {
	var buf bytes.Buffer
	s := state.TestStateInitial()
	if err := terraform.WriteState(s, &buf); err != nil {
		t.Fatalf("err: %s", err)
	}
	data := buf.Bytes()

	if err := c.Put(data); err != nil {
		t.Fatalf("put: %s", err)
	}

	p, err := c.Get()
	if err != nil {
		t.Fatalf("get: %s", err)
	}
	if !bytes.Equal(p.Data, data) {
		t.Fatalf("bad: %#v", p)
	}

	if err := c.Delete(); err != nil {
		t.Fatalf("delete: %s", err)
	}

	p, err = c.Get()
	if err != nil {
		t.Fatalf("get: %s", err)
	}
	if p != nil {
		t.Fatalf("bad: %#v", p)
	}
}
<|endoftext|>


<commit_before>package stringsutil

import (
	"regexp"
)

// PadLeft prepends a string to a base string until the string
// length is greater or equal to the desired length.
func PadLeft(str string, pad string, length int) string {
	for {
		str = pad + str
		if len(str) >= length {
			return str[0:length]
		}
	}
}

// PadRight appends a string to a base string until the string
// length is greater or equal to the desired length.
func PadRight(str string, pad string, length int) string {
	for {
		str += pad
		if len(str) > length {
			return str[0:length]
		}
	}
}

// CondenseString trims whitespace at the ends of the string
// as well as in between.
func CondenseString(content string, join_lines bool) string {
	rx_beg := regexp.MustCompile(`^\s+`)
	rx_end := regexp.MustCompile(`\s+$`)
	rx_mid := regexp.MustCompile(`\n[\s\t\r]*\n`)
	rx_pre := regexp.MustCompile(`\n[\s\t\r]*`)
	rx_spc := regexp.MustCompile(`\s+`)
	content = rx_beg.ReplaceAllString(content, "")
	content = rx_end.ReplaceAllString(content, "")
	content = rx_mid.ReplaceAllString(content, "\n")
	content = rx_pre.ReplaceAllString(content, "\n")
	content = rx_spc.ReplaceAllString(content, " ")
	if join_lines {
		rx_join := regexp.MustCompile(`\n`)
		content = rx_join.ReplaceAllString(content, " ")
	}
	return content
}
<commit_msg>add TrimSentenceLength<commit_after>package stringsutil

import (
	"regexp"
	"strings"
)

// PadLeft prepends a string to a base string until the string
// length is greater or equal to the desired length.
func PadLeft(str string, pad string, length int) string {
	for {
		str = pad + str
		if len(str) >= length {
			return str[0:length]
		}
	}
}

// PadRight appends a string to a base string until the string
// length is greater or equal to the desired length.
func PadRight(str string, pad string, length int) string {
	for {
		str += pad
		if len(str) > length {
			return str[0:length]
		}
	}
}

// CondenseString trims whitespace at the ends of the string
// as well as in between.
func CondenseString(content string, join_lines bool) string {
	rx_beg := regexp.MustCompile(`^\s+`)
	rx_end := regexp.MustCompile(`\s+$`)
	rx_mid := regexp.MustCompile(`\n[\s\t\r]*\n`)
	rx_pre := regexp.MustCompile(`\n[\s\t\r]*`)
	rx_spc := regexp.MustCompile(`\s+`)
	if join_lines {
		rx_join := regexp.MustCompile(`\n`)
		content = rx_join.ReplaceAllString(content, " ")
	}
	content = rx_beg.ReplaceAllString(content, "")
	content = rx_end.ReplaceAllString(content, "")
	content = rx_mid.ReplaceAllString(content, "\n")
	content = rx_pre.ReplaceAllString(content, "\n")
	content = rx_spc.ReplaceAllString(content, " ")
	return strings.TrimSpace(content)
}

func TrimSentenceLength(sentenceInput string, maxLength int) string {
	if len(sentenceInput) <= maxLength {
		return sentenceInput
	}
	sentenceLen := string(sentenceInput[0:maxLength]) // first350 := string(s[0:350])
	rx_end := regexp.MustCompile(`[[:punct:]][^[[:punct:]]]*$`)
	sentencePunct := rx_end.ReplaceAllString(sentenceLen, "")
	if len(sentencePunct) >= 2 {
		return sentencePunct
	}
	return sentenceLen
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/fatih/color"
)

type warning struct {
	message string
	token.Position
}

type visitor struct {
	fileSet *token.FileSet

	lastConstSpec string
	lastFuncDecl  string
	lastTypeSpec  string
	lastVarSpec   string
	warnings      []warning

	lastReceiver     string
	lastReceiverFunc string
}

func (v *visitor) Visit(node ast.Node) ast.Visitor {
	switch typedNode := node.(type) {
	case *ast.File:
		return v
	case *ast.GenDecl:
		if typedNode.Tok == token.CONST {
			v.checkConst(typedNode)
		} else if typedNode.Tok == token.VAR {
			v.checkVar(typedNode)
		}
		return v
	case *ast.FuncDecl:
		v.checkFunc(typedNode)
	case *ast.TypeSpec:
		v.checkType(typedNode)
	}

	return nil
}

func (v *visitor) addWarning(pos token.Pos, message string, subs ...interface{}) {
	coloredSubs := make([]interface{}, len(subs))
	for i, sub := range subs {
		coloredSubs[i] = color.CyanString(sub.(string))
	}

	v.warnings = append(v.warnings, warning{
		message:  fmt.Sprintf(message, coloredSubs...),
		Position: v.fileSet.Position(pos),
	})
}

func (v *visitor) checkConst(node *ast.GenDecl) {
	constName := node.Specs[0].(*ast.ValueSpec).Names[0].Name

	if v.lastFuncDecl != "" {
		v.addWarning(node.Pos(), "constant %s defined after a function declaration", constName)
	}
	if v.lastTypeSpec != "" {
		v.addWarning(node.Pos(), "constant %s defined after a type declaration", constName)
	}
	if v.lastVarSpec != "" {
		v.addWarning(node.Pos(), "constant %s defined after a variable declaration", constName)
	}

	if strings.Compare(constName, v.lastConstSpec) == -1 {
		v.addWarning(node.Pos(), "constant %s defined after constant %s", constName, v.lastConstSpec)
	}

	v.lastConstSpec = constName
}

func (v *visitor) checkFunc(node *ast.FuncDecl) {
	if node.Recv != nil {
		v.checkFuncWithReceiver(node)
	} else {
		funcName := node.Name.Name

		if strings.Compare(funcName, v.lastFuncDecl) == -1 {
			v.addWarning(node.Pos(), "function %s defined after function %s", funcName, v.lastFuncDecl)
		}

		v.lastFuncDecl = funcName
	}
}

func (v *visitor) checkFuncWithReceiver(node *ast.FuncDecl) {
	funcName := node.Name.Name

	var receiver string
	switch typedType := node.Recv.List[0].Type.(type) {
	case *ast.Ident:
		receiver = typedType.Name
	case *ast.StarExpr:
		receiver = typedType.X.(*ast.Ident).Name
	}
	if v.lastFuncDecl != "" {
		v.addWarning(node.Pos(), "method %s.%s defined after function %s", receiver, funcName, v.lastFuncDecl)
	}
	if v.lastTypeSpec != "" && receiver != v.lastTypeSpec {
		v.addWarning(node.Pos(), "method %s.%s should be defined immediately after type %s", receiver, funcName, receiver)
	}
	if receiver == v.lastReceiver {
		if strings.Compare(funcName, v.lastReceiverFunc) == -1 {
			v.addWarning(node.Pos(), "method %s.%s defined after method %s.%s", receiver, funcName, receiver, v.lastReceiverFunc)
		}
	}

	v.lastReceiver = receiver
	v.lastReceiverFunc = funcName
}

func (v *visitor) checkType(node *ast.TypeSpec) {
	typeName := node.Name.Name
	if v.lastFuncDecl != "" {
		v.addWarning(node.Pos(), "type declaration %s defined after a function declaration", typeName)
	}
	v.lastTypeSpec = typeName
}

func (v *visitor) checkVar(node *ast.GenDecl) {
	varName := node.Specs[0].(*ast.ValueSpec).Names[0].Name

	if v.lastFuncDecl != "" {
		v.addWarning(node.Pos(), "variable %s defined after a function declaration", varName)
	}
	if v.lastTypeSpec != "" {
		v.addWarning(node.Pos(), "variable %s defined after a type declaration", varName)
	}

	if strings.Compare(varName, v.lastVarSpec) == -1 {
		v.addWarning(node.Pos(), "variable %s defined after variable %s", varName, v.lastVarSpec)
	}

	v.lastVarSpec = varName
}

func main() {
	var allWarnings []warning

	fileSet := token.NewFileSet()

	err := filepath.Walk(os.Args[1], func(path string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			return nil
		}

		base := filepath.Base(path)
		if base == "vendor" || base == ".git" || strings.HasSuffix(base, "fakes") {
			return filepath.SkipDir
		}

		packages, err := parser.ParseDir(fileSet, path, shouldParseFile, 0)
		if err != nil {
			return err
		}

		var packageNames []string
		for packageName, _ := range packages {
			packageNames = append(packageNames, packageName)
		}
		sort.Strings(packageNames)

		for _, packageName := range packageNames {
			var fileNames []string
			for fileName, _ := range packages[packageName].Files {
				fileNames = append(fileNames, fileName)
			}
			sort.Strings(fileNames)

			for _, fileName := range fileNames {
				v := visitor{
					fileSet: fileSet,
				}
				ast.Walk(&v, packages[packageName].Files[fileName])
				allWarnings = append(allWarnings, v.warnings...)
			}
		}

		return nil
	})

	if err != nil {
		panic(err)
	}

	for _, warning := range allWarnings {
		fmt.Printf("%s +%d %s\n", color.CyanString(warning.Position.Filename), warning.Position.Line, warning.message)
	}

	if len(allWarnings) > 0 {
		os.Exit(1)
	}
}

func shouldParseFile(info os.FileInfo) bool {
	return !strings.HasSuffix(info.Name(), "_test.go")
}
<commit_msg>sort struct fields<commit_after>package main

import (
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/fatih/color"
)

type warning struct {
	message string
	token.Position
}

type visitor struct {
	fileSet *token.FileSet

	lastConstSpec    string
	lastFuncDecl     string
	lastReceiverFunc string
	lastReceiver     string
	lastTypeSpec     string
	lastVarSpec      string

	warnings []warning
}

func (v *visitor) Visit(node ast.Node) ast.Visitor {
	switch typedNode := node.(type) {
	case *ast.File:
		return v
	case *ast.GenDecl:
		if typedNode.Tok == token.CONST {
			v.checkConst(typedNode)
		} else if typedNode.Tok == token.VAR {
			v.checkVar(typedNode)
		}
		return v
	case *ast.FuncDecl:
		v.checkFunc(typedNode)
	case *ast.TypeSpec:
		v.checkType(typedNode)
	}

	return nil
}

func (v *visitor) addWarning(pos token.Pos, message string, subs ...interface{}) {
	coloredSubs := make([]interface{}, len(subs))
	for i, sub := range subs {
		coloredSubs[i] = color.CyanString(sub.(string))
	}

	v.warnings = append(v.warnings, warning{
		message:  fmt.Sprintf(message, coloredSubs...),
		Position: v.fileSet.Position(pos),
	})
}

func (v *visitor) checkConst(node *ast.GenDecl) {
	constName := node.Specs[0].(*ast.ValueSpec).Names[0].Name

	if v.lastFuncDecl != "" {
		v.addWarning(node.Pos(), "constant %s defined after a function declaration", constName)
	}
	if v.lastTypeSpec != "" {
		v.addWarning(node.Pos(), "constant %s defined after a type declaration", constName)
	}
	if v.lastVarSpec != "" {
		v.addWarning(node.Pos(), "constant %s defined after a variable declaration", constName)
	}

	if strings.Compare(constName, v.lastConstSpec) == -1 {
		v.addWarning(node.Pos(), "constant %s defined after constant %s", constName, v.lastConstSpec)
	}

	v.lastConstSpec = constName
}

func (v *visitor) checkFunc(node *ast.FuncDecl) {
	if node.Recv != nil {
		v.checkFuncWithReceiver(node)
	} else {
		funcName := node.Name.Name

		if strings.Compare(funcName, v.lastFuncDecl) == -1 {
			v.addWarning(node.Pos(), "function %s defined after function %s", funcName, v.lastFuncDecl)
		}

		v.lastFuncDecl = funcName
	}
}

func (v *visitor) checkFuncWithReceiver(node *ast.FuncDecl) {
	funcName := node.Name.Name

	var receiver string
	switch typedType := node.Recv.List[0].Type.(type) {
	case *ast.Ident:
		receiver = typedType.Name
	case *ast.StarExpr:
		receiver = typedType.X.(*ast.Ident).Name
	}
	if v.lastFuncDecl != "" {
		v.addWarning(node.Pos(), "method %s.%s defined after function %s", receiver, funcName, v.lastFuncDecl)
	}
	if v.lastTypeSpec != "" && receiver != v.lastTypeSpec {
		v.addWarning(node.Pos(), "method %s.%s should be defined immediately after type %s", receiver, funcName, receiver)
	}
	if receiver == v.lastReceiver {
		if strings.Compare(funcName, v.lastReceiverFunc) == -1 {
			v.addWarning(node.Pos(), "method %s.%s defined after method %s.%s", receiver, funcName, receiver, v.lastReceiverFunc)
		}
	}

	v.lastReceiver = receiver
	v.lastReceiverFunc = funcName
}

func (v *visitor) checkType(node *ast.TypeSpec) {
	typeName := node.Name.Name
	if v.lastFuncDecl != "" {
		v.addWarning(node.Pos(), "type declaration %s defined after a function declaration", typeName)
	}
	v.lastTypeSpec = typeName
}

func (v *visitor) checkVar(node *ast.GenDecl) {
	varName := node.Specs[0].(*ast.ValueSpec).Names[0].Name

	if v.lastFuncDecl != "" {
		v.addWarning(node.Pos(), "variable %s defined after a function declaration", varName)
	}
	if v.lastTypeSpec != "" {
		v.addWarning(node.Pos(), "variable %s defined after a type declaration", varName)
	}

	if strings.Compare(varName, v.lastVarSpec) == -1 {
		v.addWarning(node.Pos(), "variable %s defined after variable %s", varName, v.lastVarSpec)
	}

	v.lastVarSpec = varName
}

func main() {
	var allWarnings []warning

	fileSet := token.NewFileSet()

	err := filepath.Walk(os.Args[1], func(path string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			return nil
		}

		base := filepath.Base(path)
		if base == "vendor" || base == ".git" || strings.HasSuffix(base, "fakes") {
			return filepath.SkipDir
		}

		packages, err := parser.ParseDir(fileSet, path, shouldParseFile, 0)
		if err != nil {
			return err
		}

		var packageNames []string
		for packageName, _ := range packages {
			packageNames = append(packageNames, packageName)
		}
		sort.Strings(packageNames)

		for _, packageName := range packageNames {
			var fileNames []string
			for fileName, _ := range packages[packageName].Files {
				fileNames = append(fileNames, fileName)
			}
			sort.Strings(fileNames)

			for _, fileName := range fileNames {
				v := visitor{
					fileSet: fileSet,
				}
				ast.Walk(&v, packages[packageName].Files[fileName])
				allWarnings = append(allWarnings, v.warnings...)
			}
		}

		return nil
	})

	if err != nil {
		panic(err)
	}

	for _, warning := range allWarnings {
		fmt.Printf("%s +%d %s\n", color.CyanString(warning.Position.Filename), warning.Position.Line, warning.message)
	}

	if len(allWarnings) > 0 {
		os.Exit(1)
	}
}

func shouldParseFile(info os.FileInfo) bool {
	return !strings.HasSuffix(info.Name(), "_test.go")
}
<|endoftext|>


<commit_before>package booklog

import (
	"fmt"
	"net/http"
	"net/url"

	"encoding/json"
	"github.com/pkg/errors"
)

const (
	Status_All int = iota
	Status_WantToRead
	Status_Reading
	Status_Done
	Status_Tsundoku
)

type (
	// Client for booklog
	Client struct {
		url    *url.URL
		client *http.Client
	}

	// GetOptions is parameters for Get method.
	GetOptions struct {
		Count  int // items limit. default value is 5.
		Status int // 0 is all,

	}
	GetResult struct {
		Shelf struct {
			ID       string `json:"id"`
			Account  string `json:"acount"`
			Name     string `json:"name"`
			ImageURL string `json:"image_url"`
		} `json:"tana"`
		Categories []string `json:"category"`
		Books      []Book   `json:"books"`
	}

	Book struct {
		ID      string `json:"id"`
		ASIN    string `json:"asin"`
		URL     string `json:"url"`
		Title   string `json:"title"`
		Author  string `json:"author"`
		Image   string `json:"image"`
		Width   string `json:"width"`
		Height  string `json:"height"`
		Catalog string `json:"catalog"`
	}
)

// NewClient creates a client for booklog.
func NewClient(host string, cli *http.Client) (*Client, error) {
	const errtag = "booklob.NewClient failed"
	u, err := url.ParseRequestURI(host)
	if err != nil {
		return nil, errors.Wrap(err, errtag)
	}
	u.Path = "/json"
	if cli == nil {
		cli = http.DefaultClient
	}
	return &Client{
		url:    u,
		client: cli,
	}, nil
}

// Get returns user's booklogs.
func (c *Client) Get(id string, opts *GetOptions) (GetResult, error) {
	const errtag = "client.Get failed"
	u := *c.url
	u.Path = u.Path + "/" + id
	if opts != nil {
		q := u.Query()
		if opts.Count > 0 {
			q.Set("count", fmt.Sprint(opts.Count))
		}
		if opts.Status > 0 {
			q.Set("status", fmt.Sprint(opts.Status))
		}
		u.RawQuery = q.Encode()
	}
	fmt.Printf(u.String())
	resp, err := c.client.Get(u.String())
	if err != nil {
		return GetResult{}, errors.Wrap(err, errtag)
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return GetResult{}, fmt.Errorf("%s: status code = %d", errtag, resp.StatusCode)
	}
	var r GetResult
	if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
		return GetResult{}, err
	}

	return r, nil
}
<commit_msg>remove printf<commit_after>package booklog

import (
	"fmt"
	"net/http"
	"net/url"

	"encoding/json"
	"github.com/pkg/errors"
)

const (
	Status_All int = iota
	Status_WantToRead
	Status_Reading
	Status_Done
	Status_Tsundoku
)

type (
	// Client for booklog
	Client struct {
		url    *url.URL
		client *http.Client
	}

	// GetOptions is parameters for Get method.
	GetOptions struct {
		Count  int // items limit. default value is 5.
		Status int // 0 is all,

	}
	GetResult struct {
		Shelf struct {
			ID       string `json:"id"`
			Account  string `json:"acount"`
			Name     string `json:"name"`
			ImageURL string `json:"image_url"`
		} `json:"tana"`
		Categories []string `json:"category"`
		Books      []Book   `json:"books"`
	}

	Book struct {
		ID      string `json:"id"`
		ASIN    string `json:"asin"`
		URL     string `json:"url"`
		Title   string `json:"title"`
		Author  string `json:"author"`
		Image   string `json:"image"`
		Width   string `json:"width"`
		Height  string `json:"height"`
		Catalog string `json:"catalog"`
	}
)

// NewClient creates a client for booklog.
func NewClient(host string, cli *http.Client) (*Client, error) {
	const errtag = "booklob.NewClient failed"
	u, err := url.ParseRequestURI(host)
	if err != nil {
		return nil, errors.Wrap(err, errtag)
	}
	u.Path = "/json"
	if cli == nil {
		cli = http.DefaultClient
	}
	return &Client{
		url:    u,
		client: cli,
	}, nil
}

// Get returns user's booklogs.
func (c *Client) Get(id string, opts *GetOptions) (GetResult, error) {
	const errtag = "client.Get failed"
	u := *c.url
	u.Path = u.Path + "/" + id
	if opts != nil {
		q := u.Query()
		if opts.Count > 0 {
			q.Set("count", fmt.Sprint(opts.Count))
		}
		if opts.Status > 0 {
			q.Set("status", fmt.Sprint(opts.Status))
		}
		u.RawQuery = q.Encode()
	}
	resp, err := c.client.Get(u.String())
	if err != nil {
		return GetResult{}, errors.Wrap(err, errtag)
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return GetResult{}, fmt.Errorf("%s: status code = %d", errtag, resp.StatusCode)
	}
	var r GetResult
	if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
		return GetResult{}, err
	}

	return r, nil
}
<|endoftext|>


<commit_before>package history

import (
	"bufio"
	"flag"
	"fmt"
	"github.com/omakoto/zenlog-go/zenlog/config"
	"github.com/omakoto/zenlog-go/zenlog/util"
	"io"
	"os"
	"strings"
)

type LogFileType int

const (
	LogTypeSan LogFileType = iota
	LogTypeRaw
	LogTypeEnv
)

func logChar(logType LogFileType) string {
	switch logType {
	case LogTypeSan:
		return "S"
	case LogTypeRaw:
		return "R"
	case LogTypeEnv:
		return "E"
	}
	util.Fatalf("Unknown type %d", logType)
	return ""
}

func writeIfLink(w *bufio.Writer, filename string) bool {
	util.Debugf("Checking %s", filename)
	to, err := os.Readlink(filename)
	if err != nil {
		return false
	}
	util.Debugf(" -> %s", to)
	w.WriteString(to)
	w.WriteString("\n")
	return true
}

func history(pid, nth int, logType LogFileType, writer io.Writer) bool {
	if nth < 0 {
		util.Fatalf("Invalid argument for nth: %d", nth)
	}
	config := config.InitConfigForCommands()

	if pid <= 0 {
		pid = config.ZenlogPid
	}
	dir := fmt.Sprintf("%spids/%d/", config.LogDir, pid)
	w := bufio.NewWriter(writer)
	defer w.Flush()

	ch := logChar(logType)

	success := false
	if nth > 0 {
		success = writeIfLink(w, dir+strings.Repeat(ch, nth))
	} else {
		for i := 1; i <= 10; i++ {
			success = writeIfLink(w, dir+strings.Repeat(ch, i)) || success
		}
	}
	return success
}

func flagsToLogType(flagR, flagE bool) LogFileType {
	if flagR {
		return LogTypeRaw
	}
	if flagE {
		return LogTypeEnv
	}
	return LogTypeSan
}

func HistoryCommand(args []string) {
	flags := flag.NewFlagSet("zenlog history", flag.ExitOnError)
	r := flags.Bool("r", false, "Print RAW filename")
	e := flags.Bool("e", false, "Print ENV filename")
	n := flags.Int("n", 0, "Print nth (>=1) last filename")
	p := flags.Int("p", 0, "Specify ZENLOG_PID")

	flags.Parse(args)

	util.Exit(history(*p, *n, flagsToLogType(*r, *e), os.Stdout))
}

func CurrentLogCommand(args []string) {
	flags := flag.NewFlagSet("zenlog current-log", flag.ExitOnError)
	r := flags.Bool("r", false, "Print RAW filename")
	e := flags.Bool("e", false, "Print ENV filename")
	p := flags.Int("p", 0, "Specify ZENLOG_PID")

	flags.Parse(args)

	util.Exit(history(*p, 1, flagsToLogType(*r, *e), os.Stdout))
}

func LastLogCommand(args []string) {
	flags := flag.NewFlagSet("zenlog last-log", flag.ExitOnError)
	r := flags.Bool("r", false, "Print RAW filename")
	e := flags.Bool("e", false, "Print ENV filename")
	p := flags.Int("p", 0, "Specify ZENLOG_PID")

	flags.Parse(args)

	util.Exit(history(*p, 2, flagsToLogType(*r, *e), os.Stdout))
}
<commit_msg>Compatibility fix.<commit_after>package history

import (
	"bufio"
	"flag"
	"fmt"
	"github.com/omakoto/zenlog-go/zenlog/config"
	"github.com/omakoto/zenlog-go/zenlog/util"
	"io"
	"os"
	"strings"
)

type LogFileType int

const (
	LogTypeSan LogFileType = iota
	LogTypeRaw
	LogTypeEnv
)

func logChar(logType LogFileType) string {
	switch logType {
	case LogTypeSan:
		return "S"
	case LogTypeRaw:
		return "R"
	case LogTypeEnv:
		return "E"
	}
	util.Fatalf("Unknown type %d", logType)
	return ""
}

func writeIfLink(w *bufio.Writer, filename string) bool {
	util.Debugf("Checking %s", filename)
	to, err := os.Readlink(filename)
	if err != nil {
		return false
	}
	util.Debugf(" -> %s", to)
	w.WriteString(to)
	w.WriteString("\n")
	return true
}

func history(pid, nth int, logType LogFileType, writer io.Writer) bool {
	if nth < 0 {
		util.Fatalf("Invalid argument for nth: %d", nth)
	}
	config := config.InitConfigForCommands()

	if pid <= 0 {
		pid = config.ZenlogPid
	}
	dir := fmt.Sprintf("%spids/%d/", config.LogDir, pid)
	w := bufio.NewWriter(writer)
	defer w.Flush()

	ch := logChar(logType)

	success := false
	if nth > 0 {
		success = writeIfLink(w, dir+strings.Repeat(ch, nth))
	} else {
		for i := 10; i >= 1; i-- {
			success = writeIfLink(w, dir+strings.Repeat(ch, i)) || success
		}
	}
	return success
}

func flagsToLogType(flagR, flagE bool) LogFileType {
	if flagR {
		return LogTypeRaw
	}
	if flagE {
		return LogTypeEnv
	}
	return LogTypeSan
}

func HistoryCommand(args []string) {
	flags := flag.NewFlagSet("zenlog history", flag.ExitOnError)
	r := flags.Bool("r", false, "Print RAW filename")
	e := flags.Bool("e", false, "Print ENV filename")
	n := flags.Int("n", 0, "Print nth (>=1) last filename")
	p := flags.Int("p", 0, "Specify ZENLOG_PID")

	flags.Parse(args)

	util.Exit(history(*p, *n, flagsToLogType(*r, *e), os.Stdout))
}

func CurrentLogCommand(args []string) {
	flags := flag.NewFlagSet("zenlog current-log", flag.ExitOnError)
	r := flags.Bool("r", false, "Print RAW filename")
	e := flags.Bool("e", false, "Print ENV filename")
	p := flags.Int("p", 0, "Specify ZENLOG_PID")

	flags.Parse(args)

	util.Exit(history(*p, 1, flagsToLogType(*r, *e), os.Stdout))
}

func LastLogCommand(args []string) {
	flags := flag.NewFlagSet("zenlog last-log", flag.ExitOnError)
	r := flags.Bool("r", false, "Print RAW filename")
	e := flags.Bool("e", false, "Print ENV filename")
	p := flags.Int("p", 0, "Specify ZENLOG_PID")

	flags.Parse(args)

	util.Exit(history(*p, 2, flagsToLogType(*r, *e), os.Stdout))
}
<|endoftext|>